surfliner-metadata_consumer 0.1.0.pre.alpha.4 → 0.1.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +46 -1
- data/Dockerfile +1 -1
- data/Gemfile.lock +35 -29
- data/README.md +61 -22
- data/bin/index-on-publish +33 -0
- data/bin/simulate-publish-event +30 -20
- data/lib/surfliner/metadata_consumer/consumer.rb +41 -17
- data/lib/surfliner/metadata_consumer/version.rb +1 -2
- data/lib/surfliner/metadata_consumer.rb +4 -1
- data/lib/surfliner/mq/connection.rb +135 -0
- data/lib/surfliner/mq/connection_config.rb +83 -0
- data/lib/surfliner/mq/queue_config.rb +49 -0
- data/lib/surfliner/mq/topic.rb +65 -0
- data/lib/surfliner/mq/topic_config.rb +37 -0
- data/lib/surfliner/mq.rb +4 -0
- data/lib/surfliner/util/assert.rb +21 -0
- data/lib/surfliner/util.rb +4 -0
- metadata +64 -30
- data/bin/daylight-index-listen +0 -23
- data/lib/surfliner/metadata_consumer/mq_config.rb +0 -79
- data/lib/surfliner/metadata_consumer/mq_connection.rb +0 -136
@@ -1,79 +0,0 @@
|
|
1
|
-
module Surfliner
|
2
|
-
module MetadataConsumer
|
3
|
-
# An object encapsulating RabbitMQ configuration.
|
4
|
-
class MqConfig
|
5
|
-
# @return [String] The RabbitMQ hostname
|
6
|
-
attr_reader :host
|
7
|
-
|
8
|
-
# @return [String] The RabbitMQ AMQP port
|
9
|
-
attr_reader :port
|
10
|
-
|
11
|
-
# @return [String] The RabbitMQ username
|
12
|
-
attr_reader :username
|
13
|
-
|
14
|
-
# @return [String] The RabbitMQ passsword
|
15
|
-
attr_reader :password
|
16
|
-
|
17
|
-
# @return [String] The topic exchange to listen to
|
18
|
-
attr_reader :topic
|
19
|
-
|
20
|
-
# @return [String] The name of the queue to listen to
|
21
|
-
attr_reader :queue_name
|
22
|
-
|
23
|
-
# @return [String] The platform routing key to listen to
|
24
|
-
attr_reader :routing_key
|
25
|
-
|
26
|
-
# Initializes a new `MqConfig` object.
|
27
|
-
# @param host [The] RabbitMQ hostname
|
28
|
-
# @param port [The] RabbitMQ AMQP port
|
29
|
-
# @param username [The] RabbitMQ username
|
30
|
-
# @param password [The] RabbitMQ passsword
|
31
|
-
# @param topic [The] topic exchange to listen to
|
32
|
-
# @param queue_name [The] name of the queue to listen to
|
33
|
-
# @param routing_key [The] platform routing key to listen to
|
34
|
-
def initialize(host:, port:, username:, password:, topic:, queue_name:, routing_key:)
|
35
|
-
@host = host
|
36
|
-
@port = port
|
37
|
-
@username = username
|
38
|
-
@password = password
|
39
|
-
@topic = topic
|
40
|
-
@queue_name = queue_name
|
41
|
-
@routing_key = routing_key
|
42
|
-
end
|
43
|
-
|
44
|
-
class << self
|
45
|
-
# Reads RabbitMQ configuration from environment variables and
|
46
|
-
# returns it as a new `MqConfig` object.
|
47
|
-
#
|
48
|
-
# - `RABBITMQ_HOST` → `host`
|
49
|
-
# - `RABBITMQ_NODE_PORT_NUMBER` → `port`
|
50
|
-
# - `RABBITMQ_USERNAME` → `username`
|
51
|
-
# - `RABBITMQ_PASSWORD` → `password`
|
52
|
-
# - `RABBITMQ_TOPIC` → `topic`
|
53
|
-
# - `RABBITMQ_QUEUE` → `queue_name`
|
54
|
-
# - `RABBITMQ_PLATFORM_ROUTING_KEY` → `routing_key`
|
55
|
-
def from_env
|
56
|
-
MqConfig.new(
|
57
|
-
host: ENV.fetch("RABBITMQ_HOST"),
|
58
|
-
port: ENV.fetch("RABBITMQ_NODE_PORT_NUMBER"),
|
59
|
-
username: ENV.fetch("RABBITMQ_USERNAME"),
|
60
|
-
password: ENV.fetch("RABBITMQ_PASSWORD"),
|
61
|
-
topic: ENV.fetch("RABBITMQ_TOPIC"),
|
62
|
-
queue_name: ENV.fetch("RABBITMQ_QUEUE"),
|
63
|
-
routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
|
64
|
-
)
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# @return [String] the connection URL as a string
|
69
|
-
def connection_url
|
70
|
-
@connection_url ||= "amqp://#{username}:#{password}@#{host}:#{port}"
|
71
|
-
end
|
72
|
-
|
73
|
-
# @return [String] the connection URL as a string, without the password
|
74
|
-
def redacted_url
|
75
|
-
@redacted_url ||= connection_url.sub(password, "REDACTED")
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
require "bunny"
|
2
|
-
|
3
|
-
module Surfliner
|
4
|
-
module MetadataConsumer
|
5
|
-
# An object encapsulating a RabbitMQ connection.
|
6
|
-
class MqConnection
|
7
|
-
# @return [Logger] The logger
|
8
|
-
attr_reader :logger
|
9
|
-
|
10
|
-
# @return [Bunny::Session] The current RabbitMQ session
|
11
|
-
attr_reader :connection
|
12
|
-
|
13
|
-
# @return [Bunny::Channel] The channel being listened to
|
14
|
-
attr_reader :channel
|
15
|
-
|
16
|
-
# @return [Bunny::Exchange] The exchange being listened to
|
17
|
-
attr_reader :exchange
|
18
|
-
|
19
|
-
# @return [Bunny::Queue] The queue being listened to
|
20
|
-
attr_reader :queue
|
21
|
-
|
22
|
-
# @return [MqConfig] The configuration
|
23
|
-
attr_reader :config
|
24
|
-
|
25
|
-
# Initializes a new `MqConnection`.
|
26
|
-
#
|
27
|
-
# @param logger [Logger] the logger
|
28
|
-
# @param config [MqConfig] the configuration
|
29
|
-
def initialize(logger:, config: MqConfig.from_env)
|
30
|
-
@logger = logger
|
31
|
-
@config = config
|
32
|
-
end
|
33
|
-
|
34
|
-
# Opens a connection.
|
35
|
-
# @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
|
36
|
-
# @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
|
37
|
-
# @return [self]
|
38
|
-
# @raise RuntimeError if already connected
|
39
|
-
def connect(topic_opts: {}, queue_opts: {})
|
40
|
-
raise "RabbitMQ connection #{connection} already open." if open?
|
41
|
-
|
42
|
-
logger.info("Rabbitmq message broker connection url: #{config.redacted_url}")
|
43
|
-
@connection = Bunny.new(config.connection_url, logger: logger)
|
44
|
-
connect_on(connection)
|
45
|
-
@channel = connection.create_channel
|
46
|
-
@exchange = channel.topic(config.topic, topic_opts)
|
47
|
-
@queue = channel.queue(config.queue_name, queue_opts)
|
48
|
-
queue.bind(exchange, routing_key: config.routing_key)
|
49
|
-
|
50
|
-
self
|
51
|
-
rescue Bunny::TCPConnectionFailed => err
|
52
|
-
# TODO: realistically, this only happens in connection.start, where we're eating it
|
53
|
-
logger.error("Connection to #{config.redacted_url} failed")
|
54
|
-
raise err
|
55
|
-
rescue Bunny::PossibleAuthenticationFailureError => err
|
56
|
-
# TODO: realistically, this only happens in connection.start, where we're eating it
|
57
|
-
logger.error("Failed to authenticate to #{config.redacted_url}")
|
58
|
-
raise err
|
59
|
-
end
|
60
|
-
|
61
|
-
# Opens a connection, yields the queue, and closes the connection after
|
62
|
-
# the provided block completes.
|
63
|
-
# @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
|
64
|
-
# @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
|
65
|
-
# @yield [Bunny::Queue] the queue
|
66
|
-
def open(topic_opts: {}, queue_opts: {})
|
67
|
-
connect(topic_opts:, queue_opts:)
|
68
|
-
yield queue
|
69
|
-
ensure
|
70
|
-
close
|
71
|
-
end
|
72
|
-
|
73
|
-
# Closes the connection.
|
74
|
-
def close
|
75
|
-
return unless channel
|
76
|
-
return if channel.closed?
|
77
|
-
logger.info("closing channel")
|
78
|
-
channel.close
|
79
|
-
ensure
|
80
|
-
logger.info("closing connection")
|
81
|
-
connection&.close
|
82
|
-
end
|
83
|
-
|
84
|
-
# @return [true, false] True if the connection is open, false otherwise
|
85
|
-
def open?
|
86
|
-
connection&.status == :open
|
87
|
-
end
|
88
|
-
|
89
|
-
# @return [Symbol, nil] The connection status, or nil if there is no connection
|
90
|
-
def status
|
91
|
-
connection&.status
|
92
|
-
end
|
93
|
-
|
94
|
-
# @return [String] The RabbitMQ hostname
|
95
|
-
def host
|
96
|
-
config.host
|
97
|
-
end
|
98
|
-
|
99
|
-
# @return [String] The RabbitMQ port
|
100
|
-
def port
|
101
|
-
config.port
|
102
|
-
end
|
103
|
-
|
104
|
-
# @return [String] The routing key
|
105
|
-
def routing_key
|
106
|
-
config.routing_key
|
107
|
-
end
|
108
|
-
|
109
|
-
# Publishes the specified payload
|
110
|
-
# @param payload [String] the payload to publish
|
111
|
-
# @return [Bunny::Exchange] see #exchange
|
112
|
-
def publish(payload)
|
113
|
-
logger.info "Publishing to #{routing_key} with payload: #{payload}"
|
114
|
-
exchange.publish(payload, routing_key:)
|
115
|
-
end
|
116
|
-
|
117
|
-
private
|
118
|
-
|
119
|
-
def connect_on(connection, timeout = 120)
|
120
|
-
timer = 0
|
121
|
-
logger.info "Trying to open queue connection with timeout=#{timeout}"
|
122
|
-
while timer < timeout
|
123
|
-
begin
|
124
|
-
connection.start
|
125
|
-
rescue
|
126
|
-
# TODO: do we actually want to rescue from everything?
|
127
|
-
end
|
128
|
-
return connection if connection.status == :open
|
129
|
-
sleep 1
|
130
|
-
timer += 1
|
131
|
-
end
|
132
|
-
raise "Failed to connect to queue."
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|