surfliner-metadata_consumer 0.1.0.pre.alpha.3 → 0.1.0.pre.alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +38 -1
- data/Dockerfile +1 -1
- data/Gemfile.lock +31 -23
- data/README.md +60 -22
- data/bin/index-on-publish +33 -0
- data/bin/simulate-publish-event +30 -20
- data/lib/surfliner/metadata_consumer/consumer.rb +41 -17
- data/lib/surfliner/metadata_consumer/version.rb +1 -2
- data/lib/surfliner/metadata_consumer.rb +4 -1
- data/lib/surfliner/mq/connection.rb +127 -0
- data/lib/surfliner/mq/connection_config.rb +62 -0
- data/lib/surfliner/mq/queue_config.rb +49 -0
- data/lib/surfliner/mq/topic.rb +65 -0
- data/lib/surfliner/mq/topic_config.rb +37 -0
- data/lib/surfliner/mq.rb +4 -0
- data/lib/surfliner/util/assert.rb +21 -0
- data/lib/surfliner/util.rb +4 -0
- metadata +62 -28
- data/bin/daylight-index-listen +0 -23
- data/lib/surfliner/metadata_consumer/mq_config.rb +0 -79
- data/lib/surfliner/metadata_consumer/mq_connection.rb +0 -136
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fff0fe7a29854ecb3b75f91ed5aa1e5abdbf6fe7fc625b4b117a3e2c4e4805c
|
4
|
+
data.tar.gz: 534536637b909d309f90095aaba4e9cb51a75d4ca92a78531971d3550ea42bfc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6e20f325aa14042b5ed7e950803d8a045834df543da1f5c4a8f30297645015d97add22a5de38e01d15a6ab74adb856777bba4fccfb609d50bd622c9fae1f51d5
|
7
|
+
data.tar.gz: a69109802f8c408f077b2fc2d1c748c02a900eda042d6e40a4423ee3257b39d2dde892eecf9c77c690df06aa53d367136a8246f3d8bc69114d0e1278326b4531
|
data/CHANGES.md
CHANGED
@@ -1,3 +1,40 @@
|
|
1
|
+
# 0.1.0.pre.alpha.5 (2025-03-25)
|
2
|
+
|
3
|
+
In order to support using one connection (and one underlying `Bunny::Session`)
|
4
|
+
with multiple topics, queues, and/or routing keys, as well as publishing to
|
5
|
+
a topic without creating or subscribing to a queue, this release introduces
|
6
|
+
the following **breaking changes**:
|
7
|
+
|
8
|
+
- RabbitMQ-related configuration and connection code (i.e. not specific to
|
9
|
+
the consumer/handler setup) has been moved from `Surfliner::MetadataConsumer`
|
10
|
+
to `Surfliner::Mq`. This may move to a separate gem in a future release.
|
11
|
+
- `MetadataConsumer::MqConnection` is now `Mq::Connection`
|
12
|
+
- `MetadataConsumer::MqConfig` is now `Mq::ConnectionConfig`
|
13
|
+
- `ConnectionConfig` no longer includes topic or queue configuration;
|
14
|
+
topics are configured with `TopicConfig` and queues with
|
15
|
+
`QueueConfig`.
|
16
|
+
- The `MqConnection#open` method, which yielded a `Bunny::Queue`, has been
|
17
|
+
replaced by the `Connection#with_topic` method, which yields an
|
18
|
+
`Mq::Topic` object wrapping a single topic; call `Mq::Topic#bind_queue`
|
19
|
+
to bind to a queue. Like the removed`#open`, `#with_topic` opens and
|
20
|
+
closes the connection implicitly and allows connecting only to a single
|
21
|
+
topic.
|
22
|
+
- The `Mq::Connection#topic_from(config)` method allows you to connect to
|
23
|
+
any number of topics, but you must open and close the connection
|
24
|
+
explicitly.
|
25
|
+
- The `MqConnection#publish` method has been moved to `Mq::Topic` and can
|
26
|
+
now take an explicit routing key, although if one is not provided it will
|
27
|
+
still default to `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`.
|
28
|
+
- `MetadataConsumer::Consumer` now requires an explicit `Mq:Connection`
|
29
|
+
instead of creating one by default.
|
30
|
+
|
31
|
+
In addition, the `daylight-index-listen` script has been renamed to the more
|
32
|
+
generic `index-on-publish`.
|
33
|
+
|
34
|
+
# 0.1.0.pre.alpha.4 (2025-03-17)
|
35
|
+
|
36
|
+
- lower required Ruby version to 3.2
|
37
|
+
|
1
38
|
# 0.1.0.pre.alpha.3 (2025-03-07)
|
2
39
|
|
3
40
|
- improve topic/queue option override API
|
@@ -15,4 +52,4 @@
|
|
15
52
|
|
16
53
|
# 0.1.0.pre.alpha (2025-02-12)
|
17
54
|
|
18
|
-
- Initial release
|
55
|
+
- Initial release
|
data/Dockerfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
surfliner-metadata_consumer (0.1.0.pre.alpha.
|
4
|
+
surfliner-metadata_consumer (0.1.0.pre.alpha.5)
|
5
5
|
bunny (~> 2.23)
|
6
6
|
opentelemetry-exporter-otlp (~> 0.26.3)
|
7
7
|
opentelemetry-instrumentation-all (~> 0.60.0)
|
@@ -14,11 +14,11 @@ GEM
|
|
14
14
|
addressable (2.8.7)
|
15
15
|
public_suffix (>= 2.0.2, < 7.0)
|
16
16
|
amq-protocol (2.3.3)
|
17
|
-
ast (2.4.
|
17
|
+
ast (2.4.3)
|
18
18
|
bigdecimal (3.1.9)
|
19
19
|
builder (3.3.0)
|
20
|
-
bunny (2.
|
21
|
-
amq-protocol (~> 2.3
|
20
|
+
bunny (2.24.0)
|
21
|
+
amq-protocol (~> 2.3)
|
22
22
|
sorted_set (~> 1, >= 1.0.2)
|
23
23
|
ci_reporter (2.1.0)
|
24
24
|
builder (>= 2.1.2)
|
@@ -34,7 +34,7 @@ GEM
|
|
34
34
|
debug (1.9.2)
|
35
35
|
irb (~> 1.10)
|
36
36
|
reline (>= 0.3.8)
|
37
|
-
diff-lcs (1.6.
|
37
|
+
diff-lcs (1.6.1)
|
38
38
|
docile (1.4.1)
|
39
39
|
dotenv (2.8.1)
|
40
40
|
faraday (2.12.2)
|
@@ -43,8 +43,9 @@ GEM
|
|
43
43
|
logger
|
44
44
|
faraday-net_http (3.4.0)
|
45
45
|
net-http (>= 0.5.0)
|
46
|
+
github-markup (5.0.1)
|
46
47
|
google-protobuf (3.25.6)
|
47
|
-
googleapis-common-protos-types (1.
|
48
|
+
googleapis-common-protos-types (1.19.0)
|
48
49
|
google-protobuf (>= 3.18, < 5.a)
|
49
50
|
hashdiff (1.1.2)
|
50
51
|
io-console (0.8.0)
|
@@ -52,7 +53,7 @@ GEM
|
|
52
53
|
pp (>= 0.6.0)
|
53
54
|
rdoc (>= 4.0.0)
|
54
55
|
reline (>= 0.4.2)
|
55
|
-
json (2.10.
|
56
|
+
json (2.10.2)
|
56
57
|
language_server-protocol (3.17.0.4)
|
57
58
|
lint_roller (1.1.0)
|
58
59
|
logger (1.6.6)
|
@@ -255,12 +256,13 @@ GEM
|
|
255
256
|
opentelemetry-semantic_conventions (1.11.0)
|
256
257
|
opentelemetry-api (~> 1.0)
|
257
258
|
parallel (1.26.3)
|
258
|
-
parser (3.3.7.
|
259
|
+
parser (3.3.7.2)
|
259
260
|
ast (~> 2.4.1)
|
260
261
|
racc
|
261
262
|
pp (0.6.2)
|
262
263
|
prettyprint
|
263
264
|
prettyprint (0.2.0)
|
265
|
+
prism (1.4.0)
|
264
266
|
psych (5.2.3)
|
265
267
|
date
|
266
268
|
stringio
|
@@ -269,8 +271,9 @@ GEM
|
|
269
271
|
rainbow (3.1.1)
|
270
272
|
rake (13.2.1)
|
271
273
|
rbtree (0.4.6)
|
272
|
-
rdoc (6.
|
274
|
+
rdoc (6.13.0)
|
273
275
|
psych (>= 4.0.0)
|
276
|
+
redcarpet (3.6.1)
|
274
277
|
regexp_parser (2.10.0)
|
275
278
|
reline (0.6.0)
|
276
279
|
io-console (~> 0.5)
|
@@ -291,9 +294,10 @@ GEM
|
|
291
294
|
diff-lcs (>= 1.2.0, < 2.0)
|
292
295
|
rspec-support (~> 3.13.0)
|
293
296
|
rspec-support (3.13.2)
|
294
|
-
rubocop (1.
|
297
|
+
rubocop (1.73.2)
|
295
298
|
json (~> 2.3)
|
296
|
-
language_server-protocol (
|
299
|
+
language_server-protocol (~> 3.17.0.2)
|
300
|
+
lint_roller (~> 1.1.0)
|
297
301
|
parallel (~> 1.10)
|
298
302
|
parser (>= 3.3.0.2)
|
299
303
|
rainbow (>= 2.2.2, < 4.0)
|
@@ -301,11 +305,13 @@ GEM
|
|
301
305
|
rubocop-ast (>= 1.38.0, < 2.0)
|
302
306
|
ruby-progressbar (~> 1.7)
|
303
307
|
unicode-display_width (>= 2.4.0, < 4.0)
|
304
|
-
rubocop-ast (1.
|
305
|
-
parser (>= 3.3.
|
306
|
-
|
307
|
-
|
308
|
-
|
308
|
+
rubocop-ast (1.43.0)
|
309
|
+
parser (>= 3.3.7.2)
|
310
|
+
prism (~> 1.4)
|
311
|
+
rubocop-performance (1.24.0)
|
312
|
+
lint_roller (~> 1.1)
|
313
|
+
rubocop (>= 1.72.1, < 2.0)
|
314
|
+
rubocop-ast (>= 1.38.0, < 2.0)
|
309
315
|
ruby-progressbar (1.13.0)
|
310
316
|
set (1.1.1)
|
311
317
|
simplecov (0.22.0)
|
@@ -320,24 +326,24 @@ GEM
|
|
320
326
|
sorted_set (1.0.3)
|
321
327
|
rbtree
|
322
328
|
set (~> 1.0)
|
323
|
-
standard (1.
|
329
|
+
standard (1.47.0)
|
324
330
|
language_server-protocol (~> 3.17.0.2)
|
325
331
|
lint_roller (~> 1.0)
|
326
|
-
rubocop (~> 1.
|
332
|
+
rubocop (~> 1.73.0)
|
327
333
|
standard-custom (~> 1.0.0)
|
328
|
-
standard-performance (~> 1.
|
334
|
+
standard-performance (~> 1.7)
|
329
335
|
standard-custom (1.0.2)
|
330
336
|
lint_roller (~> 1.0)
|
331
337
|
rubocop (~> 1.50)
|
332
|
-
standard-performance (1.
|
338
|
+
standard-performance (1.7.0)
|
333
339
|
lint_roller (~> 1.1)
|
334
|
-
rubocop-performance (~> 1.
|
335
|
-
stringio (3.1.
|
340
|
+
rubocop-performance (~> 1.24.0)
|
341
|
+
stringio (3.1.6)
|
336
342
|
unicode-display_width (3.1.4)
|
337
343
|
unicode-emoji (~> 4.0, >= 4.0.4)
|
338
344
|
unicode-emoji (4.0.4)
|
339
345
|
uri (1.0.3)
|
340
|
-
webmock (3.25.
|
346
|
+
webmock (3.25.1)
|
341
347
|
addressable (>= 2.8.0)
|
342
348
|
crack (>= 0.3.2)
|
343
349
|
hashdiff (>= 0.4.0, < 2.0.0)
|
@@ -352,7 +358,9 @@ DEPENDENCIES
|
|
352
358
|
colorize (~> 0.8)
|
353
359
|
debug (~> 1.9.2)
|
354
360
|
dotenv (~> 2.7)
|
361
|
+
github-markup (~> 5.0)
|
355
362
|
rake (~> 13.0)
|
363
|
+
redcarpet (~> 3.6)
|
356
364
|
rspec (~> 3.13)
|
357
365
|
simplecov (~> 0.22)
|
358
366
|
simplecov-cobertura (~> 2.1)
|
data/README.md
CHANGED
@@ -36,23 +36,59 @@ if prior steps succeed.
|
|
36
36
|
|
37
37
|
## Configuration
|
38
38
|
|
39
|
-
|
40
|
-
|
41
|
-
|
39
|
+
> Note: Sample values taken from Surfliner's development
|
40
|
+
> [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
|
41
|
+
|
42
|
+
An `Mq::Connection` object, suitable for listening or publishing, can be configured
|
43
|
+
explicitly with a `ConnectionConfig` object, or implicitly, reading a default
|
44
|
+
configuration with `ConnectionConfig#from_env`. `ConnectionConfig#from_env` expects
|
45
|
+
the following environment variables:
|
46
|
+
|
47
|
+
| Variable | Sample value | Description |
|
48
|
+
|-----------------------------|--------------|------------------------------|
|
49
|
+
| `RABBITMQ_HOST` | `rabbitmq` | Hostname of RabbitMQ server |
|
50
|
+
| `RABBITMQ_NODE_PORT_NUMBER` | `5672` | Port name of RabbitMQ server |
|
51
|
+
| `RABBITMQ_USERNAME` | `user` | RabbitMQ username |
|
52
|
+
| `RABBITMQ_PASSWORD` | `bitnami` | RabbitMQ password |
|
53
|
+
|
54
|
+
The `Mq::Connection#with_topic` and `Mq::Connection#topic_from` can either take an
|
55
|
+
explicit `TopicConfig` object, or implicitly read a default configuration with
|
56
|
+
`TopicConfig#from_env`. `TopicConfig#from_env` expects the following environment variables:
|
57
|
+
|
58
|
+
| Variable | Sample value | Description |
|
59
|
+
|------------------|----------------------|---------------------|
|
60
|
+
| `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
|
61
|
+
|
62
|
+
Note that `TopicConfig#from_env` also accepts keyword options, which are passed through
|
63
|
+
to [`Bunny::Channel#topic`](https://api.rubybunny.info/Bunny/Channel.html#topic-instance_method).
|
64
|
+
E.g.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
topic_config = TopicConfig.from_env(durable: true, auto_delete: true)
|
68
|
+
connection.with_topic(topic_config) do |topic|
|
69
|
+
# ...
|
70
|
+
end
|
71
|
+
```
|
42
72
|
|
43
|
-
|
73
|
+
Similarly, `Mq::Topic#bind_queue` can either take an explicit `QueueConfig`, or implicitly
|
74
|
+
read a default with `QueConfig#from_env`. `QueueConfig#from_env` expects the following environment
|
75
|
+
variables:
|
44
76
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
RABBITMQ_QUEUE=surfliner.metadata
|
50
|
-
RABBITMQ_PLATFORM_ROUTING_KEY=surfliner.metadata.daylight
|
51
|
-
RABBITMQ_TOPIC=surfliner.metadata
|
52
|
-
RABBITMQ_USERNAME=user
|
53
|
-
```
|
77
|
+
| Variable | Sample value | Description |
|
78
|
+
|---------------------------------|-------------------------------|----------------------|
|
79
|
+
| `RABBITMQ_QUEUE` | `surfliner.metadata` | RabbitMQ queue name |
|
80
|
+
| `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
|
54
81
|
|
55
|
-
|
82
|
+
And `QueueConfig#from_env` similarly accepts keyword options, which are forwarded to
|
83
|
+
[`Bunny::Channel#queue`](https://api.rubybunny.info/Bunny/Channel.html#queue-instance_method):
|
84
|
+
|
85
|
+
```ruby
|
86
|
+
connection.with_topic(topic_config) do |topic|
|
87
|
+
queue_config = QueueConfig.from_env(exclusive: true, durable: true)
|
88
|
+
queue = topic.bind_queue(queue_config)
|
89
|
+
# ...
|
90
|
+
end
|
91
|
+
```
|
56
92
|
|
57
93
|
The Solr / Daylight handler implementation (see below) additionally expects
|
58
94
|
a configured `SOLR_URL`, e.g.:
|
@@ -74,17 +110,19 @@ The `Surfliner::MetadataConsumer::Solr` package contains a handler that retrieve
|
|
74
110
|
metadata for `:published`/`:updated` events, transforms results for indexing, and
|
75
111
|
updates a target Solr index.
|
76
112
|
|
77
|
-
|
113
|
+
The `bin/index-on-publish` script starts a consumer using this handler, and accepts
|
114
|
+
(but does not require) the following environment variables:
|
78
115
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
116
|
+
| Variable | Sample value | Default | Description |
|
117
|
+
|---------------------|-------------------------------|----------------|----------------------------------|
|
118
|
+
| `OTEL_SDK_DISABLED` | `true` | (not set) | Whether to disable OpenTelemetry |
|
119
|
+
| `OTEL_SERVICE_NAME` | `surfliner-daylight-consumer` | name of script | OpenTelemetry service name |
|
120
|
+
| `OTEL_TRACER_NAME` | `DaylightConsumerTracer` | name of script | OpenTelemetry tracer name |
|
83
121
|
|
84
122
|
### Running the service
|
85
123
|
|
86
124
|
<!--
|
87
|
-
TODO: Dockerfile shouldn't be
|
125
|
+
TODO: Dockerfile shouldn't be index-on-publish specific
|
88
126
|
-->
|
89
127
|
|
90
128
|
Build the listener Dockerfile and tag it with the name `solr-listener`:
|
@@ -101,5 +139,5 @@ docker run --env-file <env-file> solr-listener
|
|
101
139
|
|
102
140
|
## Utility scripts
|
103
141
|
|
104
|
-
- `bin/
|
105
|
-
- `bin/simulate-publish-event` posts a publish event to the queue configured with `
|
142
|
+
- `bin/index-on-publish` script starts a consumer using `MetadataConsumer::Solr::MessageHandler`.
|
143
|
+
- `bin/simulate-publish-event` posts a publish event to the queue configured with `Mq::Connection`
|
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require "bundler/setup"
|
3
|
+
require "logger"
|
4
|
+
|
5
|
+
require "surfliner/metadata_consumer"
|
6
|
+
require "surfliner/mq"
|
7
|
+
|
8
|
+
$stdout.sync = true # don't buffer log output
|
9
|
+
logger = Logger.new($stdout).tap do |logger|
|
10
|
+
logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
|
11
|
+
end
|
12
|
+
|
13
|
+
unless ENV["OTEL_SDK_DISABLED"] == "true"
|
14
|
+
require "opentelemetry/sdk"
|
15
|
+
require "opentelemetry-exporter-otlp"
|
16
|
+
|
17
|
+
otel_service_name = ENV.fetch('OTEL_SERVICE_NAME', File.basename(__FILE__))
|
18
|
+
otel_tracer_name = ENV.fetch('OTEL_TRACER_NAME', File.basename(__FILE__))
|
19
|
+
|
20
|
+
OpenTelemetry::SDK.configure do |c|
|
21
|
+
c.service_name = otel_service_name
|
22
|
+
c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
|
23
|
+
end
|
24
|
+
|
25
|
+
tracer = OpenTelemetry.tracer_provider.tracer(otel_tracer_name)
|
26
|
+
end
|
27
|
+
|
28
|
+
consumer = Surfliner::MetadataConsumer::Consumer.new(
|
29
|
+
connection: Surfliner::Mq::Connection.new(logger: logger),
|
30
|
+
tracer: (tracer if defined?(tracer)),
|
31
|
+
handler: Surfliner::MetadataConsumer::Solr::MessageHandler
|
32
|
+
)
|
33
|
+
consumer.run(topic_config: Surfliner::Mq::TopicConfig.from_env(auto_delete: true))
|
data/bin/simulate-publish-event
CHANGED
@@ -4,24 +4,24 @@ require "json"
|
|
4
4
|
require "logger"
|
5
5
|
|
6
6
|
require "surfliner/metadata_consumer"
|
7
|
+
require "surfliner/mq"
|
7
8
|
|
8
9
|
module Surfliner
|
9
10
|
module MetadataConsumer
|
10
11
|
class Publisher
|
12
|
+
attr_reader :topic
|
13
|
+
attr_reader :api_base
|
14
|
+
attr_reader :routing_key
|
11
15
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
def initialize(logger:, topic_opts: {}, queue_opts: {})
|
17
|
-
@logger = logger
|
18
|
-
@topic_opts = topic_opts
|
19
|
-
@queue_opts = queue_opts
|
16
|
+
def initialize(topic, api_base:, routing_key:)
|
17
|
+
@topic = topic
|
18
|
+
@api_base = api_base
|
19
|
+
@routing_key = routing_key
|
20
20
|
end
|
21
21
|
|
22
22
|
def publish(resource_id)
|
23
23
|
payload = payload_for(resource_id)
|
24
|
-
|
24
|
+
topic.publish(payload, routing_key:)
|
25
25
|
end
|
26
26
|
|
27
27
|
def payload_for(id)
|
@@ -30,27 +30,37 @@ module Surfliner
|
|
30
30
|
time_stamp: DateTime.now }.to_json
|
31
31
|
end
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
@
|
33
|
+
class << self
|
34
|
+
# Connects with default (environment-configured) settings, yields a publisher
|
35
|
+
# for the specified topic, and closes the connection after the provided block
|
36
|
+
# completes.
|
37
|
+
#
|
38
|
+
# @param logger [Logger] the logger
|
39
|
+
# @param topic_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
|
40
|
+
#
|
41
|
+
# @yieldparam publisher [Publisher] the publisher
|
42
|
+
def connect(logger:, **topic_opts)
|
43
|
+
topic_config = Mq::TopicConfig.from_env(**topic_opts)
|
44
|
+
Mq::Connection.new(logger:).with_topic(topic_config) do |topic|
|
45
|
+
yield Publisher.new(
|
46
|
+
topic,
|
47
|
+
api_base: ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources"),
|
48
|
+
routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
|
49
|
+
)
|
50
|
+
end
|
51
|
+
end
|
39
52
|
end
|
40
53
|
end
|
41
54
|
end
|
42
55
|
end
|
43
56
|
|
44
57
|
$stdout.sync = true # don't buffer log output
|
45
|
-
logger
|
58
|
+
logger = Logger.new($stdout).tap do |logger|
|
46
59
|
logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
|
47
60
|
end
|
48
61
|
|
49
|
-
|
50
|
-
begin
|
62
|
+
Surfliner::MetadataConsumer::Publisher.connect(logger:, auto_delete: true) do |publisher|
|
51
63
|
ARGV.each do |resource_id|
|
52
64
|
publisher.publish(resource_id)
|
53
65
|
end
|
54
|
-
ensure
|
55
|
-
publisher.connection.close
|
56
66
|
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
require "rsolr"
|
2
|
-
require "surfliner/
|
2
|
+
require "surfliner/util/assert"
|
3
|
+
require "surfliner/mq/connection"
|
4
|
+
require "surfliner/mq/topic_config"
|
5
|
+
require "surfliner/mq/queue_config"
|
3
6
|
require "surfliner/metadata_consumer/solr/message_handler"
|
4
7
|
|
5
8
|
module Surfliner
|
@@ -7,29 +10,40 @@ module Surfliner
|
|
7
10
|
# A metadata consumer that subscribes to a RabbitMQ queue and passes
|
8
11
|
# messages to the specified handler.
|
9
12
|
class Consumer
|
10
|
-
|
13
|
+
include Util::Assert
|
14
|
+
|
15
|
+
# @return [Mq::Connection] the connection
|
16
|
+
attr_reader :connection
|
17
|
+
|
18
|
+
# @return [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
|
19
|
+
attr_reader :tracer
|
20
|
+
|
21
|
+
# @return [#handle] an object accepting a JSON string
|
22
|
+
attr_reader :handler
|
11
23
|
|
12
24
|
# Initializes a new `Consumer`
|
25
|
+
#
|
26
|
+
# @param connection [Mq::Connection] the connection
|
27
|
+
# @param handler [#handle] an object accepting a JSON string
|
13
28
|
# @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
@connection = MqConnection.new(logger:)
|
18
|
-
@logger = logger
|
29
|
+
def initialize(connection:, handler:, tracer: nil)
|
30
|
+
@connection = not_nil!(connection, "connection")
|
31
|
+
@handler = not_nil!(handler, "handler")
|
19
32
|
@tracer = tracer
|
20
|
-
|
33
|
+
end
|
34
|
+
|
35
|
+
# @return [Logger] log message destination
|
36
|
+
def logger
|
37
|
+
connection.logger
|
21
38
|
end
|
22
39
|
|
23
40
|
# Starts listening to the message queue and passing messages to the handler.
|
24
|
-
# @param
|
25
|
-
# @param
|
26
|
-
def run(
|
27
|
-
connection.
|
28
|
-
queue.
|
29
|
-
|
30
|
-
handle(payload_json)
|
31
|
-
end
|
32
|
-
end
|
41
|
+
# @param topic_config [TopicConfig] topic configuration
|
42
|
+
# @param queue_config [QueueConfig] queue configuration
|
43
|
+
def run(topic_config: Mq::TopicConfig.from_env, queue_config: Mq::QueueConfig.from_env)
|
44
|
+
connection.with_topic(topic_config) do |topic|
|
45
|
+
queue = topic.bind_queue(queue_config)
|
46
|
+
queue.subscribe(block: true, &method(:on_delivery))
|
33
47
|
end
|
34
48
|
end
|
35
49
|
|
@@ -43,6 +57,16 @@ module Surfliner
|
|
43
57
|
rescue => err
|
44
58
|
logger.error(" [!] failed to handle message: #{err.full_message}")
|
45
59
|
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def on_delivery(_delivery_info, _properties, payload_json)
|
64
|
+
return handle(payload_json) unless tracer
|
65
|
+
|
66
|
+
tracer.in_span("surfliner metadata consumer message") do |_span, _context|
|
67
|
+
handle(payload_json)
|
68
|
+
end
|
69
|
+
end
|
46
70
|
end
|
47
71
|
end
|
48
72
|
end
|
@@ -1 +1,4 @@
|
|
1
|
-
Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).
|
1
|
+
Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).each(&method(:require))
|
2
|
+
|
3
|
+
# Functionality specific to consuming RabbitMQ feeds
|
4
|
+
module Surfliner::MetadataConsumer; end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require "bunny"
|
2
|
+
require "surfliner/util/assert"
|
3
|
+
|
4
|
+
module Surfliner
|
5
|
+
module Mq
|
6
|
+
# An object encapsulating a RabbitMQ connection.
|
7
|
+
class Connection
|
8
|
+
include Util::Assert
|
9
|
+
|
10
|
+
# @return [Logger] The logger
|
11
|
+
attr_reader :logger
|
12
|
+
|
13
|
+
# @return [Bunny::Session] The current RabbitMQ session
|
14
|
+
attr_reader :session
|
15
|
+
|
16
|
+
# @return [Bunny::Channel] The channel being listened to
|
17
|
+
attr_reader :channel
|
18
|
+
|
19
|
+
# @return [ConnectionConfig] The configuration
|
20
|
+
attr_reader :config
|
21
|
+
|
22
|
+
# Initializes a new `MqConnection`.
|
23
|
+
#
|
24
|
+
# @param logger [Logger] the logger
|
25
|
+
# @param config [ConnectionConfig] the configuration
|
26
|
+
def initialize(logger:, config: ConnectionConfig.from_env)
|
27
|
+
@logger = not_nil!(logger)
|
28
|
+
@config = not_nil!(config)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Opens a connection.
|
32
|
+
# @return [self]
|
33
|
+
# @raise [IOError] if already connected
|
34
|
+
def connect
|
35
|
+
raise IOError, "RabbitMQ session #{session} already open." if open?
|
36
|
+
|
37
|
+
logger.info("Rabbitmq message broker session url: #{config.redacted_url}")
|
38
|
+
@session = Bunny.new(config.session_url, logger: logger)
|
39
|
+
connect_on(session)
|
40
|
+
@channel = session.create_channel
|
41
|
+
|
42
|
+
self
|
43
|
+
rescue Bunny::TCPConnectionFailed => err
|
44
|
+
# TODO: realistically, this only happens in session.start, where we're eating it
|
45
|
+
logger.error("Connection to #{config.redacted_url} failed")
|
46
|
+
raise err
|
47
|
+
rescue Bunny::PossibleAuthenticationFailureError => err
|
48
|
+
# TODO: realistically, this only happens in session.start, where we're eating it
|
49
|
+
logger.error("Failed to authenticate to #{config.redacted_url}")
|
50
|
+
raise err
|
51
|
+
end
|
52
|
+
|
53
|
+
# Opens a session, yields a client for the specified topic, and closes the
|
54
|
+
# session after the provided block completes.
|
55
|
+
# @param config [TopicConfig] topic configuration
|
56
|
+
# @yield [MqTopic] A client for the topic
|
57
|
+
def with_topic(config = TopicConfig.from_env)
|
58
|
+
connect
|
59
|
+
|
60
|
+
yield topic_from(config)
|
61
|
+
ensure
|
62
|
+
close
|
63
|
+
end
|
64
|
+
|
65
|
+
# Returns a client for the specified topic. Note that this does _not_ open
|
66
|
+
# or close the connection.
|
67
|
+
#
|
68
|
+
# @param config [TopicConfig] topic configuration
|
69
|
+
# @return [Topic] A client for the topic
|
70
|
+
# @raise [IOError] if the connection is not open
|
71
|
+
def topic_from(config)
|
72
|
+
raise IOError, "RabbitMQ session not open" unless open?
|
73
|
+
|
74
|
+
Topic.from_config(config, channel:, logger:)
|
75
|
+
end
|
76
|
+
|
77
|
+
# Closes the session.
|
78
|
+
def close
|
79
|
+
return unless channel
|
80
|
+
return if channel.closed?
|
81
|
+
logger.info("closing channel")
|
82
|
+
channel.close
|
83
|
+
ensure
|
84
|
+
logger.info("closing session")
|
85
|
+
session&.close
|
86
|
+
end
|
87
|
+
|
88
|
+
# @return [true, false] True if the session is open, false otherwise
|
89
|
+
def open?
|
90
|
+
session&.status == :open
|
91
|
+
end
|
92
|
+
|
93
|
+
# @return [Symbol, nil] The session status, or nil if there is no session
|
94
|
+
def status
|
95
|
+
session&.status
|
96
|
+
end
|
97
|
+
|
98
|
+
# @return [String] The RabbitMQ hostname
|
99
|
+
def host
|
100
|
+
config.host
|
101
|
+
end
|
102
|
+
|
103
|
+
# @return [String] The RabbitMQ port
|
104
|
+
def port
|
105
|
+
config.port
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
def connect_on(session, timeout = 120)
|
111
|
+
timer = 0
|
112
|
+
logger.info "Trying to open queue session with timeout=#{timeout}"
|
113
|
+
while timer < timeout
|
114
|
+
begin
|
115
|
+
session.start
|
116
|
+
rescue
|
117
|
+
# TODO: do we actually want to rescue from everything?
|
118
|
+
end
|
119
|
+
return session if session.status == :open
|
120
|
+
sleep 1
|
121
|
+
timer += 1
|
122
|
+
end
|
123
|
+
raise "Failed to connect to queue."
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|