message_bus_client_worker 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +8 -2
- data/README.md +52 -11
- data/lib/message_bus_client_worker/services/poll.rb +4 -2
- data/lib/message_bus_client_worker/services/polling/generate_params.rb +6 -1
- data/lib/message_bus_client_worker/services/polling/get_last_id.rb +1 -6
- data/lib/message_bus_client_worker/services/polling/{get_messages.rb → get_payloads.rb} +3 -3
- data/lib/message_bus_client_worker/services/polling/process_payload.rb +23 -0
- data/lib/message_bus_client_worker/version.rb +1 -1
- data/lib/message_bus_client_worker/workers/subscription_worker.rb +2 -2
- data/lib/message_bus_client_worker.rb +5 -2
- data/message_bus_client_worker.gemspec +2 -1
- metadata +32 -18
- data/lib/message_bus_client_worker/services/polling/process_messages.rb +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: de30a005fd8d0ded3dd3abb9539f67efc2e06b526212af01a37b7c0fd1d43d31
|
4
|
+
data.tar.gz: 73de613d059e2d77b5732eeac9c6581fa25de7cdd6c1d686864059d700916238
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10a046ecaa85f5e5d92ce11b50d6cdbeebc152ba684d9514326c858dee857009186640161834f3457244cf3972d5d0e78b51f20012d868a30797fc4a7211d555
|
7
|
+
data.tar.gz: 47b26efdcc4869834fba1e49f5a44c0e8336551426c2708e2fbd101b4b3de95e145aac079d6ffd6ccb426b65140cd7f3aa05082b3edbf6f3ba987aa2aed5d7cc
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
5
5
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
+
## [0.2.0] - 2018-08-08
|
8
|
+
### Changed
|
9
|
+
- Ability to specify what message_id to start from
|
10
|
+
|
7
11
|
## [0.1.0] - 2018-08-07
|
8
12
|
### Added
|
9
13
|
- Initial release
|
data/Gemfile.lock
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
message_bus_client_worker (0.
|
4
|
+
message_bus_client_worker (0.2.0)
|
5
|
+
activesupport
|
5
6
|
addressable
|
6
7
|
gem_config
|
7
8
|
http
|
8
9
|
light-service
|
9
10
|
sidekiq
|
11
|
+
sidekiq-unique-jobs (>= 6.0.0)
|
10
12
|
|
11
13
|
GEM
|
12
14
|
remote: https://rubygems.org/
|
@@ -74,6 +76,11 @@ GEM
|
|
74
76
|
connection_pool (~> 2.2, >= 2.2.0)
|
75
77
|
rack-protection (>= 1.5.0)
|
76
78
|
redis (>= 3.3.5, < 5)
|
79
|
+
sidekiq-unique-jobs (6.0.4)
|
80
|
+
concurrent-ruby (~> 1.0, >= 1.0.5)
|
81
|
+
sidekiq (>= 4.0, < 6.0)
|
82
|
+
thor (~> 0)
|
83
|
+
thor (0.20.0)
|
77
84
|
thread_safe (0.3.6)
|
78
85
|
tzinfo (1.2.5)
|
79
86
|
thread_safe (~> 0.1)
|
@@ -86,7 +93,6 @@ PLATFORMS
|
|
86
93
|
ruby
|
87
94
|
|
88
95
|
DEPENDENCIES
|
89
|
-
activesupport
|
90
96
|
bundler (~> 1.16)
|
91
97
|
message_bus_client_worker!
|
92
98
|
pry-byebug
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Subscribe to [MessageBus](https://github.com/SamSaffron/message_bus) using Sidekiq workers. This gem was borne out of the noisy logs and difficult in debugging using [message_bus-client](https://github.com/lowjoel/message_bus-client). This gem aims to:
|
4
4
|
|
5
|
-
- allow sane debugging
|
5
|
+
- allow sane debugging. Do not have thread-related code all over the code to the MessageBus channel. Rely on Sidekiq
|
6
6
|
- keep subscriptions to 1 per channel. With current options, every web server process would start a thread that listened to the MessageBus channels.
|
7
7
|
- do not unnecessarily add noise when starting the console like `rails console`
|
8
8
|
- recover from downtime by keeping track of the last message it processed per channel
|
@@ -36,25 +36,67 @@ MessageBusClientWorker.configure do |c|
|
|
36
36
|
# "/b_channel" => "BChannelPayloadProcessor",
|
37
37
|
# }
|
38
38
|
"https://etc.com" => {
|
39
|
-
"/exchange_rates" =>
|
40
|
-
|
39
|
+
"/exchange_rates" => {
|
40
|
+
processor: "ProcessExchangeRate",
|
41
|
+
message_id: 0,
|
42
|
+
},
|
43
|
+
"/messages" => { processor: "ProcessMessage" },
|
41
44
|
},
|
42
45
|
"https://someotherdomain.com" => {
|
43
|
-
"/errors" => "ProcessError",
|
46
|
+
"/errors" => { processor: "ProcessError" },
|
44
47
|
},
|
45
48
|
}
|
46
49
|
end
|
47
50
|
```
|
48
51
|
|
52
|
+
### Processor
|
53
|
+
|
54
|
+
The processor should look like this:
|
55
|
+
|
56
|
+
```ruby
|
57
|
+
class ProcessMessage
|
58
|
+
def self.call(data, payload)
|
59
|
+
# ...
|
60
|
+
end
|
61
|
+
end
|
62
|
+
```
|
63
|
+
|
64
|
+
http://chat.samsaffron.com's `/message` channel returns JSON like this:
|
65
|
+
|
66
|
+
```json
|
67
|
+
[
|
68
|
+
{"global_id":1478,"message_id":3,"channel":"/message","data":{"data":"hey","name":"joe"}},
|
69
|
+
{"global_id":1479,"message_id":4,"channel":"/message","data":{"data":"what's up","name":"joe"}},
|
70
|
+
...
|
71
|
+
]
|
72
|
+
```
|
73
|
+
|
74
|
+
The processor you define will receive `call` for every element in the JSON. In `ProcessMessage` processor as seen above:
|
75
|
+
|
76
|
+
- `data` would be the value of the "data" key as a Ruby hash, i.e. `{"data" => "hey","name" => "joe"}`
|
77
|
+
- `payload` is the whole item as a Ruby hash, i.e. `{"global_id"=>1479, "message_id"=>4, "channel"=>"/message", "data"=>{"data"=>"what's up", "name"=>"joe"}}`
|
78
|
+
|
79
|
+
If you don't care to see the whole payload, you can do the following:
|
80
|
+
|
49
81
|
```ruby
|
50
82
|
class ProcessMessage
|
51
|
-
def self.call(
|
52
|
-
|
83
|
+
def self.call(data, _)
|
84
|
+
# ...
|
53
85
|
end
|
54
86
|
end
|
55
87
|
```
|
56
88
|
|
57
|
-
|
89
|
+
### `message_id`
|
90
|
+
|
91
|
+
`message_id` defines where the worker should start reading from.
|
92
|
+
|
93
|
+
- you do not want to use "-1" if you're short polling because no messages will be read and MessageBusClientWorker will not record the last `message_id` seen.
|
94
|
+
- this is only used when nothing has been read before. If you put 0, but the last message that MessageBusClientWorker pulled had a `message_id` of `23`, then this gem will continue where it left off, and not read from 0
|
95
|
+
- defaults to "-1"
|
96
|
+
|
97
|
+
### Staying subscribed
|
98
|
+
|
99
|
+
To keep the subscription alive if the worker dies, the app restarts, or the worker falls back to short-polling or long-polling without streaming, use a gem like [sidekiq-cron](https://github.com/ondrejbartas/sidekiq-cron) and enqueue the `MessageBusClientWorker::EnqueuingWorker`:
|
58
100
|
|
59
101
|
```yml
|
60
102
|
message_bus_client_worker:
|
@@ -74,8 +116,8 @@ Every time `MessageBusClientWorker::EnqueuingWorker` is enqueued, `EnqueuingWork
|
|
74
116
|
|
75
117
|
`SubscriptionWorker` will open a connection to the server, and try the following (not all have been implemented):
|
76
118
|
|
77
|
-
- [ ] long-poll with streaming, or if streaming is not supported...
|
78
|
-
- [ ] long-poll, or if long-polling is not supported...
|
119
|
+
- [ ] long-poll with streaming, or if streaming is not supported by the server...
|
120
|
+
- [ ] long-poll, or if long-polling is not supported by the server...
|
79
121
|
- [x] short-poll
|
80
122
|
|
81
123
|
## Development
|
@@ -84,8 +126,7 @@ Copy the config and edit (if necessary)
|
|
84
126
|
|
85
127
|
```sh
|
86
128
|
cp spec/config.yml{.sample,}
|
87
|
-
docker-compose
|
88
|
-
rspec spec
|
129
|
+
docker-compose run gem bundle exec rspec spec
|
89
130
|
```
|
90
131
|
|
91
132
|
## Contributing
|
@@ -12,8 +12,10 @@ module MessageBusClientWorker
|
|
12
12
|
Polling::GenerateClientId,
|
13
13
|
Polling::GenerateURI,
|
14
14
|
Polling::GenerateParams,
|
15
|
-
Polling::
|
16
|
-
|
15
|
+
Polling::GetPayloads,
|
16
|
+
iterate(:payloads, [
|
17
|
+
Polling::ProcessPayload,
|
18
|
+
])
|
17
19
|
)
|
18
20
|
end
|
19
21
|
|
@@ -6,10 +6,15 @@ module MessageBusClientWorker
|
|
6
6
|
expects :host, :subscriptions
|
7
7
|
promises :params, :form_params
|
8
8
|
|
9
|
+
DEFAULT_MESSAGE_ID = "-1".freeze
|
10
|
+
|
9
11
|
executed do |c|
|
10
12
|
c.params = { dlp: 't' }
|
11
13
|
c.form_params = c.subscriptions.each_with_object({}) do |sub, hash|
|
12
|
-
|
14
|
+
custom_message_id = sub[1][:message_id] ? sub[1][:message_id].to_s : nil
|
15
|
+
hash[sub[0]] = GetLastId.(c.host, sub[0]) ||
|
16
|
+
custom_message_id ||
|
17
|
+
DEFAULT_MESSAGE_ID
|
13
18
|
end
|
14
19
|
end
|
15
20
|
|
@@ -4,12 +4,7 @@ module MessageBusClientWorker
|
|
4
4
|
|
5
5
|
def self.call(host, channel)
|
6
6
|
hash_key = GenLastIdKey.(host, channel)
|
7
|
-
|
8
|
-
id = Sidekiq.redis do |r|
|
9
|
-
r.hget(SetLastId::CHANNEL_INDICES_NAME, hash_key)
|
10
|
-
end
|
11
|
-
|
12
|
-
id || "0"
|
7
|
+
Sidekiq.redis { |r| r.hget(SetLastId::CHANNEL_INDICES_NAME, hash_key) }
|
13
8
|
end
|
14
9
|
|
15
10
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
module MessageBusClientWorker
|
2
2
|
module Polling
|
3
|
-
class
|
3
|
+
class GetPayloads
|
4
4
|
extend LightService::Action
|
5
5
|
|
6
6
|
expects :params, :form_params, :uri
|
7
|
-
promises :
|
7
|
+
promises :payloads
|
8
8
|
|
9
9
|
executed do |c|
|
10
10
|
body = HTTP.post(c.uri, params: c.params, form: c.form_params).body
|
11
|
-
c.
|
11
|
+
c.payloads = JSON.parse(body.to_s)
|
12
12
|
end
|
13
13
|
end
|
14
14
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module MessageBusClientWorker
|
2
|
+
module Polling
|
3
|
+
class ProcessPayload
|
4
|
+
extend LightService::Action
|
5
|
+
|
6
|
+
expects :host, :subscriptions, :payload
|
7
|
+
|
8
|
+
executed do |c|
|
9
|
+
payload = c.payload
|
10
|
+
channel = payload["channel"]
|
11
|
+
channel_config = c.subscriptions[channel]
|
12
|
+
|
13
|
+
next c if channel_config.nil?
|
14
|
+
|
15
|
+
processor_class = Kernel.const_get(channel_config[:processor])
|
16
|
+
|
17
|
+
SetLastId.(c.host, channel, payload["message_id"])
|
18
|
+
processor_class.(payload["data"], payload)
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -2,10 +2,10 @@ module MessageBusClientWorker
|
|
2
2
|
class SubscriptionWorker
|
3
3
|
|
4
4
|
include Sidekiq::Worker
|
5
|
-
sidekiq_options retry: false
|
5
|
+
sidekiq_options retry: false, lock: :until_executed
|
6
6
|
|
7
7
|
def perform(host, subscriptions, long=false)
|
8
|
-
Poll.(host, subscriptions, long)
|
8
|
+
Poll.(host, subscriptions.with_indifferent_access, long)
|
9
9
|
end
|
10
10
|
|
11
11
|
end
|
@@ -4,6 +4,9 @@ require "http"
|
|
4
4
|
require "light-service"
|
5
5
|
require "securerandom"
|
6
6
|
require "sidekiq"
|
7
|
+
require "sidekiq-unique-jobs"
|
8
|
+
require "active_support/core_ext/hash/indifferent_access"
|
9
|
+
require "active_support/core_ext/string/inflections"
|
7
10
|
require "message_bus_client_worker/version"
|
8
11
|
require "message_bus_client_worker/workers/enqueuing_worker"
|
9
12
|
require "message_bus_client_worker/workers/subscription_worker"
|
@@ -13,8 +16,8 @@ require "message_bus_client_worker/services/polling/set_last_id"
|
|
13
16
|
require "message_bus_client_worker/services/polling/generate_client_id"
|
14
17
|
require "message_bus_client_worker/services/polling/generate_uri"
|
15
18
|
require "message_bus_client_worker/services/polling/generate_params"
|
16
|
-
require "message_bus_client_worker/services/polling/
|
17
|
-
require "message_bus_client_worker/services/polling/
|
19
|
+
require "message_bus_client_worker/services/polling/get_payloads"
|
20
|
+
require "message_bus_client_worker/services/polling/process_payload"
|
18
21
|
require "message_bus_client_worker/services/poll"
|
19
22
|
|
20
23
|
module MessageBusClientWorker
|
@@ -35,11 +35,12 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_dependency "http"
|
36
36
|
spec.add_dependency "addressable"
|
37
37
|
spec.add_dependency "light-service"
|
38
|
+
spec.add_dependency "sidekiq-unique-jobs", ">= 6.0.0"
|
39
|
+
spec.add_dependency "activesupport"
|
38
40
|
|
39
41
|
spec.add_development_dependency "bundler", "~> 1.16"
|
40
42
|
spec.add_development_dependency "rake", "~> 10.0"
|
41
43
|
spec.add_development_dependency "rspec", "~> 3.0"
|
42
44
|
spec.add_development_dependency "rspec-sidekiq"
|
43
|
-
spec.add_development_dependency "activesupport"
|
44
45
|
spec.add_development_dependency "wait"
|
45
46
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: message_bus_client_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ramon Tayag
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gem_config
|
@@ -80,6 +80,34 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: sidekiq-unique-jobs
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 6.0.0
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 6.0.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: activesupport
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
83
111
|
- !ruby/object:Gem::Dependency
|
84
112
|
name: bundler
|
85
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,20 +164,6 @@ dependencies:
|
|
136
164
|
- - ">="
|
137
165
|
- !ruby/object:Gem::Version
|
138
166
|
version: '0'
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: activesupport
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - ">="
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '0'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - ">="
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: wait
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -193,8 +207,8 @@ files:
|
|
193
207
|
- lib/message_bus_client_worker/services/polling/generate_params.rb
|
194
208
|
- lib/message_bus_client_worker/services/polling/generate_uri.rb
|
195
209
|
- lib/message_bus_client_worker/services/polling/get_last_id.rb
|
196
|
-
- lib/message_bus_client_worker/services/polling/
|
197
|
-
- lib/message_bus_client_worker/services/polling/
|
210
|
+
- lib/message_bus_client_worker/services/polling/get_payloads.rb
|
211
|
+
- lib/message_bus_client_worker/services/polling/process_payload.rb
|
198
212
|
- lib/message_bus_client_worker/services/polling/set_last_id.rb
|
199
213
|
- lib/message_bus_client_worker/version.rb
|
200
214
|
- lib/message_bus_client_worker/workers/enqueuing_worker.rb
|
@@ -1,20 +0,0 @@
|
|
1
|
-
module MessageBusClientWorker
|
2
|
-
module Polling
|
3
|
-
class ProcessMessages
|
4
|
-
extend LightService::Action
|
5
|
-
|
6
|
-
expects :host, :subscriptions, :messages
|
7
|
-
|
8
|
-
executed do |c|
|
9
|
-
c.messages.each do |message|
|
10
|
-
channel = message["channel"]
|
11
|
-
processor_class = Kernel.const_get(c.subscriptions[channel])
|
12
|
-
|
13
|
-
SetLastId.(c.host, channel, message["message_id"])
|
14
|
-
processor_class.(message["data"])
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|