railway-ipc 0.1.4 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -3
- data/CHANGELOG.MD +43 -0
- data/Gemfile +2 -2
- data/README.md +1 -1
- data/Rakefile +10 -4
- data/bin/console +3 -3
- data/bin/rspec +29 -0
- data/bin/rubocop +29 -0
- data/lib/railway_ipc.rb +6 -4
- data/lib/railway_ipc/Rakefile +2 -0
- data/lib/railway_ipc/consumer/consumer.rb +21 -81
- data/lib/railway_ipc/consumer/consumer_response_handlers.rb +2 -0
- data/lib/railway_ipc/consumer/process_incoming_message.rb +105 -0
- data/lib/railway_ipc/errors.rb +9 -1
- data/lib/railway_ipc/handler.rb +5 -6
- data/lib/railway_ipc/handler_manifest.rb +2 -0
- data/lib/railway_ipc/handler_store.rb +3 -0
- data/lib/railway_ipc/incoming_message.rb +51 -0
- data/lib/railway_ipc/logger.rb +4 -3
- data/lib/railway_ipc/models/consumed_message.rb +41 -27
- data/lib/railway_ipc/models/published_message.rb +11 -9
- data/lib/railway_ipc/publisher.rb +58 -1
- data/lib/railway_ipc/rabbitmq/adapter.rb +24 -14
- data/lib/railway_ipc/rabbitmq/payload.rb +9 -4
- data/lib/railway_ipc/railtie.rb +2 -0
- data/lib/railway_ipc/responder.rb +6 -3
- data/lib/railway_ipc/response.rb +4 -1
- data/lib/railway_ipc/rpc/client/client.rb +27 -17
- data/lib/railway_ipc/rpc/client/client_response_handlers.rb +2 -0
- data/lib/railway_ipc/rpc/client/errors/timeout_error.rb +2 -0
- data/lib/railway_ipc/rpc/concerns/error_adapter_configurable.rb +2 -0
- data/lib/railway_ipc/rpc/concerns/message_observation_configurable.rb +2 -0
- data/lib/railway_ipc/rpc/concerns/publish_location_configurable.rb +2 -0
- data/lib/railway_ipc/rpc/rpc.rb +2 -0
- data/lib/railway_ipc/rpc/server/server.rb +10 -3
- data/lib/railway_ipc/rpc/server/server_response_handlers.rb +2 -0
- data/lib/railway_ipc/tasks/generate_migrations.rake +16 -16
- data/lib/railway_ipc/tasks/start_consumers.rake +3 -1
- data/lib/railway_ipc/tasks/start_servers.rake +3 -1
- data/lib/railway_ipc/unhandled_message_error.rb +2 -0
- data/lib/railway_ipc/unknown_message.pb.rb +18 -0
- data/lib/railway_ipc/version.rb +3 -1
- data/priv/migrations/add_railway_ipc_consumed_messages.rb +3 -1
- data/priv/migrations/add_railway_ipc_published_messages.rb +3 -1
- data/railway_ipc.gemspec +33 -30
- metadata +64 -65
- data/.rspec +0 -3
- data/.tool-versions +0 -1
- data/.travis.yml +0 -7
- data/Gemfile.lock +0 -186
- data/lib/railway_ipc/base_message.pb.rb +0 -21
- data/lib/railway_ipc/null_handler.rb +0 -7
- data/lib/railway_ipc/null_message.rb +0 -7
data/lib/railway_ipc/errors.rb
CHANGED
@@ -1 +1,9 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Style/ClassAndModuleChildren
|
4
|
+
class RailwayIpc::Error < StandardError; end
|
5
|
+
class RailwayIpc::InvalidProtobuf < RailwayIpc::Error; end
|
6
|
+
class RailwayIpc::FailedToStoreOutgoingMessage < RailwayIpc::Error; end
|
7
|
+
class RailwayIpc::IncomingMessage::ParserError < RailwayIpc::Error; end
|
8
|
+
class RailwayIpc::IncomingMessage::InvalidMessage < RailwayIpc::Error; end
|
9
|
+
# rubocop:enable Style/ClassAndModuleChildren
|
data/lib/railway_ipc/handler.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
class Handler
|
3
|
-
include Sneakers::Worker
|
4
5
|
class << self
|
5
6
|
attr_reader :block
|
6
7
|
end
|
@@ -10,14 +11,12 @@ module RailwayIpc
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def handle(message)
|
13
|
-
RailwayIpc.logger.info(message,
|
14
|
+
RailwayIpc.logger.info(message, 'Handling message')
|
14
15
|
response = self.class.block.call(message)
|
15
16
|
if response.success?
|
16
|
-
RailwayIpc.logger.info(message,
|
17
|
-
ack!
|
17
|
+
RailwayIpc.logger.info(message, 'Successfully handled message')
|
18
18
|
else
|
19
|
-
RailwayIpc.logger.error(message,
|
20
|
-
ack!
|
19
|
+
RailwayIpc.logger.error(message, 'Failed to handle message')
|
21
20
|
end
|
22
21
|
|
23
22
|
response
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module RailwayIpc
|
4
|
+
class IncomingMessage
|
5
|
+
attr_reader :type, :payload, :parsed_payload, :errors
|
6
|
+
|
7
|
+
def initialize(payload)
|
8
|
+
@parsed_payload = JSON.parse(payload)
|
9
|
+
@type = parsed_payload['type']
|
10
|
+
@payload = payload
|
11
|
+
@errors = {}
|
12
|
+
rescue JSON::ParserError => e
|
13
|
+
raise RailwayIpc::IncomingMessage::ParserError.new(e)
|
14
|
+
end
|
15
|
+
|
16
|
+
def uuid
|
17
|
+
decoded.uuid
|
18
|
+
end
|
19
|
+
|
20
|
+
def user_uuid
|
21
|
+
decoded.user_uuid
|
22
|
+
end
|
23
|
+
|
24
|
+
def correlation_id
|
25
|
+
decoded.correlation_id
|
26
|
+
end
|
27
|
+
|
28
|
+
def valid?
|
29
|
+
errors[:uuid] = 'uuid is required' unless uuid.present?
|
30
|
+
errors[:correlation_id] = 'correlation_id is required' unless correlation_id.present?
|
31
|
+
errors.none?
|
32
|
+
end
|
33
|
+
|
34
|
+
def decoded
|
35
|
+
@decoded ||=
|
36
|
+
begin
|
37
|
+
protobuf_msg = Base64.decode64(parsed_payload['encoded_message'])
|
38
|
+
decoder = Kernel.const_get(type)
|
39
|
+
decoder.decode(protobuf_msg)
|
40
|
+
rescue Google::Protobuf::ParseError => e
|
41
|
+
raise RailwayIpc::IncomingMessage::ParserError.new(e)
|
42
|
+
rescue NameError
|
43
|
+
RailwayIpc::Messages::Unknown.decode(protobuf_msg)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def stringify_errors
|
48
|
+
errors.values.join(', ')
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/lib/railway_ipc/logger.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
class Logger
|
3
|
-
|
4
5
|
attr_reader :logger
|
5
6
|
|
6
7
|
def initialize(logger)
|
@@ -23,8 +24,8 @@ module RailwayIpc
|
|
23
24
|
logger.error("[#{message_header(message)}] #{statement}")
|
24
25
|
end
|
25
26
|
|
26
|
-
def log_exception(
|
27
|
-
logger.error(
|
27
|
+
def log_exception(exception)
|
28
|
+
logger.error(exception)
|
28
29
|
end
|
29
30
|
|
30
31
|
def message_header(message)
|
@@ -1,29 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
class ConsumedMessage < ActiveRecord::Base
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
STATUS_SUCCESS = 'success'
|
6
|
+
STATUS_PROCESSING = 'processing'
|
7
|
+
STATUS_IGNORED = 'ignored'
|
8
|
+
STATUS_UNKNOWN_MESSAGE_TYPE = 'unknown_message_type'
|
9
|
+
STATUS_FAILED_TO_PROCESS = 'failed_to_process'
|
10
|
+
|
11
|
+
VALID_STATUSES = [
|
12
|
+
STATUS_SUCCESS,
|
13
|
+
STATUS_PROCESSING,
|
14
|
+
STATUS_IGNORED,
|
15
|
+
STATUS_UNKNOWN_MESSAGE_TYPE,
|
16
|
+
STATUS_FAILED_TO_PROCESS
|
17
|
+
].freeze
|
9
18
|
|
10
19
|
attr_reader :decoded_message
|
20
|
+
|
11
21
|
self.table_name = 'railway_ipc_consumed_messages'
|
12
22
|
self.primary_key = 'uuid'
|
13
23
|
|
14
24
|
validates :uuid, :status, presence: true
|
15
|
-
validates :status, inclusion: { in:
|
16
|
-
|
17
|
-
def
|
18
|
-
|
25
|
+
validates :status, inclusion: { in: VALID_STATUSES }
|
26
|
+
|
27
|
+
def self.create_processing(consumer, incoming_message)
|
28
|
+
# rubocop:disable Style/RedundantSelf
|
29
|
+
self.create!(
|
30
|
+
uuid: incoming_message.uuid,
|
31
|
+
status: STATUS_PROCESSING,
|
32
|
+
message_type: incoming_message.type,
|
33
|
+
user_uuid: incoming_message.user_uuid,
|
34
|
+
correlation_id: incoming_message.correlation_id,
|
35
|
+
queue: consumer.queue_name,
|
36
|
+
exchange: consumer.exchange_name,
|
37
|
+
encoded_message: incoming_message.payload
|
38
|
+
)
|
39
|
+
# rubocop:enable Style/RedundantSelf
|
19
40
|
end
|
20
41
|
|
21
|
-
def
|
22
|
-
|
42
|
+
def update_with_lock(job)
|
43
|
+
with_lock('FOR UPDATE NOWAIT') do
|
44
|
+
job.run
|
45
|
+
self.status = job.status
|
46
|
+
save
|
47
|
+
end
|
23
48
|
end
|
24
49
|
|
25
|
-
def
|
26
|
-
|
50
|
+
def processed?
|
51
|
+
# rubocop:disable Style/RedundantSelf
|
52
|
+
self.status == STATUS_SUCCESS
|
53
|
+
# rubocop:enable Style/RedundantSelf
|
27
54
|
end
|
28
55
|
|
29
56
|
private
|
@@ -31,18 +58,5 @@ module RailwayIpc
|
|
31
58
|
def timestamp_attributes_for_create
|
32
59
|
super << :inserted_at
|
33
60
|
end
|
34
|
-
|
35
|
-
def decode_message
|
36
|
-
begin
|
37
|
-
message_class = Kernel.const_get(self.message_type)
|
38
|
-
rescue NameError
|
39
|
-
message_class = RailwayIpc::BaseMessage
|
40
|
-
end
|
41
|
-
message_class.decode(decoded_protobuf)
|
42
|
-
end
|
43
|
-
|
44
|
-
def decoded_protobuf
|
45
|
-
Base64.decode64(self.encoded_message)
|
46
|
-
end
|
47
61
|
end
|
48
62
|
end
|
@@ -1,20 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
class PublishedMessage < ActiveRecord::Base
|
3
5
|
self.table_name = 'railway_ipc_published_messages'
|
4
|
-
self.primary_key =
|
6
|
+
self.primary_key = 'uuid'
|
5
7
|
|
6
8
|
validates :uuid, :status, presence: true
|
7
9
|
|
8
10
|
def self.store_message(exchange_name, message)
|
9
11
|
encoded_message = RailwayIpc::Rabbitmq::Payload.encode(message)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
12
|
+
create!(
|
13
|
+
uuid: message.uuid,
|
14
|
+
message_type: message.class.to_s,
|
15
|
+
user_uuid: message.user_uuid,
|
16
|
+
correlation_id: message.correlation_id,
|
17
|
+
encoded_message: encoded_message,
|
18
|
+
status: 'sent',
|
19
|
+
exchange: exchange_name
|
18
20
|
)
|
19
21
|
end
|
20
22
|
|
@@ -1,7 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'singleton'
|
2
4
|
|
3
5
|
module RailwayIpc
|
4
|
-
class
|
6
|
+
class SingletonPublisher < Sneakers::Publisher
|
5
7
|
include ::Singleton
|
6
8
|
|
7
9
|
def self.exchange(exchange)
|
@@ -10,6 +12,7 @@ module RailwayIpc
|
|
10
12
|
|
11
13
|
def self.exchange_name
|
12
14
|
raise 'Subclass must set the exchange' unless @exchange_name
|
15
|
+
|
13
16
|
@exchange_name
|
14
17
|
end
|
15
18
|
|
@@ -18,7 +21,11 @@ module RailwayIpc
|
|
18
21
|
end
|
19
22
|
|
20
23
|
def publish(message, published_message_store=RailwayIpc::PublishedMessage)
|
24
|
+
RailwayIpc.logger.logger.warn('DEPRECATED: Use new PublisherInstance class')
|
25
|
+
ensure_message_uuid(message)
|
26
|
+
ensure_correlation_id(message)
|
21
27
|
RailwayIpc.logger.info(message, 'Publishing message')
|
28
|
+
|
22
29
|
result = super(RailwayIpc::Rabbitmq::Payload.encode(message))
|
23
30
|
published_message_store.store_message(self.class.exchange_name, message)
|
24
31
|
result
|
@@ -26,5 +33,55 @@ module RailwayIpc
|
|
26
33
|
RailwayIpc.logger.error(message, 'Invalid protobuf')
|
27
34
|
raise e
|
28
35
|
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def ensure_message_uuid(message)
|
40
|
+
message.uuid = SecureRandom.uuid if message.uuid.blank?
|
41
|
+
message
|
42
|
+
end
|
43
|
+
|
44
|
+
def ensure_correlation_id(message)
|
45
|
+
message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
|
46
|
+
message
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
module RailwayIpc
|
52
|
+
class Publisher < Sneakers::Publisher
|
53
|
+
attr_reader :exchange_name, :message_store
|
54
|
+
|
55
|
+
def initialize(opts={})
|
56
|
+
@exchange_name = opts.fetch(:exchange_name)
|
57
|
+
@message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
|
58
|
+
connection = opts.fetch(:connection, nil)
|
59
|
+
options = {
|
60
|
+
exchange: exchange_name,
|
61
|
+
connection: connection,
|
62
|
+
exchange_type: :fanout
|
63
|
+
}.compact
|
64
|
+
super(options)
|
65
|
+
end
|
66
|
+
|
67
|
+
# rubocop:disable Metrics/AbcSize
|
68
|
+
def publish(message)
|
69
|
+
message.uuid = SecureRandom.uuid if message.uuid.blank?
|
70
|
+
message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
|
71
|
+
RailwayIpc.logger.info(message, 'Publishing message')
|
72
|
+
|
73
|
+
stored_message = message_store.store_message(exchange_name, message)
|
74
|
+
super(RailwayIpc::Rabbitmq::Payload.encode(message))
|
75
|
+
rescue RailwayIpc::InvalidProtobuf => e
|
76
|
+
RailwayIpc.logger.error(message, 'Invalid protobuf')
|
77
|
+
raise e
|
78
|
+
rescue ActiveRecord::RecordInvalid => e
|
79
|
+
RailwayIpc.logger.error(message, 'Failed to store outgoing message')
|
80
|
+
raise RailwayIpc::FailedToStoreOutgoingMessage.new(e)
|
81
|
+
rescue StandardError => e
|
82
|
+
stored_message&.destroy
|
83
|
+
raise e
|
84
|
+
end
|
85
|
+
# rubocop:enable Metrics/AbcSize
|
29
86
|
end
|
30
87
|
end
|
@@ -1,10 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
module Rabbitmq
|
3
5
|
class Adapter
|
4
|
-
class TimeoutError < StandardError;
|
5
|
-
end
|
6
|
+
class TimeoutError < StandardError; end
|
6
7
|
extend Forwardable
|
7
8
|
attr_reader :connection, :exchange, :exchange_name, :queue, :queue_name, :channel
|
9
|
+
|
8
10
|
def_delegators :connection,
|
9
11
|
:automatically_recover?,
|
10
12
|
:connected?,
|
@@ -14,22 +16,26 @@ module RailwayIpc
|
|
14
16
|
:port,
|
15
17
|
:user
|
16
18
|
|
17
|
-
def initialize(amqp_url: ENV[
|
19
|
+
def initialize(amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], exchange_name:, queue_name: '', options: {})
|
18
20
|
@queue_name = queue_name
|
19
21
|
@exchange_name = exchange_name
|
20
22
|
settings = AMQ::Settings.parse_amqp_url(amqp_url)
|
23
|
+
vhost = settings[:vhost] || '/'
|
21
24
|
@connection = Bunny.new({
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
host: settings[:host],
|
26
|
+
user: settings[:user],
|
27
|
+
pass: settings[:pass],
|
28
|
+
port: settings[:port],
|
29
|
+
vhost: vhost,
|
30
|
+
automatic_recovery: false,
|
31
|
+
logger: RailwayIpc.bunny_logger
|
32
|
+
}.merge(options))
|
29
33
|
end
|
30
34
|
|
31
|
-
def publish(message, options
|
35
|
+
def publish(message, options={})
|
36
|
+
# rubocop:disable Style/SafeNavigation
|
32
37
|
exchange.publish(message, options) if exchange
|
38
|
+
# rubocop:enable Style/SafeNavigation
|
33
39
|
end
|
34
40
|
|
35
41
|
def reply(message, from)
|
@@ -64,17 +70,19 @@ module RailwayIpc
|
|
64
70
|
self
|
65
71
|
end
|
66
72
|
|
67
|
-
def create_exchange(strategy: :fanout, options: {durable: true})
|
68
|
-
@exchange = Bunny::Exchange.new(connection.channel,
|
73
|
+
def create_exchange(strategy: :fanout, options: { durable: true })
|
74
|
+
@exchange = Bunny::Exchange.new(connection.channel, strategy, exchange_name, options)
|
69
75
|
self
|
70
76
|
end
|
71
77
|
|
72
78
|
def delete_exchange
|
79
|
+
# rubocop:disable Style/SafeNavigation
|
73
80
|
exchange.delete if exchange
|
81
|
+
# rubocop:enable Style/SafeNavigation
|
74
82
|
self
|
75
83
|
end
|
76
84
|
|
77
|
-
def create_queue(options
|
85
|
+
def create_queue(options={ durable: true })
|
78
86
|
@queue = @channel.queue(queue_name, options)
|
79
87
|
self
|
80
88
|
end
|
@@ -85,7 +93,9 @@ module RailwayIpc
|
|
85
93
|
end
|
86
94
|
|
87
95
|
def delete_queue
|
96
|
+
# rubocop:disable Style/SafeNavigation
|
88
97
|
queue.delete if queue
|
98
|
+
# rubocop:enable Style/SafeNavigation
|
89
99
|
self
|
90
100
|
end
|
91
101
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module RailwayIpc
|
2
4
|
module Rabbitmq
|
3
5
|
class Payload
|
@@ -7,6 +9,7 @@ module RailwayIpc
|
|
7
9
|
type = message.class.to_s
|
8
10
|
begin
|
9
11
|
message = Base64.encode64(message.class.encode(message))
|
12
|
+
# TODO: also need to rescue Google::Protobuf::TypeError
|
10
13
|
rescue NoMethodError
|
11
14
|
raise RailwayIpc::InvalidProtobuf.new("Message #{message} is not a valid protobuf")
|
12
15
|
end
|
@@ -15,8 +18,8 @@ module RailwayIpc
|
|
15
18
|
|
16
19
|
def self.decode(message)
|
17
20
|
message = JSON.parse(message)
|
18
|
-
type = message[
|
19
|
-
message = Base64.decode64(message[
|
21
|
+
type = message['type']
|
22
|
+
message = Base64.decode64(message['encoded_message'])
|
20
23
|
new(type, message)
|
21
24
|
end
|
22
25
|
|
@@ -25,12 +28,14 @@ module RailwayIpc
|
|
25
28
|
@message = message
|
26
29
|
end
|
27
30
|
|
31
|
+
# rubocop:disable Lint/ToJSON
|
28
32
|
def to_json
|
29
33
|
{
|
30
|
-
|
31
|
-
|
34
|
+
type: type,
|
35
|
+
encoded_message: message
|
32
36
|
}.to_json
|
33
37
|
end
|
38
|
+
# rubocop:enable Lint/ToJSON
|
34
39
|
end
|
35
40
|
end
|
36
41
|
end
|