railway-ipc 0.1.4 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/CHANGELOG.MD +43 -0
  4. data/Gemfile +2 -2
  5. data/README.md +1 -1
  6. data/Rakefile +10 -4
  7. data/bin/console +3 -3
  8. data/bin/rspec +29 -0
  9. data/bin/rubocop +29 -0
  10. data/lib/railway_ipc.rb +6 -4
  11. data/lib/railway_ipc/Rakefile +2 -0
  12. data/lib/railway_ipc/consumer/consumer.rb +21 -81
  13. data/lib/railway_ipc/consumer/consumer_response_handlers.rb +2 -0
  14. data/lib/railway_ipc/consumer/process_incoming_message.rb +105 -0
  15. data/lib/railway_ipc/errors.rb +9 -1
  16. data/lib/railway_ipc/handler.rb +5 -6
  17. data/lib/railway_ipc/handler_manifest.rb +2 -0
  18. data/lib/railway_ipc/handler_store.rb +3 -0
  19. data/lib/railway_ipc/incoming_message.rb +51 -0
  20. data/lib/railway_ipc/logger.rb +4 -3
  21. data/lib/railway_ipc/models/consumed_message.rb +41 -27
  22. data/lib/railway_ipc/models/published_message.rb +11 -9
  23. data/lib/railway_ipc/publisher.rb +58 -1
  24. data/lib/railway_ipc/rabbitmq/adapter.rb +24 -14
  25. data/lib/railway_ipc/rabbitmq/payload.rb +9 -4
  26. data/lib/railway_ipc/railtie.rb +2 -0
  27. data/lib/railway_ipc/responder.rb +6 -3
  28. data/lib/railway_ipc/response.rb +4 -1
  29. data/lib/railway_ipc/rpc/client/client.rb +27 -17
  30. data/lib/railway_ipc/rpc/client/client_response_handlers.rb +2 -0
  31. data/lib/railway_ipc/rpc/client/errors/timeout_error.rb +2 -0
  32. data/lib/railway_ipc/rpc/concerns/error_adapter_configurable.rb +2 -0
  33. data/lib/railway_ipc/rpc/concerns/message_observation_configurable.rb +2 -0
  34. data/lib/railway_ipc/rpc/concerns/publish_location_configurable.rb +2 -0
  35. data/lib/railway_ipc/rpc/rpc.rb +2 -0
  36. data/lib/railway_ipc/rpc/server/server.rb +10 -3
  37. data/lib/railway_ipc/rpc/server/server_response_handlers.rb +2 -0
  38. data/lib/railway_ipc/tasks/generate_migrations.rake +16 -16
  39. data/lib/railway_ipc/tasks/start_consumers.rake +3 -1
  40. data/lib/railway_ipc/tasks/start_servers.rake +3 -1
  41. data/lib/railway_ipc/unhandled_message_error.rb +2 -0
  42. data/lib/railway_ipc/unknown_message.pb.rb +18 -0
  43. data/lib/railway_ipc/version.rb +3 -1
  44. data/priv/migrations/add_railway_ipc_consumed_messages.rb +3 -1
  45. data/priv/migrations/add_railway_ipc_published_messages.rb +3 -1
  46. data/railway_ipc.gemspec +33 -30
  47. metadata +64 -65
  48. data/.rspec +0 -3
  49. data/.tool-versions +0 -1
  50. data/.travis.yml +0 -7
  51. data/Gemfile.lock +0 -186
  52. data/lib/railway_ipc/base_message.pb.rb +0 -21
  53. data/lib/railway_ipc/null_handler.rb +0 -7
  54. data/lib/railway_ipc/null_message.rb +0 -7
@@ -1 +1,9 @@
1
- class RailwayIpc::InvalidProtobuf < StandardError; end
1
+ # frozen_string_literal: true
2
+
3
+ # rubocop:disable Style/ClassAndModuleChildren
4
+ class RailwayIpc::Error < StandardError; end
5
+ class RailwayIpc::InvalidProtobuf < RailwayIpc::Error; end
6
+ class RailwayIpc::FailedToStoreOutgoingMessage < RailwayIpc::Error; end
7
+ class RailwayIpc::IncomingMessage::ParserError < RailwayIpc::Error; end
8
+ class RailwayIpc::IncomingMessage::InvalidMessage < RailwayIpc::Error; end
9
+ # rubocop:enable Style/ClassAndModuleChildren
@@ -1,6 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class Handler
3
- include Sneakers::Worker
4
5
  class << self
5
6
  attr_reader :block
6
7
  end
@@ -10,14 +11,12 @@ module RailwayIpc
10
11
  end
11
12
 
12
13
  def handle(message)
13
- RailwayIpc.logger.info(message, "Handling message")
14
+ RailwayIpc.logger.info(message, 'Handling message')
14
15
  response = self.class.block.call(message)
15
16
  if response.success?
16
- RailwayIpc.logger.info(message, "Successfully handled message")
17
- ack!
17
+ RailwayIpc.logger.info(message, 'Successfully handled message')
18
18
  else
19
- RailwayIpc.logger.error(message, "Failed to handle message")
20
- ack!
19
+ RailwayIpc.logger.error(message, 'Failed to handle message')
21
20
  end
22
21
 
23
22
  response
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class HandlerManifest
3
5
  attr_reader :message, :handler
@@ -1,7 +1,10 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'railway_ipc/handler_manifest'
2
4
  module RailwayIpc
3
5
  class HandlerStore
4
6
  attr_reader :handler_map
7
+
5
8
  def initialize
6
9
  @handler_map = {}
7
10
  end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RailwayIpc
4
+ class IncomingMessage
5
+ attr_reader :type, :payload, :parsed_payload, :errors
6
+
7
+ def initialize(payload)
8
+ @parsed_payload = JSON.parse(payload)
9
+ @type = parsed_payload['type']
10
+ @payload = payload
11
+ @errors = {}
12
+ rescue JSON::ParserError => e
13
+ raise RailwayIpc::IncomingMessage::ParserError.new(e)
14
+ end
15
+
16
+ def uuid
17
+ decoded.uuid
18
+ end
19
+
20
+ def user_uuid
21
+ decoded.user_uuid
22
+ end
23
+
24
+ def correlation_id
25
+ decoded.correlation_id
26
+ end
27
+
28
+ def valid?
29
+ errors[:uuid] = 'uuid is required' unless uuid.present?
30
+ errors[:correlation_id] = 'correlation_id is required' unless correlation_id.present?
31
+ errors.none?
32
+ end
33
+
34
+ def decoded
35
+ @decoded ||=
36
+ begin
37
+ protobuf_msg = Base64.decode64(parsed_payload['encoded_message'])
38
+ decoder = Kernel.const_get(type)
39
+ decoder.decode(protobuf_msg)
40
+ rescue Google::Protobuf::ParseError => e
41
+ raise RailwayIpc::IncomingMessage::ParserError.new(e)
42
+ rescue NameError
43
+ RailwayIpc::Messages::Unknown.decode(protobuf_msg)
44
+ end
45
+ end
46
+
47
+ def stringify_errors
48
+ errors.values.join(', ')
49
+ end
50
+ end
51
+ end
@@ -1,6 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class Logger
3
-
4
5
  attr_reader :logger
5
6
 
6
7
  def initialize(logger)
@@ -23,8 +24,8 @@ module RailwayIpc
23
24
  logger.error("[#{message_header(message)}] #{statement}")
24
25
  end
25
26
 
26
- def log_exception(e)
27
- logger.error(e)
27
+ def log_exception(exception)
28
+ logger.error(exception)
28
29
  end
29
30
 
30
31
  def message_header(message)
@@ -1,29 +1,56 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class ConsumedMessage < ActiveRecord::Base
3
- STATUSES = {
4
- success: 'success',
5
- processing: 'processing',
6
- unknown_message_type: 'unknown_message_type',
7
- failed_to_process: 'failed_to_process'
8
- }
5
+ STATUS_SUCCESS = 'success'
6
+ STATUS_PROCESSING = 'processing'
7
+ STATUS_IGNORED = 'ignored'
8
+ STATUS_UNKNOWN_MESSAGE_TYPE = 'unknown_message_type'
9
+ STATUS_FAILED_TO_PROCESS = 'failed_to_process'
10
+
11
+ VALID_STATUSES = [
12
+ STATUS_SUCCESS,
13
+ STATUS_PROCESSING,
14
+ STATUS_IGNORED,
15
+ STATUS_UNKNOWN_MESSAGE_TYPE,
16
+ STATUS_FAILED_TO_PROCESS
17
+ ].freeze
9
18
 
10
19
  attr_reader :decoded_message
20
+
11
21
  self.table_name = 'railway_ipc_consumed_messages'
12
22
  self.primary_key = 'uuid'
13
23
 
14
24
  validates :uuid, :status, presence: true
15
- validates :status, inclusion: { in: STATUSES.values }
16
-
17
- def processed?
18
- self.status == STATUSES[:success]
25
+ validates :status, inclusion: { in: VALID_STATUSES }
26
+
27
+ def self.create_processing(consumer, incoming_message)
28
+ # rubocop:disable Style/RedundantSelf
29
+ self.create!(
30
+ uuid: incoming_message.uuid,
31
+ status: STATUS_PROCESSING,
32
+ message_type: incoming_message.type,
33
+ user_uuid: incoming_message.user_uuid,
34
+ correlation_id: incoming_message.correlation_id,
35
+ queue: consumer.queue_name,
36
+ exchange: consumer.exchange_name,
37
+ encoded_message: incoming_message.payload
38
+ )
39
+ # rubocop:enable Style/RedundantSelf
19
40
  end
20
41
 
21
- def encoded_protobuf=(encoded_protobuf)
22
- self.encoded_message = Base64.encode64(encoded_protobuf)
42
+ def update_with_lock(job)
43
+ with_lock('FOR UPDATE NOWAIT') do
44
+ job.run
45
+ self.status = job.status
46
+ save
47
+ end
23
48
  end
24
49
 
25
- def decoded_message
26
- @decoded_message ||= decode_message
50
+ def processed?
51
+ # rubocop:disable Style/RedundantSelf
52
+ self.status == STATUS_SUCCESS
53
+ # rubocop:enable Style/RedundantSelf
27
54
  end
28
55
 
29
56
  private
@@ -31,18 +58,5 @@ module RailwayIpc
31
58
  def timestamp_attributes_for_create
32
59
  super << :inserted_at
33
60
  end
34
-
35
- def decode_message
36
- begin
37
- message_class = Kernel.const_get(self.message_type)
38
- rescue NameError
39
- message_class = RailwayIpc::BaseMessage
40
- end
41
- message_class.decode(decoded_protobuf)
42
- end
43
-
44
- def decoded_protobuf
45
- Base64.decode64(self.encoded_message)
46
- end
47
61
  end
48
62
  end
@@ -1,20 +1,22 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class PublishedMessage < ActiveRecord::Base
3
5
  self.table_name = 'railway_ipc_published_messages'
4
- self.primary_key = "uuid"
6
+ self.primary_key = 'uuid'
5
7
 
6
8
  validates :uuid, :status, presence: true
7
9
 
8
10
  def self.store_message(exchange_name, message)
9
11
  encoded_message = RailwayIpc::Rabbitmq::Payload.encode(message)
10
- self.create(
11
- uuid: message.uuid,
12
- message_type: message.class.to_s,
13
- user_uuid: message.user_uuid,
14
- correlation_id: message.correlation_id,
15
- encoded_message: encoded_message,
16
- status: "sent",
17
- exchange: exchange_name
12
+ create!(
13
+ uuid: message.uuid,
14
+ message_type: message.class.to_s,
15
+ user_uuid: message.user_uuid,
16
+ correlation_id: message.correlation_id,
17
+ encoded_message: encoded_message,
18
+ status: 'sent',
19
+ exchange: exchange_name
18
20
  )
19
21
  end
20
22
 
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'singleton'
2
4
 
3
5
  module RailwayIpc
4
- class Publisher < Sneakers::Publisher
6
+ class SingletonPublisher < Sneakers::Publisher
5
7
  include ::Singleton
6
8
 
7
9
  def self.exchange(exchange)
@@ -10,6 +12,7 @@ module RailwayIpc
10
12
 
11
13
  def self.exchange_name
12
14
  raise 'Subclass must set the exchange' unless @exchange_name
15
+
13
16
  @exchange_name
14
17
  end
15
18
 
@@ -18,7 +21,11 @@ module RailwayIpc
18
21
  end
19
22
 
20
23
  def publish(message, published_message_store=RailwayIpc::PublishedMessage)
24
+ RailwayIpc.logger.logger.warn('DEPRECATED: Use new PublisherInstance class')
25
+ ensure_message_uuid(message)
26
+ ensure_correlation_id(message)
21
27
  RailwayIpc.logger.info(message, 'Publishing message')
28
+
22
29
  result = super(RailwayIpc::Rabbitmq::Payload.encode(message))
23
30
  published_message_store.store_message(self.class.exchange_name, message)
24
31
  result
@@ -26,5 +33,55 @@ module RailwayIpc
26
33
  RailwayIpc.logger.error(message, 'Invalid protobuf')
27
34
  raise e
28
35
  end
36
+
37
+ private
38
+
39
+ def ensure_message_uuid(message)
40
+ message.uuid = SecureRandom.uuid if message.uuid.blank?
41
+ message
42
+ end
43
+
44
+ def ensure_correlation_id(message)
45
+ message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
46
+ message
47
+ end
48
+ end
49
+ end
50
+
51
+ module RailwayIpc
52
+ class Publisher < Sneakers::Publisher
53
+ attr_reader :exchange_name, :message_store
54
+
55
+ def initialize(opts={})
56
+ @exchange_name = opts.fetch(:exchange_name)
57
+ @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
58
+ connection = opts.fetch(:connection, nil)
59
+ options = {
60
+ exchange: exchange_name,
61
+ connection: connection,
62
+ exchange_type: :fanout
63
+ }.compact
64
+ super(options)
65
+ end
66
+
67
+ # rubocop:disable Metrics/AbcSize
68
+ def publish(message)
69
+ message.uuid = SecureRandom.uuid if message.uuid.blank?
70
+ message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
71
+ RailwayIpc.logger.info(message, 'Publishing message')
72
+
73
+ stored_message = message_store.store_message(exchange_name, message)
74
+ super(RailwayIpc::Rabbitmq::Payload.encode(message))
75
+ rescue RailwayIpc::InvalidProtobuf => e
76
+ RailwayIpc.logger.error(message, 'Invalid protobuf')
77
+ raise e
78
+ rescue ActiveRecord::RecordInvalid => e
79
+ RailwayIpc.logger.error(message, 'Failed to store outgoing message')
80
+ raise RailwayIpc::FailedToStoreOutgoingMessage.new(e)
81
+ rescue StandardError => e
82
+ stored_message&.destroy
83
+ raise e
84
+ end
85
+ # rubocop:enable Metrics/AbcSize
29
86
  end
30
87
  end
@@ -1,10 +1,12 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  module Rabbitmq
3
5
  class Adapter
4
- class TimeoutError < StandardError;
5
- end
6
+ class TimeoutError < StandardError; end
6
7
  extend Forwardable
7
8
  attr_reader :connection, :exchange, :exchange_name, :queue, :queue_name, :channel
9
+
8
10
  def_delegators :connection,
9
11
  :automatically_recover?,
10
12
  :connected?,
@@ -14,22 +16,26 @@ module RailwayIpc
14
16
  :port,
15
17
  :user
16
18
 
17
- def initialize(amqp_url: ENV["RAILWAY_RABBITMQ_CONNECTION_URL"], exchange_name:, queue_name: '', options: {})
19
+ def initialize(amqp_url: ENV['RAILWAY_RABBITMQ_CONNECTION_URL'], exchange_name:, queue_name: '', options: {})
18
20
  @queue_name = queue_name
19
21
  @exchange_name = exchange_name
20
22
  settings = AMQ::Settings.parse_amqp_url(amqp_url)
23
+ vhost = settings[:vhost] || '/'
21
24
  @connection = Bunny.new({
22
- host: settings[:host],
23
- user: settings[:user],
24
- pass: settings[:pass],
25
- port: settings[:port],
26
- automatic_recovery: false,
27
- logger: RailwayIpc.bunny_logger}.merge(options)
28
- )
25
+ host: settings[:host],
26
+ user: settings[:user],
27
+ pass: settings[:pass],
28
+ port: settings[:port],
29
+ vhost: vhost,
30
+ automatic_recovery: false,
31
+ logger: RailwayIpc.bunny_logger
32
+ }.merge(options))
29
33
  end
30
34
 
31
- def publish(message, options = {})
35
+ def publish(message, options={})
36
+ # rubocop:disable Style/SafeNavigation
32
37
  exchange.publish(message, options) if exchange
38
+ # rubocop:enable Style/SafeNavigation
33
39
  end
34
40
 
35
41
  def reply(message, from)
@@ -64,17 +70,19 @@ module RailwayIpc
64
70
  self
65
71
  end
66
72
 
67
- def create_exchange(strategy: :fanout, options: {durable: true})
68
- @exchange = Bunny::Exchange.new(connection.channel, :fanout, exchange_name, options)
73
+ def create_exchange(strategy: :fanout, options: { durable: true })
74
+ @exchange = Bunny::Exchange.new(connection.channel, strategy, exchange_name, options)
69
75
  self
70
76
  end
71
77
 
72
78
  def delete_exchange
79
+ # rubocop:disable Style/SafeNavigation
73
80
  exchange.delete if exchange
81
+ # rubocop:enable Style/SafeNavigation
74
82
  self
75
83
  end
76
84
 
77
- def create_queue(options = {durable: true})
85
+ def create_queue(options={ durable: true })
78
86
  @queue = @channel.queue(queue_name, options)
79
87
  self
80
88
  end
@@ -85,7 +93,9 @@ module RailwayIpc
85
93
  end
86
94
 
87
95
  def delete_queue
96
+ # rubocop:disable Style/SafeNavigation
88
97
  queue.delete if queue
98
+ # rubocop:enable Style/SafeNavigation
89
99
  self
90
100
  end
91
101
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  module Rabbitmq
3
5
  class Payload
@@ -7,6 +9,7 @@ module RailwayIpc
7
9
  type = message.class.to_s
8
10
  begin
9
11
  message = Base64.encode64(message.class.encode(message))
12
+ # TODO: also need to rescue Google::Protobuf::TypeError
10
13
  rescue NoMethodError
11
14
  raise RailwayIpc::InvalidProtobuf.new("Message #{message} is not a valid protobuf")
12
15
  end
@@ -15,8 +18,8 @@ module RailwayIpc
15
18
 
16
19
  def self.decode(message)
17
20
  message = JSON.parse(message)
18
- type = message["type"]
19
- message = Base64.decode64(message["encoded_message"])
21
+ type = message['type']
22
+ message = Base64.decode64(message['encoded_message'])
20
23
  new(type, message)
21
24
  end
22
25
 
@@ -25,12 +28,14 @@ module RailwayIpc
25
28
  @message = message
26
29
  end
27
30
 
31
+ # rubocop:disable Lint/ToJSON
28
32
  def to_json
29
33
  {
30
- type: type,
31
- encoded_message: message
34
+ type: type,
35
+ encoded_message: message
32
36
  }.to_json
33
37
  end
38
+ # rubocop:enable Lint/ToJSON
34
39
  end
35
40
  end
36
41
  end