railway-ipc 0.1.7 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/CHANGELOG.md +67 -0
  4. data/Gemfile +2 -2
  5. data/README.md +1 -1
  6. data/Rakefile +10 -4
  7. data/bin/console +3 -3
  8. data/bin/rspec +29 -0
  9. data/bin/rubocop +29 -0
  10. data/lib/railway_ipc.rb +9 -11
  11. data/lib/railway_ipc/Rakefile +2 -0
  12. data/lib/railway_ipc/consumer/consumer.rb +33 -73
  13. data/lib/railway_ipc/consumer/process_incoming_message.rb +111 -0
  14. data/lib/railway_ipc/errors.rb +9 -1
  15. data/lib/railway_ipc/handler.rb +17 -3
  16. data/lib/railway_ipc/handler_store.rb +5 -2
  17. data/lib/railway_ipc/incoming_message.rb +51 -0
  18. data/lib/railway_ipc/logger.rb +44 -26
  19. data/lib/railway_ipc/models/consumed_message.rb +40 -35
  20. data/lib/railway_ipc/models/published_message.rb +11 -9
  21. data/lib/railway_ipc/publisher.rb +67 -16
  22. data/lib/railway_ipc/rabbitmq/adapter.rb +23 -15
  23. data/lib/railway_ipc/rabbitmq/payload.rb +9 -4
  24. data/lib/railway_ipc/railtie.rb +2 -0
  25. data/lib/railway_ipc/responder.rb +10 -3
  26. data/lib/railway_ipc/response.rb +4 -1
  27. data/lib/railway_ipc/rpc/client/client.rb +43 -18
  28. data/lib/railway_ipc/rpc/client/client_response_handlers.rb +2 -0
  29. data/lib/railway_ipc/rpc/client/errors/timeout_error.rb +2 -0
  30. data/lib/railway_ipc/rpc/concerns/error_adapter_configurable.rb +2 -0
  31. data/lib/railway_ipc/rpc/concerns/message_observation_configurable.rb +2 -0
  32. data/lib/railway_ipc/rpc/concerns/publish_location_configurable.rb +2 -0
  33. data/lib/railway_ipc/rpc/rpc.rb +2 -0
  34. data/lib/railway_ipc/rpc/server/server.rb +25 -7
  35. data/lib/railway_ipc/rpc/server/server_response_handlers.rb +2 -0
  36. data/lib/railway_ipc/tasks/generate_migrations.rake +16 -16
  37. data/lib/railway_ipc/tasks/start_consumers.rake +3 -1
  38. data/lib/railway_ipc/tasks/start_servers.rake +3 -1
  39. data/lib/railway_ipc/unhandled_message_error.rb +2 -0
  40. data/lib/railway_ipc/unknown_message.pb.rb +18 -0
  41. data/lib/railway_ipc/version.rb +3 -1
  42. data/priv/migrations/add_railway_ipc_consumed_messages.rb +5 -3
  43. data/priv/migrations/add_railway_ipc_published_messages.rb +3 -1
  44. data/railway_ipc.gemspec +34 -30
  45. metadata +62 -64
  46. data/.rspec +0 -3
  47. data/.travis.yml +0 -7
  48. data/CHANGELOG.MD +0 -7
  49. data/Gemfile.lock +0 -186
  50. data/lib/railway_ipc/base_message.pb.rb +0 -21
  51. data/lib/railway_ipc/consumer/consumer_response_handlers.rb +0 -14
  52. data/lib/railway_ipc/handler_manifest.rb +0 -10
  53. data/lib/railway_ipc/null_handler.rb +0 -7
  54. data/lib/railway_ipc/null_message.rb +0 -7
@@ -1 +1,9 @@
1
- class RailwayIpc::InvalidProtobuf < StandardError; end
1
+ # frozen_string_literal: true
2
+
3
+ # rubocop:disable Style/ClassAndModuleChildren
4
+ class RailwayIpc::Error < StandardError; end
5
+ class RailwayIpc::InvalidProtobuf < RailwayIpc::Error; end
6
+ class RailwayIpc::FailedToStoreOutgoingMessage < RailwayIpc::Error; end
7
+ class RailwayIpc::IncomingMessage::ParserError < RailwayIpc::Error; end
8
+ class RailwayIpc::IncomingMessage::InvalidMessage < RailwayIpc::Error; end
9
+ # rubocop:enable Style/ClassAndModuleChildren
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class Handler
3
5
  class << self
@@ -9,15 +11,27 @@ module RailwayIpc
9
11
  end
10
12
 
11
13
  def handle(message)
12
- RailwayIpc.logger.info(message, 'Handling message')
14
+ RailwayIpc.logger.info('Handling message', log_message_options(message))
13
15
  response = self.class.block.call(message)
14
16
  if response.success?
15
- RailwayIpc.logger.info(message, 'Successfully handled message')
17
+ RailwayIpc.logger.info('Successfully handled message', log_message_options(message))
16
18
  else
17
- RailwayIpc.logger.error(message, 'Failed to handle message')
19
+ RailwayIpc.logger.error('Failed to handle message', log_message_options(message))
18
20
  end
19
21
 
20
22
  response
21
23
  end
24
+
25
+ private
26
+
27
+ def log_message_options(message)
28
+ {
29
+ feature: 'railway_ipc_consumer',
30
+ protobuf: {
31
+ type: message.class.name,
32
+ data: message
33
+ }
34
+ }
35
+ end
22
36
  end
23
37
  end
@@ -1,7 +1,10 @@
1
- require 'railway_ipc/handler_manifest'
1
+ # frozen_string_literal: true
2
+
2
3
  module RailwayIpc
4
+ HandlerManifest = Struct.new(:message, :handler)
3
5
  class HandlerStore
4
6
  attr_reader :handler_map
7
+
5
8
  def initialize
6
9
  @handler_map = {}
7
10
  end
@@ -11,7 +14,7 @@ module RailwayIpc
11
14
  end
12
15
 
13
16
  def register(message:, handler:)
14
- handler_map[message.to_s] = HandlerManifest.new(message: message, handler: handler)
17
+ handler_map[message.to_s] = HandlerManifest.new(message, handler)
15
18
  end
16
19
 
17
20
  def get(response_message)
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RailwayIpc
4
+ class IncomingMessage
5
+ attr_reader :type, :payload, :parsed_payload, :errors
6
+
7
+ def initialize(payload)
8
+ @parsed_payload = JSON.parse(payload)
9
+ @type = parsed_payload['type']
10
+ @payload = payload
11
+ @errors = {}
12
+ rescue JSON::ParserError => e
13
+ raise RailwayIpc::IncomingMessage::ParserError.new(e)
14
+ end
15
+
16
+ def uuid
17
+ decoded.uuid
18
+ end
19
+
20
+ def user_uuid
21
+ decoded.user_uuid
22
+ end
23
+
24
+ def correlation_id
25
+ decoded.correlation_id
26
+ end
27
+
28
+ def valid?
29
+ errors[:uuid] = 'uuid is required' unless uuid.present?
30
+ errors[:correlation_id] = 'correlation_id is required' unless correlation_id.present?
31
+ errors.none?
32
+ end
33
+
34
+ def decoded
35
+ @decoded ||=
36
+ begin
37
+ protobuf_msg = Base64.decode64(parsed_payload['encoded_message'])
38
+ decoder = Kernel.const_get(type)
39
+ decoder.decode(protobuf_msg)
40
+ rescue Google::Protobuf::ParseError => e
41
+ raise RailwayIpc::IncomingMessage::ParserError.new(e)
42
+ rescue NameError
43
+ RailwayIpc::Messages::Unknown.decode(protobuf_msg)
44
+ end
45
+ end
46
+
47
+ def stringify_errors
48
+ errors.values.join(', ')
49
+ end
50
+ end
51
+ end
@@ -1,35 +1,53 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
4
+ # Custom logger that accepts a `device`, `level`, and `formatter`.
5
+ # `formatter` can be any object that responds to `call`; a
6
+ # `Logger::Formatter` is used if the argument is not provided.
7
+ #
8
+ # Here is an example formatter that uses `Oj` to format structured log
9
+ # messages:
10
+ #
11
+ # require 'oj'
12
+ # OjFormatter = proc do |severity, datetime, progname, data|
13
+ # data.merge!(
14
+ # name: progname,
15
+ # timestamp: datetime,
16
+ # severity: severity
17
+ # )
18
+ # Oj.dump(data, { mode: :compat, time_format: :xmlschema })
19
+ # end
20
+ #
21
+ # logger = RailwayIpc::Logger.new(STDOUT, Logger::INFO, OjFormatter)
22
+ #
2
23
  class Logger
3
-
4
- attr_reader :logger
5
-
6
- def initialize(logger)
7
- @logger = logger
24
+ def initialize(device=STDOUT, level=::Logger::INFO, formatter=nil)
25
+ @logger = ::Logger.new(device)
26
+ @logger.level = level
27
+ @logger.formatter = formatter if formatter
8
28
  end
9
29
 
10
- def info(message, statement)
11
- logger.info("[#{message_header(message)}] #{statement}")
30
+ %w[fatal error warn info debug].each do |level|
31
+ define_method(level) do |message=nil, data={}, &block|
32
+ data.merge!(feature: 'railway_ipc') unless data.key?(:feature)
33
+ return logger.send(level, data.merge(message: message)) unless block
34
+
35
+ data = message.merge(data) if message&.is_a?(Hash)
36
+ data.merge!(message: block.call)
37
+
38
+ # This is for compatability w/ Ruby's Logger. Ruby's Logger class
39
+ # assumes that if both a `message` argument and a block are given,
40
+ # that the block contains the actual message. The `message` argument
41
+ # is assumed to be the `progname`.
42
+ #
43
+ # https://github.com/ruby/logger/blob/master/lib/logger.rb#L471
44
+ data.merge!(progname: message) if message&.is_a?(String)
45
+ logger.send(level, data)
46
+ end
12
47
  end
13
48
 
14
- def warn(message, statement)
15
- logger.warn("[#{message_header(message)}] #{statement}")
16
- end
17
-
18
- def debug(message, statement)
19
- logger.debug("[#{message_header(message)}] #{statement}")
20
- end
49
+ private
21
50
 
22
- def error(message, statement)
23
- logger.error("[#{message_header(message)}] #{statement}")
24
- end
25
-
26
- def log_exception(e)
27
- logger.error(e)
28
- end
29
-
30
- def message_header(message)
31
- log_statement = "message type: #{message.class}, uuid: #{message.uuid}, correlation_id: #{message.correlation_id}"
32
- message.respond_to?(:user_uuid) ? "#{log_statement}, user_uuid: #{message.user_uuid}" : log_statement
33
- end
51
+ attr_reader :logger
34
52
  end
35
53
  end
@@ -1,37 +1,55 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class ConsumedMessage < ActiveRecord::Base
3
- STATUSES = {
4
- success: 'success',
5
- processing: 'processing',
6
- unknown_message_type: 'unknown_message_type',
7
- failed_to_process: 'failed_to_process'
8
- }
5
+ STATUS_SUCCESS = 'success'
6
+ STATUS_PROCESSING = 'processing'
7
+ STATUS_IGNORED = 'ignored'
8
+ STATUS_UNKNOWN_MESSAGE_TYPE = 'unknown_message_type'
9
+ STATUS_FAILED_TO_PROCESS = 'failed_to_process'
10
+
11
+ VALID_STATUSES = [
12
+ STATUS_SUCCESS,
13
+ STATUS_PROCESSING,
14
+ STATUS_IGNORED,
15
+ STATUS_UNKNOWN_MESSAGE_TYPE,
16
+ STATUS_FAILED_TO_PROCESS
17
+ ].freeze
9
18
 
10
19
  attr_reader :decoded_message
20
+
11
21
  self.table_name = 'railway_ipc_consumed_messages'
12
- self.primary_key = 'uuid'
13
22
 
14
23
  validates :uuid, :status, presence: true
15
- validates :status, inclusion: { in: STATUSES.values }
24
+ validates :status, inclusion: { in: VALID_STATUSES }
25
+
26
+ def self.create_processing(consumer, incoming_message)
27
+ # rubocop:disable Style/RedundantSelf
28
+ self.create!(
29
+ uuid: incoming_message.uuid,
30
+ status: STATUS_PROCESSING,
31
+ message_type: incoming_message.type,
32
+ user_uuid: incoming_message.user_uuid,
33
+ correlation_id: incoming_message.correlation_id,
34
+ queue: consumer.queue_name,
35
+ exchange: consumer.exchange_name,
36
+ encoded_message: incoming_message.payload
37
+ )
38
+ # rubocop:enable Style/RedundantSelf
39
+ end
16
40
 
17
- def self.response_to_status(response)
18
- if response.success?
19
- STATUSES[:success]
20
- else
21
- STATUSES[:failed_to_process]
41
+ def update_with_lock(job)
42
+ with_lock('FOR UPDATE NOWAIT') do
43
+ job.run
44
+ self.status = job.status
45
+ save
22
46
  end
23
47
  end
24
48
 
25
49
  def processed?
26
- self.status == STATUSES[:success]
27
- end
28
-
29
- def encoded_protobuf=(encoded_protobuf)
30
- self.encoded_message = Base64.encode64(encoded_protobuf)
31
- end
32
-
33
- def decoded_message
34
- @decoded_message ||= decode_message
50
+ # rubocop:disable Style/RedundantSelf
51
+ self.status == STATUS_SUCCESS
52
+ # rubocop:enable Style/RedundantSelf
35
53
  end
36
54
 
37
55
  private
@@ -39,18 +57,5 @@ module RailwayIpc
39
57
  def timestamp_attributes_for_create
40
58
  super << :inserted_at
41
59
  end
42
-
43
- def decode_message
44
- begin
45
- message_class = Kernel.const_get(self.message_type)
46
- rescue NameError
47
- message_class = RailwayIpc::BaseMessage
48
- end
49
- message_class.decode(decoded_protobuf)
50
- end
51
-
52
- def decoded_protobuf
53
- Base64.decode64(self.encoded_message)
54
- end
55
60
  end
56
61
  end
@@ -1,20 +1,22 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module RailwayIpc
2
4
  class PublishedMessage < ActiveRecord::Base
3
5
  self.table_name = 'railway_ipc_published_messages'
4
- self.primary_key = "uuid"
6
+ self.primary_key = 'uuid'
5
7
 
6
8
  validates :uuid, :status, presence: true
7
9
 
8
10
  def self.store_message(exchange_name, message)
9
11
  encoded_message = RailwayIpc::Rabbitmq::Payload.encode(message)
10
- self.create(
11
- uuid: message.uuid,
12
- message_type: message.class.to_s,
13
- user_uuid: message.user_uuid,
14
- correlation_id: message.correlation_id,
15
- encoded_message: encoded_message,
16
- status: "sent",
17
- exchange: exchange_name
12
+ create!(
13
+ uuid: message.uuid,
14
+ message_type: message.class.to_s,
15
+ user_uuid: message.user_uuid,
16
+ correlation_id: message.correlation_id,
17
+ encoded_message: encoded_message,
18
+ status: 'sent',
19
+ exchange: exchange_name
18
20
  )
19
21
  end
20
22
 
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'singleton'
2
4
 
3
5
  module RailwayIpc
4
- class Publisher < Sneakers::Publisher
6
+ class SingletonPublisher < Sneakers::Publisher
5
7
  include ::Singleton
6
8
 
7
9
  def self.exchange(exchange)
@@ -10,6 +12,7 @@ module RailwayIpc
10
12
 
11
13
  def self.exchange_name
12
14
  raise 'Subclass must set the exchange' unless @exchange_name
15
+
13
16
  @exchange_name
14
17
  end
15
18
 
@@ -18,36 +21,84 @@ module RailwayIpc
18
21
  end
19
22
 
20
23
  def publish(message, published_message_store=RailwayIpc::PublishedMessage)
24
+ RailwayIpc.logger.warn('DEPRECATED: Use new PublisherInstance class', log_message_options)
21
25
  ensure_message_uuid(message)
22
26
  ensure_correlation_id(message)
23
- RailwayIpc.logger.info(message, 'Publishing message')
24
-
27
+ RailwayIpc.logger.info('Publishing message', log_message_options(message))
25
28
  result = super(RailwayIpc::Rabbitmq::Payload.encode(message))
26
29
  published_message_store.store_message(self.class.exchange_name, message)
27
30
  result
28
31
  rescue RailwayIpc::InvalidProtobuf => e
29
- RailwayIpc.logger.error(message, 'Invalid protobuf')
32
+ RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
30
33
  raise e
31
34
  end
32
35
 
33
36
  private
34
37
 
35
38
  def ensure_message_uuid(message)
36
- if message.uuid.blank?
37
- message.uuid = SecureRandom.uuid
38
- message
39
- else
40
- message
41
- end
39
+ message.uuid = SecureRandom.uuid if message.uuid.blank?
40
+ message
42
41
  end
43
42
 
44
43
  def ensure_correlation_id(message)
45
- if message.correlation_id.blank?
46
- message.correlation_id = SecureRandom.uuid
47
- message
48
- else
49
- message
50
- end
44
+ message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
45
+ message
46
+ end
47
+
48
+ def log_message_options(message=nil)
49
+ options = { feature: 'railway_ipc_publisher', exchange: self.class.exchange_name }
50
+ message.nil? ? options : options.merge(protobuf: { type: message.class, data: message })
51
+ end
52
+ end
53
+ end
54
+
55
+ module RailwayIpc
56
+ class Publisher < Sneakers::Publisher
57
+ attr_reader :exchange_name, :message_store
58
+
59
+ def initialize(opts={})
60
+ @exchange_name = opts.fetch(:exchange_name)
61
+ @message_store = opts.fetch(:message_store, RailwayIpc::PublishedMessage)
62
+ connection = opts.fetch(:connection, nil)
63
+ options = {
64
+ exchange: exchange_name,
65
+ connection: connection,
66
+ exchange_type: :fanout
67
+ }.compact
68
+ super(options)
69
+ end
70
+
71
+ # rubocop:disable Metrics/AbcSize
72
+ def publish(message)
73
+ message.uuid = SecureRandom.uuid if message.uuid.blank?
74
+ message.correlation_id = SecureRandom.uuid if message.correlation_id.blank?
75
+ RailwayIpc.logger.info('Publishing message', log_message_options(message))
76
+
77
+ stored_message = message_store.store_message(exchange_name, message)
78
+ super(RailwayIpc::Rabbitmq::Payload.encode(message))
79
+ rescue RailwayIpc::InvalidProtobuf => e
80
+ RailwayIpc.logger.error('Invalid protobuf', log_message_options(message))
81
+ raise e
82
+ rescue ActiveRecord::RecordInvalid => e
83
+ RailwayIpc.logger.error('Failed to store outgoing message', log_message_options(message))
84
+ raise RailwayIpc::FailedToStoreOutgoingMessage.new(e)
85
+ rescue StandardError => e
86
+ stored_message&.destroy
87
+ raise e
88
+ end
89
+ # rubocop:enable Metrics/AbcSize
90
+
91
+ private
92
+
93
+ def log_message_options(message)
94
+ {
95
+ feature: 'railway_ipc_publisher',
96
+ exchange: exchange_name,
97
+ protobuf: {
98
+ type: message.class,
99
+ data: message
100
+ }
101
+ }
51
102
  end
52
103
  end
53
104
  end