messaging 0.0.2 → 3.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +68 -0
  3. data/.gitignore +11 -0
  4. data/.rspec +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +15 -0
  7. data/Gemfile.lock +222 -0
  8. data/README.md +53 -0
  9. data/Rakefile +5 -30
  10. data/_config.yml +1 -0
  11. data/bin/console +11 -0
  12. data/bin/setup +8 -0
  13. data/exe/messaging +11 -0
  14. data/lib/messaging.rb +62 -11
  15. data/lib/messaging/adapters.rb +35 -0
  16. data/lib/messaging/adapters/kafka.rb +55 -0
  17. data/lib/messaging/adapters/kafka/consumer.rb +101 -0
  18. data/lib/messaging/adapters/kafka/producer.rb +52 -0
  19. data/lib/messaging/adapters/postgres.rb +23 -0
  20. data/lib/messaging/adapters/postgres/advisory_transaction_lock.rb +28 -0
  21. data/lib/messaging/adapters/postgres/serialized_message.rb +74 -0
  22. data/lib/messaging/adapters/postgres/store.rb +66 -0
  23. data/lib/messaging/adapters/postgres/stream.rb +37 -0
  24. data/lib/messaging/adapters/postgres/streams.rb +27 -0
  25. data/lib/messaging/adapters/test.rb +59 -0
  26. data/lib/messaging/adapters/test/consumer.rb +46 -0
  27. data/lib/messaging/adapters/test/store.rb +67 -0
  28. data/lib/messaging/adapters/test/stream.rb +21 -0
  29. data/lib/messaging/base_handler.rb +45 -0
  30. data/lib/messaging/cli.rb +62 -0
  31. data/lib/messaging/config.rb +66 -0
  32. data/lib/messaging/consumer_supervisor.rb +63 -0
  33. data/lib/messaging/exception_handler.rb +15 -0
  34. data/lib/messaging/expected_version.rb +31 -0
  35. data/lib/messaging/instrumentation.rb +21 -0
  36. data/lib/messaging/message.rb +103 -0
  37. data/lib/messaging/message/from_json.rb +31 -0
  38. data/lib/messaging/meter.rb +113 -0
  39. data/lib/messaging/middleware.rb +14 -0
  40. data/lib/messaging/middleware/after_active_record_transaction.rb +13 -0
  41. data/lib/messaging/middleware/rails_wrapper.rb +22 -0
  42. data/lib/messaging/publish.rb +33 -0
  43. data/lib/messaging/rails/railtie.rb +36 -0
  44. data/lib/messaging/resque_worker.rb +11 -0
  45. data/lib/messaging/routes.rb +50 -0
  46. data/lib/messaging/routing.rb +67 -0
  47. data/lib/messaging/routing/background_job_subscriber.rb +11 -0
  48. data/lib/messaging/routing/enqueue_message_handler.rb +15 -0
  49. data/lib/messaging/routing/message_matcher.rb +62 -0
  50. data/lib/messaging/routing/subscriber.rb +35 -0
  51. data/lib/messaging/sidekiq_worker.rb +12 -0
  52. data/lib/messaging/version.rb +1 -1
  53. data/messaging.gemspec +40 -0
  54. metadata +299 -102
  55. data/MIT-LICENSE +0 -20
  56. data/app/models/message.rb +0 -5
  57. data/app/models/receipt.rb +0 -13
  58. data/app/uploaders/attachment_uploader.rb +0 -9
  59. data/db/migrate/20130801214110_initial_migration.rb +0 -22
  60. data/lib/generators/messaging/install_generator.rb +0 -26
  61. data/lib/generators/messaging/templates/initializer.rb +0 -11
  62. data/lib/messaging/concerns/configurable_mailer.rb +0 -13
  63. data/lib/messaging/engine.rb +0 -9
  64. data/lib/messaging/models/messageable.rb +0 -24
  65. data/lib/tasks/messaging_tasks.rake +0 -4
  66. data/test/dummy/README.rdoc +0 -28
  67. data/test/dummy/Rakefile +0 -6
  68. data/test/dummy/app/assets/javascripts/application.js +0 -13
  69. data/test/dummy/app/assets/stylesheets/application.css +0 -13
  70. data/test/dummy/app/controllers/application_controller.rb +0 -5
  71. data/test/dummy/app/helpers/application_helper.rb +0 -2
  72. data/test/dummy/app/views/layouts/application.html.erb +0 -14
  73. data/test/dummy/bin/bundle +0 -3
  74. data/test/dummy/bin/rails +0 -4
  75. data/test/dummy/bin/rake +0 -4
  76. data/test/dummy/config.ru +0 -4
  77. data/test/dummy/config/application.rb +0 -23
  78. data/test/dummy/config/boot.rb +0 -5
  79. data/test/dummy/config/database.yml +0 -25
  80. data/test/dummy/config/environment.rb +0 -5
  81. data/test/dummy/config/environments/development.rb +0 -29
  82. data/test/dummy/config/environments/production.rb +0 -80
  83. data/test/dummy/config/environments/test.rb +0 -36
  84. data/test/dummy/config/initializers/backtrace_silencers.rb +0 -7
  85. data/test/dummy/config/initializers/filter_parameter_logging.rb +0 -4
  86. data/test/dummy/config/initializers/inflections.rb +0 -16
  87. data/test/dummy/config/initializers/mime_types.rb +0 -5
  88. data/test/dummy/config/initializers/secret_token.rb +0 -12
  89. data/test/dummy/config/initializers/session_store.rb +0 -3
  90. data/test/dummy/config/initializers/wrap_parameters.rb +0 -14
  91. data/test/dummy/config/locales/en.yml +0 -23
  92. data/test/dummy/config/routes.rb +0 -56
  93. data/test/dummy/public/404.html +0 -58
  94. data/test/dummy/public/422.html +0 -58
  95. data/test/dummy/public/500.html +0 -57
  96. data/test/dummy/public/favicon.ico +0 -0
  97. data/test/messaging_test.rb +0 -7
  98. data/test/test_helper.rb +0 -15
@@ -0,0 +1,63 @@
1
+ module Messaging
2
+ # @private
3
+ class ConsumerSupervisor
4
+ include Messaging::Instrumentation
5
+
6
+ def initialize
7
+ @threads = Concurrent::Array.new
8
+ Messaging.routes.define_consumers!
9
+ end
10
+
11
+ def start
12
+ Concurrent.use_simple_logger
13
+ Messaging.logger.info 'Consumers starting'
14
+ @signal_to_stop = false
15
+ @threads.clear
16
+ @thread_pool = Concurrent::FixedThreadPool.new(consumers.size, auto_terminate: false)
17
+
18
+ consumers.each do |consumer|
19
+ @thread_pool.post do
20
+ thread = Thread.current
21
+ thread.abort_on_exception = true
22
+ @threads << thread
23
+ run_consumer(consumer)
24
+ end
25
+ end
26
+
27
+ true
28
+ end
29
+
30
+ def stop
31
+ return if @signal_to_stop
32
+
33
+ instrument('consumer_supervisor.stop') do
34
+ @signal_to_stop = true
35
+ consumers.map { |consumer| Thread.new { consumer.stop } }.join
36
+ @threads.select(&:alive?).each { |thread| thread&.wakeup }
37
+ @thread_pool&.shutdown
38
+ @thread_pool&.wait_for_termination(60)
39
+ Messaging.logger.info 'Consumers stopped'
40
+ end
41
+ end
42
+
43
+ def status
44
+ consumers.map(&:log_current_status)
45
+ end
46
+
47
+ def consumers
48
+ Messaging.routes.consumers
49
+ end
50
+
51
+ private
52
+
53
+ def run_consumer(consumer)
54
+ consumer.start
55
+ rescue StandardError => e
56
+ return if @signal_to_stop
57
+
58
+ ExceptionHandler.call(e)
59
+ sleep 2
60
+ retry
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,15 @@
1
+ module Messaging
2
+ class ExceptionHandler
3
+ def self.call(exception, context = {})
4
+ Config.error_handlers.each do |handler|
5
+ begin
6
+ handler.call(exception, context)
7
+ rescue => e
8
+ Messaging.logger.error '!!! ERROR HANDLER THREW AN ERROR !!!'
9
+ Messaging.logger.error e
10
+ Messaging.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,31 @@
1
+ module Messaging
2
+ class ExpectedVersion
3
+ Error = Class.new(StandardError)
4
+
5
+ attr_reader :version
6
+
7
+ def initialize(version)
8
+ @version = version
9
+ end
10
+
11
+ def any?
12
+ version == :any
13
+ end
14
+
15
+ def none?
16
+ version == :none || version == -1
17
+ end
18
+
19
+ def matches?(other_version)
20
+ return true if any?
21
+ return true if none? && other_version == -1
22
+ return true if version == other_version
23
+ false
24
+ end
25
+
26
+ def match!(other_version)
27
+ return true if matches?(other_version)
28
+ raise Error, "expected: #{version} actual: #{other_version}"
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,21 @@
1
+ module Messaging
2
+ module Instrumentation
3
+ NAMESPACE = 'messaging'
4
+
5
+ def instrument(event, extra = {})
6
+ ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |extra|
7
+ yield(extra) if block_given?
8
+ end
9
+ end
10
+
11
+ def self.subscribe(event)
12
+ ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
13
+ yield ActiveSupport::Notifications::Event.new(*args) if block_given?
14
+ end
15
+ end
16
+
17
+ def self.unsubscribe(subscriber)
18
+ ActiveSupport::Notifications.unsubscribe(subscriber)
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,103 @@
1
+ require 'active_support'
2
+ require 'active_support/concern'
3
+ require 'active_support/core_ext/class'
4
+ require 'active_support/core_ext/class/attribute'
5
+ require 'active_support/core_ext/time'
6
+ require 'dry-equalizer'
7
+
8
+ module Messaging
9
+ module Message
10
+ extend ActiveSupport::Concern
11
+
12
+ included do
13
+ include Virtus.model
14
+ include Dry::Equalizer(:attributes)
15
+
16
+ attr_accessor :stream_position
17
+ attr_accessor :expected_version
18
+
19
+ key_attribute :uuid
20
+
21
+ attribute :message_name, String, default: ->(message, _) { message.class.message_name }
22
+
23
+ attribute :timestamp, Time, default: ->(*) { Time.current.utc }
24
+ attribute :uuid, String, default: ->(*) { SecureRandom.uuid }
25
+ end
26
+
27
+ module ClassMethods
28
+ # By default the topic is the same as the name of the message.
29
+ # We change the / that would be set for a namespaced message as "/" isn't valid in a topic
30
+ # To change the topic for a message just set it to whatever you want in your class definition.
31
+ def topic(topic_name = nil)
32
+ @topic ||= topic_name&.to_s || default_topic_name
33
+ end
34
+
35
+ # The attribute which value should be used as the key of the message.
36
+ # Must specify an attribute if ordering is important.
37
+ def key_attribute(attribute = nil)
38
+ @key_attribute = attribute if attribute
39
+ @key_attribute
40
+ end
41
+
42
+ def default_topic_name
43
+ return superclass.topic if superclass.respond_to?(:topic)
44
+
45
+ message_name.gsub('/', '-')
46
+ end
47
+
48
+ # Shorcut for creating a new message and publishing it
49
+ #
50
+ # @param attributes [Hash] The attributes of the message
51
+ # @option attributes [:any, Integer] :expected_version Concurrency control
52
+ # @return [Message] the message just published
53
+ # @raise [ExpectedVersion::Error] If the expected_version does not match
54
+ def publish(attributes)
55
+ new(attributes).publish
56
+ end
57
+
58
+ def message_name
59
+ name.underscore
60
+ end
61
+
62
+ def message_type
63
+ to_s
64
+ end
65
+ end
66
+
67
+ # We do our own conversion for datetimes as by default they will only
68
+ # be stored with microseconds, which makes us loose precision when we
69
+ # serialize to json and back to message objects
70
+ def attributes_as_json(*_args)
71
+ attributes.transform_values { |v| v.respond_to?(:nsec) ? v.iso8601(9) : v }
72
+ end
73
+
74
+ def as_json(*_args)
75
+ attributes_as_json.merge(stream_position: stream_position)
76
+ end
77
+
78
+ def message_key
79
+ attributes[key_attribute&.to_sym]
80
+ end
81
+
82
+ def key_attribute
83
+ self.class.key_attribute
84
+ end
85
+
86
+ def topic
87
+ self.class.topic
88
+ end
89
+
90
+ def stream_name
91
+ # define stream_name in your message class to override
92
+ nil
93
+ end
94
+
95
+ def message_type
96
+ self.class.to_s
97
+ end
98
+
99
+ def publish
100
+ Messaging::Publish.(message: self)
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,31 @@
1
+ module Messaging
2
+ module Message
3
+ class FromJson
4
+
5
+ def self.call(json)
6
+ new(json).to_message
7
+ end
8
+
9
+ def initialize(json)
10
+ @attributes = JSON.parse(json, symbolize_names: true)
11
+ @klass = message_class_from_name(@attributes[:message_name])
12
+ end
13
+
14
+ def empty_message
15
+ Hash[@klass.attribute_set.map { |a| [a.name, nil] }]
16
+ end
17
+
18
+ def new_without_defaults(attributes)
19
+ @klass.new(empty_message.merge(attributes))
20
+ end
21
+
22
+ def to_message
23
+ new_without_defaults(@attributes)
24
+ end
25
+
26
+ def message_class_from_name(message_name)
27
+ "::#{message_name.camelize}".constantize
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,113 @@
1
+ ActiveSupport::Notifications.subscribe /request.connection.kafka/ do |*args|
2
+ event = ActiveSupport::Notifications::Event.new(*args)
3
+ if event.payload.key?(:exception)
4
+ tags = {
5
+ api: event.payload.fetch(:api, 'unknown'),
6
+ broker: event.payload.fetch(:broker_host)
7
+ }
8
+ Meter.increment('messaging.api.errors', tags: tags)
9
+ end
10
+ end
11
+
12
+ ActiveSupport::Notifications.subscribe /process_message.consumer.kafka/ do |*args|
13
+ event = ActiveSupport::Notifications::Event.new(*args)
14
+
15
+ tags = {
16
+ group_id: event.payload.fetch(:group_id),
17
+ topic: event.payload.fetch(:topic),
18
+ partition: event.payload.fetch(:partition),
19
+ }
20
+
21
+ if event.payload.key?(:exception)
22
+ Meter.increment('messaging.consumer.process_message.errors', tags: tags)
23
+ else
24
+ Meter.increment('messaging.consumer.messages', tags: tags)
25
+ end
26
+ end
27
+
28
+ ActiveSupport::Notifications.subscribe /process_batch.consumer.kafka/ do |*args|
29
+ event = ActiveSupport::Notifications::Event.new(*args)
30
+ message_count = event.payload.fetch(:message_count)
31
+
32
+ tags = {
33
+ group_id: event.payload.fetch(:group_id),
34
+ topic: event.payload.fetch(:topic),
35
+ partition: event.payload.fetch(:partition),
36
+ }
37
+
38
+ if event.payload.key?(:exception)
39
+ Meter.increment('messaging.consumer.process_batch.errors', tags: tags)
40
+ else
41
+ Meter.increment('messaging.consumer.messages', value: message_count, tags: tags)
42
+ end
43
+ end
44
+
45
+ ActiveSupport::Notifications.subscribe /produce_message.producer.kafka/ do |*args|
46
+ event = ActiveSupport::Notifications::Event.new(*args)
47
+ topic = event.payload.fetch(:topic)
48
+ buffer_size = event.payload.fetch(:buffer_size)
49
+ max_buffer_size = event.payload.fetch(:max_buffer_size)
50
+ buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
51
+
52
+ # This gets us the write rate.
53
+ Meter.increment('messaging.producer.produce.messages', tags: { topic: topic })
54
+
55
+ # This gets us the avg/max buffer size per producer.
56
+ Meter.histogram('messaging.producer.buffer.size', buffer_size)
57
+
58
+ # This gets us the avg/max buffer fill ratio per producer.
59
+ Meter.histogram('messaging.producer.buffer.fill_ratio', buffer_fill_ratio)
60
+ end
61
+
62
+ ActiveSupport::Notifications.subscribe /buffer_overflow.producer.kafka/ do |*args|
63
+ event = ActiveSupport::Notifications::Event.new(*args)
64
+ tags = {
65
+ topic: event.payload.fetch(:topic)
66
+ }
67
+
68
+ Meter.increment('messaging.producer.produce.errors', tags: tags)
69
+ end
70
+
71
+ ActiveSupport::Notifications.subscribe /deliver_messages.producer.kafka/ do |*args|
72
+ event = ActiveSupport::Notifications::Event.new(*args)
73
+ message_count = event.payload.fetch(:delivered_message_count)
74
+ attempts = event.payload.fetch(:attempts)
75
+
76
+ Meter.increment('messaging.producer.deliver.errors') if event.payload.key?(:exception)
77
+
78
+ # Messages delivered to Kafka:
79
+ Meter.increment('messaging.producer.deliver.messages', value: message_count)
80
+
81
+ # Number of attempts to deliver messages:
82
+ Meter.histogram('messaging.producer.deliver.attempts', attempts)
83
+ end
84
+
85
+ ActiveSupport::Notifications.subscribe /topic_error.producer.kafka/ do |*args|
86
+ event = ActiveSupport::Notifications::Event.new(*args)
87
+ tags = {
88
+ topic: event.payload.fetch(:topic)
89
+ }
90
+
91
+ Meter.increment('messaging.producer.ack.errors', tags: tags)
92
+ end
93
+
94
+ ActiveSupport::Notifications.subscribe /enqueue_message.async_producer.kafka/ do |*args|
95
+ event = ActiveSupport::Notifications::Event.new(*args)
96
+ queue_size = event.payload.fetch(:queue_size)
97
+ max_queue_size = event.payload.fetch(:max_queue_size)
98
+ queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
99
+
100
+ # This gets us the avg/max queue size per producer.
101
+ Meter.histogram('messaging.async_producer.queue.size', queue_size)
102
+
103
+ # This gets us the avg/max queue fill ratio per producer.
104
+ Meter.histogram('messaging.async_producer.queue.fill_ratio', queue_fill_ratio)
105
+ end
106
+
107
+ ActiveSupport::Notifications.subscribe /buffer_overflow.async_producer.kafka/ do |*args|
108
+ event = ActiveSupport::Notifications::Event.new(*args)
109
+ tags = {
110
+ topic: event.payload.fetch(:topic)
111
+ }
112
+ Meter.increment('messaging.async_producer.produce.errors', tags: tags)
113
+ end
@@ -0,0 +1,14 @@
1
+ module Messaging
2
+ module Middleware
3
+ def self.run(middlewares, *args)
4
+ chain = Array(middlewares.dup)
5
+
6
+ call_stack = lambda do
7
+ return yield if chain.empty?
8
+
9
+ chain.shift.call(*args, &call_stack)
10
+ end
11
+ call_stack.call
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,13 @@
1
+ require 'after_transaction'
2
+
3
+ module Messaging
4
+ module Middleware
5
+ class AfterActiveRecordTransaction
6
+ def self.call(*_args)
7
+ AfterTransaction.call do
8
+ yield
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,22 @@
1
+ module Messaging
2
+ module Middleware
3
+ # Used for hot code reloading (in development) and connection handling.
4
+ # This makes Rails automatically return AR connections to the pool after we
5
+ # have used them when processing a messages.
6
+ #
7
+ # See https://guides.rubyonrails.org/threading_and_code_execution.html for more information.
8
+ class RailsWrapper
9
+ attr_accessor :app
10
+
11
+ def initialize(app)
12
+ self.app = app
13
+ end
14
+
15
+ def call(*_args)
16
+ app.reloader.wrap do
17
+ yield
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,33 @@
1
+ module Messaging
2
+ class Publish
3
+ include MethodObject
4
+
5
+ option :message
6
+
7
+ option :message_store, default: -> { Messaging.message_store }
8
+ option :dispatcher, default: -> { Messaging.dispatcher }
9
+ option :middlewares, default: -> { Config.dispatcher.middlewares }
10
+
11
+ class << self
12
+ # @!method call(message:, stream:)
13
+ # Publishes the message
14
+ #
15
+ # This will persist the message first (if given a stream),
16
+ # if that succeeds it will be dispatched to the configured
17
+ # dispatcher and all handlers configured in the routes.
18
+ # @param message [Message] The message to publish
19
+ # @return [Message] the message just published
20
+ end
21
+
22
+ # @see .call
23
+ # @!visibility private
24
+ def call
25
+ persisted_message = message_store.call(message)
26
+ Middleware.run(middlewares, persisted_message) do
27
+ dispatcher.call(persisted_message)
28
+ Messaging.routes.handle persisted_message
29
+ end
30
+ persisted_message
31
+ end
32
+ end
33
+ end