messaging 0.0.2 → 3.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +68 -0
  3. data/.gitignore +11 -0
  4. data/.rspec +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +15 -0
  7. data/Gemfile.lock +222 -0
  8. data/README.md +53 -0
  9. data/Rakefile +5 -30
  10. data/_config.yml +1 -0
  11. data/bin/console +11 -0
  12. data/bin/setup +8 -0
  13. data/exe/messaging +11 -0
  14. data/lib/messaging.rb +62 -11
  15. data/lib/messaging/adapters.rb +35 -0
  16. data/lib/messaging/adapters/kafka.rb +55 -0
  17. data/lib/messaging/adapters/kafka/consumer.rb +101 -0
  18. data/lib/messaging/adapters/kafka/producer.rb +52 -0
  19. data/lib/messaging/adapters/postgres.rb +23 -0
  20. data/lib/messaging/adapters/postgres/advisory_transaction_lock.rb +28 -0
  21. data/lib/messaging/adapters/postgres/serialized_message.rb +74 -0
  22. data/lib/messaging/adapters/postgres/store.rb +66 -0
  23. data/lib/messaging/adapters/postgres/stream.rb +37 -0
  24. data/lib/messaging/adapters/postgres/streams.rb +27 -0
  25. data/lib/messaging/adapters/test.rb +59 -0
  26. data/lib/messaging/adapters/test/consumer.rb +46 -0
  27. data/lib/messaging/adapters/test/store.rb +67 -0
  28. data/lib/messaging/adapters/test/stream.rb +21 -0
  29. data/lib/messaging/base_handler.rb +45 -0
  30. data/lib/messaging/cli.rb +62 -0
  31. data/lib/messaging/config.rb +66 -0
  32. data/lib/messaging/consumer_supervisor.rb +63 -0
  33. data/lib/messaging/exception_handler.rb +15 -0
  34. data/lib/messaging/expected_version.rb +31 -0
  35. data/lib/messaging/instrumentation.rb +21 -0
  36. data/lib/messaging/message.rb +103 -0
  37. data/lib/messaging/message/from_json.rb +31 -0
  38. data/lib/messaging/meter.rb +113 -0
  39. data/lib/messaging/middleware.rb +14 -0
  40. data/lib/messaging/middleware/after_active_record_transaction.rb +13 -0
  41. data/lib/messaging/middleware/rails_wrapper.rb +22 -0
  42. data/lib/messaging/publish.rb +33 -0
  43. data/lib/messaging/rails/railtie.rb +36 -0
  44. data/lib/messaging/resque_worker.rb +11 -0
  45. data/lib/messaging/routes.rb +50 -0
  46. data/lib/messaging/routing.rb +67 -0
  47. data/lib/messaging/routing/background_job_subscriber.rb +11 -0
  48. data/lib/messaging/routing/enqueue_message_handler.rb +15 -0
  49. data/lib/messaging/routing/message_matcher.rb +62 -0
  50. data/lib/messaging/routing/subscriber.rb +35 -0
  51. data/lib/messaging/sidekiq_worker.rb +12 -0
  52. data/lib/messaging/version.rb +1 -1
  53. data/messaging.gemspec +40 -0
  54. metadata +299 -102
  55. data/MIT-LICENSE +0 -20
  56. data/app/models/message.rb +0 -5
  57. data/app/models/receipt.rb +0 -13
  58. data/app/uploaders/attachment_uploader.rb +0 -9
  59. data/db/migrate/20130801214110_initial_migration.rb +0 -22
  60. data/lib/generators/messaging/install_generator.rb +0 -26
  61. data/lib/generators/messaging/templates/initializer.rb +0 -11
  62. data/lib/messaging/concerns/configurable_mailer.rb +0 -13
  63. data/lib/messaging/engine.rb +0 -9
  64. data/lib/messaging/models/messageable.rb +0 -24
  65. data/lib/tasks/messaging_tasks.rake +0 -4
  66. data/test/dummy/README.rdoc +0 -28
  67. data/test/dummy/Rakefile +0 -6
  68. data/test/dummy/app/assets/javascripts/application.js +0 -13
  69. data/test/dummy/app/assets/stylesheets/application.css +0 -13
  70. data/test/dummy/app/controllers/application_controller.rb +0 -5
  71. data/test/dummy/app/helpers/application_helper.rb +0 -2
  72. data/test/dummy/app/views/layouts/application.html.erb +0 -14
  73. data/test/dummy/bin/bundle +0 -3
  74. data/test/dummy/bin/rails +0 -4
  75. data/test/dummy/bin/rake +0 -4
  76. data/test/dummy/config.ru +0 -4
  77. data/test/dummy/config/application.rb +0 -23
  78. data/test/dummy/config/boot.rb +0 -5
  79. data/test/dummy/config/database.yml +0 -25
  80. data/test/dummy/config/environment.rb +0 -5
  81. data/test/dummy/config/environments/development.rb +0 -29
  82. data/test/dummy/config/environments/production.rb +0 -80
  83. data/test/dummy/config/environments/test.rb +0 -36
  84. data/test/dummy/config/initializers/backtrace_silencers.rb +0 -7
  85. data/test/dummy/config/initializers/filter_parameter_logging.rb +0 -4
  86. data/test/dummy/config/initializers/inflections.rb +0 -16
  87. data/test/dummy/config/initializers/mime_types.rb +0 -5
  88. data/test/dummy/config/initializers/secret_token.rb +0 -12
  89. data/test/dummy/config/initializers/session_store.rb +0 -3
  90. data/test/dummy/config/initializers/wrap_parameters.rb +0 -14
  91. data/test/dummy/config/locales/en.yml +0 -23
  92. data/test/dummy/config/routes.rb +0 -56
  93. data/test/dummy/public/404.html +0 -58
  94. data/test/dummy/public/422.html +0 -58
  95. data/test/dummy/public/500.html +0 -57
  96. data/test/dummy/public/favicon.ico +0 -0
  97. data/test/messaging_test.rb +0 -7
  98. data/test/test_helper.rb +0 -15
@@ -0,0 +1,66 @@
1
+ require_relative 'stream'
2
+ require_relative 'streams'
3
+
4
+ module Messaging
5
+ module Adapters
6
+ class Postgres
7
+ # Message store adapter using Postgres and ActiveRecord.
8
+ # Prefer accessing the message store through Messaging.message_store
9
+ # instead of using it directly.
10
+ #
11
+ # @example Using Postgres as the default message store adapter:
12
+ # # Put this in an initializer
13
+ # Messaging.setup do |config|
14
+ # config.message_store.adapter = :postgres
15
+ # end
16
+ #
17
+ # @see Messaging.message_store
18
+ class Store
19
+ # @return [Streams] all the streams in the store
20
+ # @see Streams
21
+ attr_reader :streams
22
+
23
+ # Should not be used directly. Access the store though
24
+ # Messaging.message_store or Messaging::Adapters::Store[:postgres]
25
+ # @api private
26
+ def initialize
27
+ @streams = Streams.new
28
+ end
29
+
30
+ # Get a specific stream by name
31
+ # @return [Stream]
32
+ # @see Messaging.stream
33
+ def stream(name)
34
+ streams[name]
35
+ end
36
+
37
+ # Access to all messages.
38
+ # Use with caution in production as there are probably
39
+ # a lot of messages so queries could take a long time or timeout.
40
+ #
41
+ # @example Check that a message has been added to the store with Rspec
42
+ # expect do
43
+ # # Your code that should add a message to the store
44
+ # end.to change { Messaging.message_store.messages.count }.from(0).to(1)
45
+ #
46
+ # @return [SerializedMessage]
47
+ def messages
48
+ SerializedMessage
49
+ end
50
+
51
+ # Writes the message to Postgres
52
+ # Skips messages that hasn't defined a stream name
53
+ # We do this to begin with so PG is opt-in per message
54
+ #
55
+ # @param message [Messaging::Message] The message to persist
56
+ # @return [Messaging::Message] A new copy of the message with
57
+ # the stream position added (if persisted)
58
+ def call(message)
59
+ return message unless message.stream_name
60
+
61
+ SerializedMessage.create!(message: message).to_message
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,37 @@
1
+ module Messaging
2
+ module Adapters
3
+ class Postgres
4
+ class Stream
5
+ # @return [String] the name of the stream
6
+ attr_reader :name
7
+
8
+ # Should not be used directly.
9
+ # Use {Messaging.stream} or {Store#stream}
10
+ # @api private
11
+ # @see Messaging.stream
12
+ # @see Store.stream
13
+ def initialize(name)
14
+ @name = name
15
+ end
16
+
17
+ # Access to all messages in the stream sorted by stream_position
18
+ # @return [ActiveRecord::Relation]
19
+ def messages
20
+ SerializedMessage.where(stream: name).order(:stream_position)
21
+ end
22
+
23
+ # The current position of the last message in the stream
24
+ # @return [-1] if the stream is empty -1 will be returned
25
+ # @return [Integer] the stream position of the last message in the stream
26
+ def current_position
27
+ messages.maximum(:stream_position) || -1
28
+ end
29
+
30
+ def inspect
31
+ info = "current_position: #{current_position}"
32
+ "#<Stream:#{name}> #{info}>"
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,27 @@
1
+ module Messaging
2
+ module Adapters
3
+ class Postgres
4
+ class Streams
5
+ include Enumerable
6
+
7
+ def each
8
+ return enum_for(:each) unless block_given?
9
+
10
+ all_streams.each do |name|
11
+ yield Stream.new(name)
12
+ end
13
+ end
14
+
15
+ def [](name)
16
+ Stream.new(name)
17
+ end
18
+
19
+ private
20
+
21
+ def all_streams
22
+ SerializedMessage.distinct.pluck(:stream).lazy
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,59 @@
1
+ require_relative 'test/consumer'
2
+ require_relative 'test/store'
3
+ module Messaging
4
+ module Adapters
5
+ # Public: An adapter useful for test.
6
+ #
7
+ # Do NOT use this outside tests.
8
+ # All messages will be saved in memory until cleared.
9
+ class Test
10
+ # Public: Used to inspect what have been published
11
+ #
12
+ # It's a hash keyed on the topic name.
13
+ # Each value is an array of the messages that has been published to the topic
14
+ attr_reader :topics
15
+
16
+ def initialize
17
+ # See https://www.rubytapas.com/2013/01/11/episode-045-hash-default-value/
18
+ # for why this is needed for default values in a Hash
19
+ @topics = Hash.new { |h, k| h[k] = [] }
20
+ end
21
+
22
+ def call(message)
23
+ topics[message.topic] << message
24
+ Messaging.routes.consumers.each do |c|
25
+ Messaging.logger.info "Sending #{message} to #{c.queue} with length #{c.queue.length}"
26
+ c.queue << message
27
+ end
28
+ end
29
+
30
+ def create_consumer(name, **options)
31
+ Consumer.new(name: name, **options)
32
+ end
33
+
34
+ # Resests the topics to a blank state.
35
+ #
36
+ # Useful to run before each example in specs to not share state.
37
+ def clear_topics
38
+ topics.clear
39
+ end
40
+
41
+ # @api private
42
+ def store
43
+ @store ||= Store.new
44
+ end
45
+
46
+ def self.register!
47
+ return if Adapters.key? :test
48
+
49
+ Adapters.register(:test, memoize: true) { Test.new }
50
+ Adapters::Consumer.register(:test, memoize: true) { Adapters[:test] }
51
+ Adapters::Dispatcher.register(:test, memoize: true) { Adapters[:test] }
52
+ Adapters::Store.register(:test, memoize: true) { Adapters[:test].store }
53
+ end
54
+ private_class_method :register!
55
+
56
+ register!
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,46 @@
1
+ require 'dry-initializer'
2
+ module Messaging
3
+ module Adapters
4
+ class Test
5
+ class Consumer
6
+ include Messaging::Routing
7
+ extend Dry::Initializer
8
+
9
+ option :name
10
+
11
+ option :logger, default: -> { Config.logger }
12
+ option :queue, default: -> { Queue.new }
13
+
14
+ def start
15
+ logger.info "Consumer #{name} started"
16
+ @running = true
17
+ process_messages
18
+ ensure
19
+ shutdown
20
+ end
21
+
22
+ def stop
23
+ @running = false
24
+ queue << :stop
25
+ logger.info "Consumer #{name} stopping"
26
+ end
27
+
28
+ def shutdown
29
+ logger.info "Consumer #{name} stopped"
30
+ end
31
+
32
+ private
33
+
34
+ def process_messages
35
+ while @running
36
+ message = queue.pop
37
+ logger.info "Consumer #{name} got #{message}"
38
+ break if message == :stop
39
+
40
+ RunMiddlewares.call(Config.consumer.middlewares, message) { handle message }
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,67 @@
1
+ require_relative 'stream'
2
+
3
+ module Messaging
4
+ module Adapters
5
+ class Test
6
+ # Message store adapter used for testing.
7
+ # It stores all messages in memory.
8
+ # Prefer accessing the message store through Messaging.message_store
9
+ # instead of using it directly.
10
+ #
11
+ # @example Using Test as the default message store adapter during tests:
12
+ # # Put this in an initializer
13
+ # Messaging.setup do |config|
14
+ # config.message_store.adapter = Rails.env.test? ? :test : :postgres
15
+ # end
16
+ #
17
+ # @see Messaging.message_store
18
+ class Store
19
+ # Access to all messages.
20
+ #
21
+ # @example Check that a message has been added to the store with Rspec
22
+ # expect do
23
+ # # Your code that should add a message to the store
24
+ # end.to change { Messaging.message_store.messages.count }.from(0).to(1)
25
+ #
26
+ # @return [Array<Messaging::Message>]
27
+ attr_reader :messages
28
+ # @return [Streams] all the streams in the store
29
+ # @see Stream
30
+ attr_reader :streams
31
+
32
+ def initialize
33
+ clear!
34
+ end
35
+
36
+ # Get a specific stream by name
37
+ # @return [Stream]
38
+ # @see Messaging.strea
39
+ def stream(name)
40
+ streams[name]
41
+ end
42
+
43
+ def clear!
44
+ @streams = Hash.new { |h, k| h[k] = Stream.new(k) }
45
+ @messages = []
46
+ end
47
+
48
+ # Writes the message to the store
49
+ # Skips messages that hasn't defined a stream name
50
+ #
51
+ # @param message [Messaging::Message] The message to persist
52
+ # @return [Messaging::Message] A new copy of the message with
53
+ # the stream position added (if persisted)
54
+ def call(message)
55
+ return message unless message.stream_name
56
+
57
+ stream = stream(message.stream_name)
58
+ ExpectedVersion.new(message.expected_version || :any).match!(stream.current_position)
59
+ persisted_message = message.class.new(message.attributes.merge(stream_position: stream.current_position + 1))
60
+ @messages << persisted_message
61
+ stream.messages << persisted_message
62
+ persisted_message
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,21 @@
1
+ module Messaging
2
+ module Adapters
3
+ class Test
4
+ class Stream
5
+ attr_accessor :name
6
+
7
+ def initialize(name)
8
+ self.name = name
9
+ end
10
+
11
+ def messages
12
+ @messages ||= []
13
+ end
14
+
15
+ def current_position
16
+ messages.length - 1
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Messaging
4
+ # @private
5
+ # @deprecated
6
+ class BaseHandler
7
+ class << self
8
+ def only_messages(messages = nil)
9
+ @only_messages = Array(messages) if messages
10
+ @only_messages || []
11
+ end
12
+ alias only_message only_messages
13
+
14
+ def group_id(group_id = nil)
15
+ @group_id = group_id if group_id
16
+ @group_id || default_group_id
17
+ end
18
+
19
+ def default_group_id
20
+ Config.app_name + '-' + name.underscore
21
+ end
22
+
23
+ def call(message)
24
+ return unless allowed_message?(message.class)
25
+
26
+ new.on_message(message, nil)
27
+ end
28
+
29
+ def listen_on(topic:)
30
+ topics = Array(topic).map(&:to_s)
31
+ Messaging.routes.consumer(name, group_id: group_id) do |c|
32
+ topics.each { |t| c.on(/.*/, topic: t, call: self) }
33
+ end
34
+ end
35
+
36
+ def allowed_message?(message_class)
37
+ return true if only_messages.empty?
38
+
39
+ only_messages.include? message_class
40
+ end
41
+ end
42
+
43
+ def on_message(message, _metadata); end
44
+ end
45
+ end
@@ -0,0 +1,62 @@
1
+ require 'messaging'
2
+ require_relative 'consumer_supervisor'
3
+
4
+ module Messaging
5
+ # @private
6
+ # Starts up the consumer supervisor and handles signals
7
+ class CLI
8
+
9
+ SIGNALS = %i[INT TERM QUIT].freeze
10
+
11
+ def initialize
12
+ @signal_queue = []
13
+ @reader, @writer = IO.pipe
14
+ @consumer_supervisor = Messaging::ConsumerSupervisor.new
15
+ end
16
+
17
+ def run
18
+ setup_signals
19
+ consumer_supervisor.start
20
+ wait_for_signal
21
+ end
22
+
23
+ # See https://www.sitepoint.com/the-self-pipe-trick-explained/ for background on this
24
+ def wait_for_signal
25
+ loop do
26
+ case signal_queue.pop
27
+ when *SIGNALS
28
+ consumer_supervisor.stop
29
+ break
30
+ when "USR1"
31
+ consumer_supervisor.status
32
+ else
33
+ ready = IO.select([reader, writer])
34
+
35
+ # drain the self-pipe so it won't be returned again next time
36
+ reader.read_nonblock(1) if ready[0].include?(reader)
37
+ end
38
+ end
39
+ end
40
+
41
+ private
42
+
43
+ attr_reader :reader, :writer, :signal_queue, :consumer_supervisor
44
+
45
+ def logger
46
+ @logger ||= Logger.new(STDOUT)
47
+ end
48
+
49
+ def setup_signals
50
+ SIGNALS.each do |signal|
51
+ Signal.trap(signal) { unblock(signal) }
52
+ end
53
+ Signal.trap("USR1") { unblock("USR1") }
54
+ end
55
+
56
+ def unblock(signal)
57
+ writer.write_nonblock('.')
58
+ signal_queue << signal
59
+ end
60
+
61
+ end
62
+ end
@@ -0,0 +1,66 @@
1
+ module Messaging
2
+ class Config
3
+ extend ::Dry::Configurable
4
+
5
+ setting :app_name, 'messaging', reader: true
6
+
7
+ setting(:error_handlers, [], reader: true) { |value| Array(value) }
8
+
9
+ setting :background_job_handler, ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
10
+
11
+ setting :logger, Logger.new(STDOUT), reader: true
12
+
13
+ setting :consumer, reader: true do
14
+ setting :adapter, :kafka
15
+ setting(:middlewares, []) { |value| Array(value) }
16
+ end
17
+
18
+ setting :dispatcher, reader: true do
19
+ setting :adapter, :kafka
20
+ setting(:middlewares, []) { |value| Array(value) }
21
+ end
22
+
23
+ setting :message_store, reader: true do
24
+ setting :adapter, :postgres
25
+ setting(:middlewares, []) { |value| Array(value) }
26
+ end
27
+
28
+ setting :kafka, reader: true do
29
+ setting :log_level, :warn
30
+ setting :pause_timeout, 10
31
+
32
+ setting :client do
33
+ setting :seed_brokers, ['localhost:9092']
34
+ setting :connect_timeout
35
+ setting :socket_timeout
36
+ setting :ssl_ca_certs_from_system
37
+ setting :sasl_plain_username
38
+ setting :sasl_plain_password
39
+ setting :ssl_ca_cert, ENV['KAFKA_SSL_CA']
40
+ setting :ssl_client_cert, ENV['KAFKA_SSL_CERTIFICATE']
41
+ setting :ssl_client_cert_key, ENV['KAFKA_SSL_KEY']
42
+ end
43
+
44
+ setting :consumer do
45
+ setting :session_timeout, 30
46
+ setting :offset_commit_interval, 10
47
+ setting :offset_commit_threshold, 0
48
+ setting :heartbeat_interval, 10
49
+ end
50
+
51
+ setting :producer do
52
+ setting :max_queue_size, 5_000
53
+ setting :delivery_threshold, 10
54
+ setting :delivery_interval, 0.05
55
+ end
56
+ end
57
+
58
+ class << self
59
+ def setup
60
+ configure do |config|
61
+ yield(config)
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end