euston-daemons 1.0.5 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,22 +0,0 @@
1
- module Euston
2
- module Daemons
3
- class ChannelThread
4
- include ComponentShutdown
5
-
6
- def run &blk
7
- begin
8
- channel = AMQP::Channel.new
9
- channel.prefetch(1)
10
-
11
- yield channel
12
- rescue => e
13
- Thread.current[:exception] = e
14
- ensure
15
- channel.disconnect unless channel.nil?
16
- end
17
-
18
- check_exception_and_shutdown
19
- end
20
- end
21
- end
22
- end
@@ -1,22 +0,0 @@
1
- module Euston
2
- module Daemons
3
- module ComponentShutdown
4
- attr_writer :daemon
5
-
6
- def check_exception_and_shutdown(errors=[])
7
- if errors.compact.empty?
8
- errors << Thread.current[:exception]
9
- return if errors.compact.empty?
10
- end
11
- errors.compact.each do |e|
12
- if e.respond_to?(:cause) #its a serious (Java?) exception
13
- Thread.current[:stop] = true
14
- @daemon.irrecoverable_signal_from_component! e
15
- else
16
- Safely.report! e
17
- end
18
- end
19
- end
20
- end
21
- end
22
- end
@@ -1,27 +0,0 @@
1
- module Euston
2
- module Daemon
3
- def errors
4
- @errors ||= Queue.new
5
- end
6
-
7
- def irrecoverable_signal_from_component!(e)
8
- self.errors.push e
9
- @queue.push "SHUTDOWN"
10
- end
11
-
12
- def report_shutdown_reasons
13
- sleep(0.25)
14
- reports = []
15
- until self.errors.empty? do
16
- e = self.errors.pop(true) rescue nil
17
- next if e.nil?
18
- case e
19
- when NativeException
20
- Safely.report!(e.cause)
21
- else
22
- Safely.report!(e)
23
- end
24
- end
25
- end
26
- end
27
- end
@@ -1,56 +0,0 @@
1
- # module Euston
2
- # module Daemons
3
- # class HandlerBindingsComponent
4
- # include Euston::Daemons::ComponentShutdown
5
-
6
- # attr_reader :cli_threads
7
-
8
- # def initialize
9
- # @cli_threads = {}
10
- # end
11
-
12
- # def start
13
- # inspector = Euston::RabbitMq::HandlerInspector.new
14
- # inspector.namespaces.push *handler_namespaces
15
- # inspector.when(:handler_found => bind_thread_to_handler).start
16
- # end
17
-
18
- # def thread_state
19
- # states = {}
20
- # @cli_threads.each { |key, thread| out[key] = thread.status }
21
- # states.inspect
22
- # end
23
-
24
- # def stop
25
- # @cli_threads.values.each { |th| th[:stop] = true }
26
- # end
27
-
28
- # private
29
-
30
- # def bind_thread_to_handler namespace, handler
31
- # @cli_threads["#{namespace}.#{handler}"] = Thread.new(namespace, handler) do |namespace, handler|
32
- # begin
33
- # channel = AMQP::Channel.new
34
- # channel.prefetch(1)
35
-
36
- # EUSTON_LOG.debug "Listening on: #{namespace}.#{handler}"
37
-
38
- # binding = HandlerBinding.new channel, namespace, handler, routing_key_prefix
39
- # binding.bind
40
- # binding.listen
41
- # rescue => e
42
- # Thread.current[:exception] = e
43
- # ensure
44
- # channel.disconnect
45
- # end
46
-
47
- # check_exception_and_shutdown
48
- # end
49
- # end
50
-
51
- # def handler_namespaces
52
- # # abstract
53
- # end
54
- # end
55
- # end
56
- # end
@@ -1,71 +0,0 @@
1
- class RabbitMQClient
2
- class Queue
3
- include Hollywood
4
-
5
- attr_writer :timeout
6
-
7
- def delivery_timeout
8
- @timeout ||= 500
9
- end
10
-
11
- def safe_subscribe
12
- _consumer = self.consumer
13
- until Thread.current[:stop] do
14
- safe_subscribe_with_timeout(_consumer, self.delivery_timeout)
15
- end
16
- end
17
-
18
- def safe_handle_message(msg)
19
- begin
20
- message = JSON.parse_sym(msg.body)
21
- begin
22
- callback :message_received, message
23
- msg.ack!
24
- rescue Euston::EventStore::ConcurrencyError
25
- msg.reject! true #requeue
26
- rescue => e
27
- callback :message_failed, message, e, msg
28
- Safely.report! e
29
- end
30
- rescue => e
31
- callback :message_decode_failed, msg.body, e
32
- msg.ack!
33
- Safely.report! e
34
- end
35
- end
36
-
37
- def consumer(auto_ack=false)
38
- consumer = QueueingConsumer.new(@channel)
39
- @channel.basic_consume(@name, auto_ack, consumer)
40
- consumer
41
- end
42
-
43
- def safe_subscribe_with_timeout(consumer, timeout=500)
44
- loop do
45
- delivery = nil
46
- begin
47
- delivery = consumer.next_delivery(timeout)
48
- rescue NativeException => e
49
- Thread.current[:exception] = e
50
- break
51
- end
52
- break if delivery.nil?
53
- remsg = ReactiveMessage.new(@channel, delivery, String.from_java_bytes(delivery.get_body))
54
- safe_handle_message remsg
55
- @channel.basic_ack(delivery.envelope.delivery_tag, false) if remsg.should_acknowledge?
56
- end
57
- end
58
- end
59
-
60
- class ReactiveMessage
61
- def reject(opts={})
62
- self.reject!(opts.fetch(:requeue, true))
63
- end
64
- def ack(multiple=false)
65
- self.ack!
66
- end
67
- def method()
68
- self.envelope
69
- end
70
- end
71
- end
@@ -1,28 +0,0 @@
1
- require 'jessica'
2
- require 'euston'
3
- require 'euston-eventstore'
4
- require 'euston-rabbitmq'
5
-
6
- require_rel '../lib'
7
-
8
- Safely::Strategy::Log.logger = EUSTON_LOG
9
- AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
10
- Euston::MessageBufferDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
11
-
12
- hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
13
-
14
- event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
15
- read_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
16
-
17
- Euston::RabbitMq.event_store_mongodb = Mongo::DB.new(hash[:event_store_database], event_connection)
18
- Euston::RabbitMq.read_model_mongodb = Mongo::DB.new(hash[:read_model_database], read_connection)
19
-
20
- Euston::EventStore::Persistence::Mongodb::Config.instance.logger = EUSTON_LOG
21
- Euston::EventStore::Persistence::Mongodb::Config.instance.database = hash[:event_store_database]
22
-
23
- Euston::RabbitMq.event_store = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
24
- Euston::RabbitMq.event_store.init
25
-
26
- Euston::Repository.event_store = Euston::EventStore::OptimisticEventStore.new Euston::RabbitMq.event_store
27
-
28
-
@@ -1,73 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class BufferComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- class << self
7
- def command_component pub_channel, sub_channel
8
- self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
9
- end
10
-
11
- def event_component pub_channel, sub_channel
12
- self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
13
- end
14
- end
15
-
16
- attr_reader :pub_thread, :sub_thread
17
- attr_writer :wait_time, :subscribe_timeout
18
-
19
- def initialize pub_buffer, sub_buffer
20
- @pub_buffer = pub_buffer
21
- @sub_buffer = sub_buffer
22
- end
23
-
24
- def wait_time
25
- @wait_time ||= 0.2
26
- end
27
-
28
- def subscribe_timeout
29
- @subscribe_timeout ||= 2000
30
- end
31
-
32
- def thread_state
33
- {:publish => @pub_thread.status, :subscribe => @sub_thread.status}
34
- end
35
-
36
- def stop
37
- @pub_thread[:stop], @sub_thread[:stop] = true, true
38
- @pub_buffer.disconnect
39
- @sub_buffer.disconnect
40
- end
41
-
42
- def start
43
- @pub_thread = Thread.new do
44
- until Thread.current[:stop] do
45
- begin
46
- @msg_dispatched_count = 0
47
- begin
48
- @msg_dispatched_count = @pub_buffer.dispatch_due_messages
49
- check_exception_and_shutdown
50
- end until Thread.current[:stop] || @msg_dispatched_count.zero?
51
- rescue => e
52
- Thread.current[:exception] = e
53
- end
54
- check_exception_and_shutdown
55
- sleep self.wait_time
56
- end
57
- end
58
-
59
- @sub_thread = Thread.new do
60
- until Thread.current[:stop] do
61
- begin
62
- @sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
63
- check_exception_and_shutdown
64
- rescue => e
65
- Thread.current[:exception] = e
66
- end
67
- check_exception_and_shutdown
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end
@@ -1,52 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class EventStoreComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_thread
7
- attr_writer :wait_time
8
-
9
- def initialize channel
10
- @channel = channel
11
- end
12
-
13
- def wait_time
14
- @wait_time ||= 0.2
15
- end
16
-
17
- def start
18
- @cli_thread = Thread.new do
19
- @event_buffer = Euston::RabbitMq::ReadModel::MessageBuffer.events
20
- @event_store = Euston::RabbitMq.event_store
21
-
22
- until Thread.current[:stop] do
23
- begin
24
- loop do
25
- commits = @event_store.get_undispatched_commits
26
- break if commits.empty?
27
- commits.each do |commit|
28
- commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
29
- @event_store.mark_commit_as_dispatched commit
30
- end
31
- end
32
- rescue => e
33
- Thread.current[:exception] = e
34
- end
35
- check_exception_and_shutdown
36
- sleep self.wait_time
37
- end
38
- end
39
- end
40
-
41
- def thread_state
42
- @cli_thread.status
43
- end
44
-
45
- def stop
46
- @cli_thread[:stop] = true
47
- @channel.disconnect
48
- end
49
-
50
- end
51
- end
52
- end
@@ -1,48 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Daemon
4
- include Euston::Daemon
5
-
6
- attr_reader :clients, :queue
7
-
8
- def initialize()
9
- @queue = Queue.new
10
-
11
- @clients = {
12
- :commands_logger => Euston::Daemons::BasicComponent.new(MessageLogger.commands_logger AMQP::Channel.new),
13
- :events_logger => Euston::Daemons::BasicComponent.new(MessageLogger.events_logger AMQP::Channel.new),
14
- :command_buffer_handler => BufferComponent.command_component(AMQP::Channel.new, AMQP::Channel.new),
15
- :event_buffer_handler => BufferComponent.event_component(AMQP::Channel.new, AMQP::Channel.new),
16
- :event_store_dispatcher => EventStoreComponent.new(AMQP::Channel.new)
17
- }
18
- end
19
-
20
- def run
21
- Thread.abort_on_exception = true
22
-
23
- @clients.each do |name, component|
24
- EUSTON_LOG.debug("Starting component: #{name}")
25
- component.daemon = self
26
- component.start
27
- sleep 0.350
28
- end
29
-
30
- @clients.each do |name, component|
31
- EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state}")
32
- end
33
-
34
- EUSTON_LOG.debug "Components started"
35
-
36
- @queue.pop #<-------- stops here until interrupted
37
-
38
- @clients.each do |name, component|
39
- EUSTON_LOG.debug "Stopping component: #{name}"
40
- component.stop
41
- end
42
-
43
- report_shutdown_reasons
44
- EUSTON_LOG.debug "Components stopped"
45
- end
46
- end
47
- end
48
- end
@@ -1,54 +0,0 @@
1
- module Euston
2
- class MessageLogger
3
- class << self
4
- def commands_logger channel
5
- self.new channel, :commands
6
- end
7
-
8
- def events_logger channel
9
- self.new channel, :events
10
- end
11
- end
12
-
13
- include Euston::RabbitMq::Exchanges
14
- include Euston::RabbitMq::Queues
15
-
16
- def initialize channel, exchange_name
17
- @channel = channel
18
- @exchange = get_exchange channel, exchange_name
19
- @read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
20
-
21
- @queue = get_queue channel, "#{exchange_name}_log"
22
- @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
23
-
24
- @queue.when(:message_decode_failed => method(:log_failure),
25
- :message_failed => method(:message_failed),
26
- :message_received => method(:write_message_to_log))
27
- end
28
-
29
- def message_failed(message, error, header)
30
- log_failure message, error
31
- header.ack
32
- end
33
-
34
- def disconnect
35
- @channel.disconnect
36
- end
37
-
38
- def start
39
- @queue.safe_subscribe
40
- end
41
-
42
- def log_failure message, error
43
- text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
44
- err = Euston::RabbitMq::MessageDecodeFailedError.new text
45
- err.set_backtrace error.backtrace
46
-
47
- Safely.report! err
48
- end
49
-
50
- def write_message_to_log message
51
- @read_model.log_new_message message
52
- end
53
- end
54
- end