euston-daemons 1.0.5 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,56 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Publisher
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
-
16
- def initialize channel, exchange_name
17
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
18
-
19
- @channel = channel
20
- @exchange = get_exchange @channel, exchange_name
21
- end
22
-
23
- def disconnect
24
- @channel.disconnect
25
- end
26
-
27
- def dispatch_one_due_message
28
- if (message = @read_model.find_next_message)
29
- @read_model.set_next_attempt message
30
- protected_publish(message)
31
- end
32
- end
33
-
34
- def dispatch_due_messages
35
- dispatched = 0
36
- @read_model.find_due_messages.each do |message|
37
- @read_model.set_next_attempt message
38
- break unless protected_publish(message)
39
- dispatched += 1
40
- end
41
- dispatched
42
- end
43
-
44
- def protected_publish(message)
45
- ret = true
46
- begin
47
- @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
48
- rescue NativeException => e
49
- Thread.current[:exception] = e
50
- ret = false
51
- end
52
- ret
53
- end
54
- end
55
- end
56
- end
@@ -1,36 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module ReadModel
4
- class MessageLog
5
- class << self
6
- def commands
7
- self.new "command_log", Euston::RabbitMq.event_store_mongodb
8
- end
9
-
10
- def events
11
- self.new "event_log", Euston::RabbitMq.event_store_mongodb
12
- end
13
- end
14
-
15
- def initialize name, mongodb
16
- mongodb.create_collection name unless mongodb.collection_names.include? name
17
-
18
- @collection = mongodb.collection name
19
- @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
20
- end
21
-
22
- def find_all
23
- @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
24
- end
25
-
26
- def log_new_message message
27
- @collection.save({ '_id' => message[:headers][:id],
28
- 'type' => message[:headers][:type],
29
- 'version' => message[:headers][:version],
30
- 'timestamp' => Time.now.to_f,
31
- 'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
32
- end
33
- end
34
- end
35
- end
36
- end
@@ -1,14 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.debug
10
- @config[:debug]
11
- end
12
- end
13
- end
14
- end
@@ -1,60 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Subscriber
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
- include Euston::RabbitMq::Queues
16
- include Hollywood
17
-
18
- def initialize channel, exchange_name
19
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
20
-
21
- @channel = channel
22
- @channel.prefetch(1)
23
- @exchange = get_exchange @channel, exchange_name
24
- @queue = get_queue @channel, "#{exchange_name}_buffer"
25
- @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
26
-
27
- @queue.when(:message_decode_failed => method(:log_failure),
28
- :message_failed => method(:message_failed),
29
- :message_received => method(:remove_message_from_buffer))
30
-
31
- @consumer = @queue.consumer
32
- end
33
-
34
- def disconnect
35
- @channel.disconnect
36
- end
37
-
38
- def dequeue_buffered_messages(timeout) #will block until message or timeout
39
- @queue.safe_subscribe_with_timeout(@consumer, timeout)
40
- end
41
-
42
- def log_failure message, error
43
- text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
44
- err = Euston::RabbitMq::MessageDecodeFailedError.new text
45
- err.set_backtrace error.backtrace
46
-
47
- Safely.report! err
48
- end
49
-
50
- def message_failed(message, error, header)
51
- header.ack!
52
- log_failure message, error
53
- end
54
-
55
- def remove_message_from_buffer message
56
- @read_model.remove_published_message message[:headers][:id]
57
- end
58
- end
59
- end
60
- end
@@ -1,30 +0,0 @@
1
- module Euston
2
- class MessageBufferRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize
6
- super(:message_buffer_daemon)
7
- end
8
-
9
- def before_creating_task
10
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
11
- @daemon_class = 'Euston::MessageBufferDaemon::Daemon'
12
- end
13
-
14
- def initialize_paths
15
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
16
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
17
-
18
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
19
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
20
-
21
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
22
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
23
- end
24
-
25
- def load_environment
26
- EUSTON_LOG.debug "Loading environment"
27
- require_rel 'config/environment.rb'
28
- end
29
- end
30
- end