euston-daemons 1.0.5 → 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,56 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Publisher
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
-
16
- def initialize channel, exchange_name
17
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
18
-
19
- @channel = channel
20
- @exchange = get_exchange @channel, exchange_name
21
- end
22
-
23
- def disconnect
24
- @channel.disconnect
25
- end
26
-
27
- def dispatch_one_due_message
28
- if (message = @read_model.find_next_message)
29
- @read_model.set_next_attempt message
30
- protected_publish(message)
31
- end
32
- end
33
-
34
- def dispatch_due_messages
35
- dispatched = 0
36
- @read_model.find_due_messages.each do |message|
37
- @read_model.set_next_attempt message
38
- break unless protected_publish(message)
39
- dispatched += 1
40
- end
41
- dispatched
42
- end
43
-
44
- def protected_publish(message)
45
- ret = true
46
- begin
47
- @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
48
- rescue NativeException => e
49
- Thread.current[:exception] = e
50
- ret = false
51
- end
52
- ret
53
- end
54
- end
55
- end
56
- end
@@ -1,36 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module ReadModel
4
- class MessageLog
5
- class << self
6
- def commands
7
- self.new "command_log", Euston::RabbitMq.event_store_mongodb
8
- end
9
-
10
- def events
11
- self.new "event_log", Euston::RabbitMq.event_store_mongodb
12
- end
13
- end
14
-
15
- def initialize name, mongodb
16
- mongodb.create_collection name unless mongodb.collection_names.include? name
17
-
18
- @collection = mongodb.collection name
19
- @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
20
- end
21
-
22
- def find_all
23
- @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
24
- end
25
-
26
- def log_new_message message
27
- @collection.save({ '_id' => message[:headers][:id],
28
- 'type' => message[:headers][:type],
29
- 'version' => message[:headers][:version],
30
- 'timestamp' => Time.now.to_f,
31
- 'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
32
- end
33
- end
34
- end
35
- end
36
- end
@@ -1,14 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.debug
10
- @config[:debug]
11
- end
12
- end
13
- end
14
- end
@@ -1,60 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Subscriber
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
- include Euston::RabbitMq::Queues
16
- include Hollywood
17
-
18
- def initialize channel, exchange_name
19
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
20
-
21
- @channel = channel
22
- @channel.prefetch(1)
23
- @exchange = get_exchange @channel, exchange_name
24
- @queue = get_queue @channel, "#{exchange_name}_buffer"
25
- @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
26
-
27
- @queue.when(:message_decode_failed => method(:log_failure),
28
- :message_failed => method(:message_failed),
29
- :message_received => method(:remove_message_from_buffer))
30
-
31
- @consumer = @queue.consumer
32
- end
33
-
34
- def disconnect
35
- @channel.disconnect
36
- end
37
-
38
- def dequeue_buffered_messages(timeout) #will block until message or timeout
39
- @queue.safe_subscribe_with_timeout(@consumer, timeout)
40
- end
41
-
42
- def log_failure message, error
43
- text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
44
- err = Euston::RabbitMq::MessageDecodeFailedError.new text
45
- err.set_backtrace error.backtrace
46
-
47
- Safely.report! err
48
- end
49
-
50
- def message_failed(message, error, header)
51
- header.ack!
52
- log_failure message, error
53
- end
54
-
55
- def remove_message_from_buffer message
56
- @read_model.remove_published_message message[:headers][:id]
57
- end
58
- end
59
- end
60
- end
@@ -1,30 +0,0 @@
1
- module Euston
2
- class MessageBufferRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize
6
- super(:message_buffer_daemon)
7
- end
8
-
9
- def before_creating_task
10
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
11
- @daemon_class = 'Euston::MessageBufferDaemon::Daemon'
12
- end
13
-
14
- def initialize_paths
15
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
16
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
17
-
18
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
19
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
20
-
21
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
22
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
23
- end
24
-
25
- def load_environment
26
- EUSTON_LOG.debug "Loading environment"
27
- require_rel 'config/environment.rb'
28
- end
29
- end
30
- end