euston-daemons 1.1.0-java → 1.2.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. data/Gemfile +1 -0
  2. data/euston-daemons.gemspec +27 -52
  3. data/lib/euston-daemons/euston/daemon.rb +7 -1
  4. data/lib/euston-daemons/euston/daemon_component.rb +9 -49
  5. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  6. data/lib/euston-daemons/euston/daemon_environment.rb +10 -5
  7. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  8. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  9. data/lib/euston-daemons/pipeline/config/environment.rb +74 -0
  10. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  11. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  12. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  13. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  15. data/lib/euston-daemons/pipeline/lib/daemon.rb +77 -0
  16. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  17. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  18. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  19. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  20. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  21. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  22. data/lib/euston-daemons/pipeline/rake_task.rb +45 -0
  23. data/lib/euston-daemons/rake_task.rb +40 -19
  24. data/lib/euston-daemons/rake_tasks.rb +1 -3
  25. data/lib/euston-daemons/version.rb +1 -1
  26. data/lib/euston-daemons.rb +3 -1
  27. data/spec/daemons/{command_handler_spec.rb → command_processor_spec.rb} +3 -3
  28. data/spec/daemons/{event_handler_spec.rb → event_processor_spec.rb} +2 -2
  29. data/spec/daemons/{command_buffer_publisher_spec.rb → message_buffer_spec.rb} +19 -23
  30. data/spec/daemons/{snapshot_client_spec.rb → snapshotter_spec.rb} +3 -5
  31. data/spec/spec_helper.rb +23 -9
  32. data/spec/support/filters.rb +4 -5
  33. metadata +73 -88
  34. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -34
  35. data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +0 -37
  36. data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +0 -35
  37. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -22
  38. data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +0 -34
  39. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -36
  40. data/lib/euston-daemons/euston/daemon_client.rb +0 -24
  41. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -36
  42. data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +0 -42
  43. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -24
  44. data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +0 -27
  45. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -39
  46. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -42
  47. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +0 -9
  48. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +0 -9
  49. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +0 -9
  50. data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +0 -15
  51. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +0 -9
  52. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +0 -9
  53. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +0 -9
  54. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +0 -25
  55. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +0 -52
  56. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +0 -37
  57. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +0 -45
  58. data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +0 -21
  59. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -17
  60. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +0 -11
  61. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +0 -11
  62. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +0 -11
  63. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +0 -11
  64. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +0 -57
  65. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_log.rb +0 -29
  66. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -32
  67. data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +0 -43
  68. data/sample/Rakefile +0 -63
  69. data/sample/amqp_config.yml +0 -14
  70. data/sample/command_handlers.rb +0 -17
  71. data/sample/command_processor_daemon_config.yml +0 -9
  72. data/sample/event_handlers.rb +0 -21
  73. data/sample/event_processor_daemon_config.yml +0 -8
  74. data/sample/message_buffer_daemon_config.yml +0 -8
  75. data/sample/mongoid_config.yml +0 -13
  76. data/sample/pids/.placeholder +0 -0
@@ -1,57 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module MongoModel
4
- class MessageBuffer
5
- def initialize name, mongodb
6
- mongodb.create_collection name unless mongodb.collection_names.include? name
7
-
8
- @name = name
9
- @collection = mongodb.collection name
10
- @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
11
- end
12
-
13
- attr_reader :name
14
-
15
- def buffer_new_message message, dispatched_at = nil
16
- next_attempt = (dispatched_at || Time.now.to_f).to_f
17
-
18
- @collection.save({ '_id' => message[:headers][:id],
19
- 'type' => message[:headers][:type],
20
- 'next_attempt' => next_attempt,
21
- 'next_attempt_for_humans' => Time.at(next_attempt),
22
- 'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
23
- end
24
-
25
- def find_next_message
26
- query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
27
- @collection.find_one(query)
28
- end
29
-
30
- def find_due_messages
31
- query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
32
- order = [ 'next_attempt', Mongo::ASCENDING ]
33
-
34
- @collection.find(query).sort(order)
35
- end
36
-
37
- def get_by_id id
38
- @collection.find_one({ '_id' => id })
39
- end
40
-
41
- def set_next_attempt message
42
- next_attempt = Time.now.to_f + 10
43
-
44
- id = { '_id' => message['_id'] }
45
- doc = { '$set' => { 'next_attempt' => next_attempt,
46
- 'next_attempt_for_humans' => Time.at(next_attempt) } }
47
-
48
- @collection.update id, doc, :safe => { :fsync => true }
49
- end
50
-
51
- def remove_published_message id
52
- @collection.remove({ '_id' => id }, :safe => { :fsync => true })
53
- end
54
- end
55
- end
56
- end
57
- end
@@ -1,29 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module MongoModel
4
- class MessageLog
5
- def initialize name, mongodb
6
- mongodb.create_collection name unless mongodb.collection_names.include? name
7
-
8
- @name = name
9
- @collection = mongodb.collection name
10
- @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
11
- end
12
-
13
- attr_reader :name
14
-
15
- def find_all
16
- @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
17
- end
18
-
19
- def log_new_message message
20
- @collection.save({ '_id' => message[:headers][:id],
21
- 'type' => message[:headers][:type],
22
- 'version' => message[:headers][:version],
23
- 'timestamp' => Time.now.to_f,
24
- 'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
25
- end
26
- end
27
- end
28
- end
29
- end
@@ -1,32 +0,0 @@
1
- module Euston
2
- class MessageBufferRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize environment
6
- super(environment, :message_buffer_daemon)
7
- end
8
-
9
- def before_creating_task
10
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
11
- @daemon_class = 'Euston::MessageBufferDaemon::Daemon'
12
- end
13
-
14
- def initialize_settings
15
- @logger.debug "AMQP config path: #{@amqp_config_path}"
16
- @data[:amqp_config_path] = @amqp_config_path
17
-
18
- @logger.debug "Daemon config path: #{@daemon_config_path}"
19
- @data[:daemon_config_path] = @daemon_config_path
20
-
21
- @logger.debug "Mongoid config path: #{@mongoid_config_path}"
22
- @data[:mongoid_config_path] = @mongoid_config_path
23
- end
24
-
25
- def load_environment
26
- @logger.debug "Loading environment"
27
- require 'euston-daemons/message_buffer_daemon/config/environment'
28
-
29
- Euston::MessageBufferDaemon::DaemonEnvironment.new(@data).setup
30
- end
31
- end
32
- end
@@ -1,43 +0,0 @@
1
- module Euston
2
- module SnapshotDaemon
3
- class Snapshotter < Euston::DaemonClient
4
- def initialize event_store, threshold, logger
5
- @event_store = event_store
6
- @threshold = threshold
7
- @log = logger
8
- end
9
-
10
- private
11
-
12
- def next_iteration
13
- begin
14
- stream_heads = @event_store.get_streams_to_snapshot @threshold
15
- @log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})"
16
-
17
- stream_heads.each do |stream_head|
18
- pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
19
-
20
- loader = RabbitMq::ConstantLoader.new
21
- loader.when(:hit => ->(klass) { take_snapshot klass, pair },
22
- :miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
23
-
24
- loader.load pair.stream.committed_headers[:aggregate_type]
25
- end
26
- end until stopped || stream_heads.empty?
27
- end
28
-
29
- def take_snapshot klass, pair
30
- instance = klass.hydrate pair.stream, pair.snapshot
31
- snapshot = instance.take_snapshot
32
- snapshot = EventStore::Snapshot.new pair.stream.stream_id,
33
- pair.stream.stream_revision,
34
- snapshot[:payload],
35
- :version => snapshot[:version]
36
-
37
- @log.debug "Writing snapshot: #{snapshot.inspect}"
38
-
39
- @event_store.add_snapshot snapshot
40
- end
41
- end
42
- end
43
- end
data/sample/Rakefile DELETED
@@ -1,63 +0,0 @@
1
- unless defined?(Bundler)
2
- require 'rubygems'
3
-
4
- begin
5
- require 'bundler'
6
- rescue => e
7
- STDERR.puts e.message
8
- STDERR.puts "Try running `bundle install`."
9
- exit!
10
- end
11
- end
12
-
13
- RAKE_ENV = (ENV['RAKE_ENV'] || 'development').to_sym
14
-
15
- Bundler.require :default, RAKE_ENV
16
-
17
- require 'euston'
18
- require 'euston-daemons/rake_tasks'
19
-
20
- sample_path = File.dirname __FILE__
21
- pid_path = File.join sample_path, 'pids'
22
- amqp_config_path = File.join sample_path, "amqp_config.yml"
23
- mongoid_config_path = File.join sample_path, 'mongoid_config.yml'
24
-
25
- require 'erb-yaml'
26
- amqp_config = ErbYaml.read amqp_config_path, RAKE_ENV
27
-
28
- rabbitmqadmin = RabbitMqAdminCli.new
29
- rabbitmqadmin.initialize_vhost amqp_config[:vhost]
30
- rabbitmqadmin.initialize_user amqp_config[:vhost], amqp_config[:user], amqp_config[:pass]
31
-
32
- desc "Run the command processor daemon"
33
- Euston::CommandProcessorRakeTask.new(RAKE_ENV) do |t|
34
- require File.join sample_path, 'command_handlers'
35
-
36
- t.amqp_config_path = amqp_config_path
37
- t.command_handler_namespaces = ['Euston::Daemons::Sample::CommandHandlers']
38
- t.daemon_config_path = File.join sample_path, "command_processor_daemon_config.yml"
39
- t.logger = Logger.new(STDOUT)
40
- t.mongoid_config_path = mongoid_config_path
41
- t.pid_path = pid_path
42
- end
43
-
44
- desc "Run the event processor daemon"
45
- Euston::EventProcessorRakeTask.new(RAKE_ENV) do |t|
46
- require File.join sample_path, 'event_handlers'
47
-
48
- t.amqp_config_path = amqp_config_path
49
- t.daemon_config_path = File.join sample_path, "event_processor_daemon_config.yml"
50
- t.event_handler_namespaces = ['Euston::Daemons::Sample::EventHandlers']
51
- t.logger = Logger.new(STDOUT)
52
- t.mongoid_config_path = mongoid_config_path
53
- t.pid_path = pid_path
54
- end
55
-
56
- desc "Run the message buffer daemon"
57
- Euston::MessageBufferRakeTask.new(RAKE_ENV) do |t|
58
- t.amqp_config_path = amqp_config_path
59
- t.daemon_config_path = File.join sample_path, "message_buffer_daemon_config.yml"
60
- t.logger = Logger.new(STDOUT)
61
- t.mongoid_config_path = mongoid_config_path
62
- t.pid_path = pid_path
63
- end
@@ -1,14 +0,0 @@
1
- defaults: &defaults
2
- host: localhost
3
- vhost: euston-daemons-sample
4
- user: euston-daemons-sample-user
5
- pass: password
6
-
7
- development:
8
- <<: *defaults
9
-
10
- test:
11
- <<: *defaults
12
-
13
- production:
14
- <<: *defaults
@@ -1,17 +0,0 @@
1
- module Euston
2
- module Daemons
3
- module Sample
4
- module CommandHandlers
5
- class ContactCustomer
6
- include Euston::CommandHandler
7
- version 1
8
- end
9
-
10
- class PostGoods
11
- include Euston::CommandHandler
12
- version 1
13
- end
14
- end
15
- end
16
- end
17
- end
@@ -1,9 +0,0 @@
1
- defaults: &defaults
2
- client_instances: 3
3
- mongo_db_name: euston-daemons-sample-event-store
4
-
5
- development:
6
- <<: *defaults
7
-
8
- test:
9
- <<: *defaults
@@ -1,21 +0,0 @@
1
- module Euston
2
- module Daemons
3
- module Sample
4
- module EventHandlers
5
- class SalesOffice
6
- include Euston::EventHandler
7
-
8
- consumes :customer_contacted, 1 do |headers, event|
9
- end
10
- end
11
-
12
- class Warehouse
13
- include Euston::EventHandler
14
-
15
- consumes :goods_posted, 1 do |headers, event|
16
- end
17
- end
18
- end
19
- end
20
- end
21
- end
@@ -1,8 +0,0 @@
1
- defaults: &defaults
2
- mongo_db_name: euston-daemons-sample-read-model
3
-
4
- development:
5
- <<: *defaults
6
-
7
- test:
8
- <<: *defaults
@@ -1,8 +0,0 @@
1
- defaults: &defaults
2
- debug: false
3
-
4
- development:
5
- <<: *defaults
6
-
7
- test:
8
- <<: *defaults
@@ -1,13 +0,0 @@
1
- defaults: &defaults
2
- host: localhost
3
- port: 27017
4
- safe: true
5
- use_utc: true
6
- event_store_database: euston-daemons-sample-event-store
7
- read_model_database: euston-daemons-sample-read-model
8
-
9
- development:
10
- <<: *defaults
11
-
12
- test:
13
- <<: *defaults
File without changes