euston-daemons 1.1.0-java → 1.2.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. data/Gemfile +1 -0
  2. data/euston-daemons.gemspec +27 -52
  3. data/lib/euston-daemons/euston/daemon.rb +7 -1
  4. data/lib/euston-daemons/euston/daemon_component.rb +9 -49
  5. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  6. data/lib/euston-daemons/euston/daemon_environment.rb +10 -5
  7. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  8. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  9. data/lib/euston-daemons/pipeline/config/environment.rb +74 -0
  10. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  11. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  12. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  13. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  15. data/lib/euston-daemons/pipeline/lib/daemon.rb +77 -0
  16. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  17. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  18. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  19. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  20. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  21. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  22. data/lib/euston-daemons/pipeline/rake_task.rb +45 -0
  23. data/lib/euston-daemons/rake_task.rb +40 -19
  24. data/lib/euston-daemons/rake_tasks.rb +1 -3
  25. data/lib/euston-daemons/version.rb +1 -1
  26. data/lib/euston-daemons.rb +3 -1
  27. data/spec/daemons/{command_handler_spec.rb → command_processor_spec.rb} +3 -3
  28. data/spec/daemons/{event_handler_spec.rb → event_processor_spec.rb} +2 -2
  29. data/spec/daemons/{command_buffer_publisher_spec.rb → message_buffer_spec.rb} +19 -23
  30. data/spec/daemons/{snapshot_client_spec.rb → snapshotter_spec.rb} +3 -5
  31. data/spec/spec_helper.rb +23 -9
  32. data/spec/support/filters.rb +4 -5
  33. metadata +73 -88
  34. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -34
  35. data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +0 -37
  36. data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +0 -35
  37. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -22
  38. data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +0 -34
  39. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -36
  40. data/lib/euston-daemons/euston/daemon_client.rb +0 -24
  41. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -36
  42. data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +0 -42
  43. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -24
  44. data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +0 -27
  45. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -39
  46. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -42
  47. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +0 -9
  48. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +0 -9
  49. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +0 -9
  50. data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +0 -15
  51. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +0 -9
  52. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +0 -9
  53. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +0 -9
  54. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +0 -25
  55. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +0 -52
  56. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +0 -37
  57. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +0 -45
  58. data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +0 -21
  59. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -17
  60. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +0 -11
  61. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +0 -11
  62. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +0 -11
  63. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +0 -11
  64. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +0 -57
  65. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_log.rb +0 -29
  66. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -32
  67. data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +0 -43
  68. data/sample/Rakefile +0 -63
  69. data/sample/amqp_config.yml +0 -14
  70. data/sample/command_handlers.rb +0 -17
  71. data/sample/command_processor_daemon_config.yml +0 -9
  72. data/sample/event_handlers.rb +0 -21
  73. data/sample/event_processor_daemon_config.yml +0 -8
  74. data/sample/message_buffer_daemon_config.yml +0 -8
  75. data/sample/mongoid_config.yml +0 -13
  76. data/sample/pids/.placeholder +0 -0
@@ -1,57 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module MongoModel
4
- class MessageBuffer
5
- def initialize name, mongodb
6
- mongodb.create_collection name unless mongodb.collection_names.include? name
7
-
8
- @name = name
9
- @collection = mongodb.collection name
10
- @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
11
- end
12
-
13
- attr_reader :name
14
-
15
- def buffer_new_message message, dispatched_at = nil
16
- next_attempt = (dispatched_at || Time.now.to_f).to_f
17
-
18
- @collection.save({ '_id' => message[:headers][:id],
19
- 'type' => message[:headers][:type],
20
- 'next_attempt' => next_attempt,
21
- 'next_attempt_for_humans' => Time.at(next_attempt),
22
- 'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
23
- end
24
-
25
- def find_next_message
26
- query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
27
- @collection.find_one(query)
28
- end
29
-
30
- def find_due_messages
31
- query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
32
- order = [ 'next_attempt', Mongo::ASCENDING ]
33
-
34
- @collection.find(query).sort(order)
35
- end
36
-
37
- def get_by_id id
38
- @collection.find_one({ '_id' => id })
39
- end
40
-
41
- def set_next_attempt message
42
- next_attempt = Time.now.to_f + 10
43
-
44
- id = { '_id' => message['_id'] }
45
- doc = { '$set' => { 'next_attempt' => next_attempt,
46
- 'next_attempt_for_humans' => Time.at(next_attempt) } }
47
-
48
- @collection.update id, doc, :safe => { :fsync => true }
49
- end
50
-
51
- def remove_published_message id
52
- @collection.remove({ '_id' => id }, :safe => { :fsync => true })
53
- end
54
- end
55
- end
56
- end
57
- end
@@ -1,29 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module MongoModel
4
- class MessageLog
5
- def initialize name, mongodb
6
- mongodb.create_collection name unless mongodb.collection_names.include? name
7
-
8
- @name = name
9
- @collection = mongodb.collection name
10
- @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
11
- end
12
-
13
- attr_reader :name
14
-
15
- def find_all
16
- @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
17
- end
18
-
19
- def log_new_message message
20
- @collection.save({ '_id' => message[:headers][:id],
21
- 'type' => message[:headers][:type],
22
- 'version' => message[:headers][:version],
23
- 'timestamp' => Time.now.to_f,
24
- 'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
25
- end
26
- end
27
- end
28
- end
29
- end
@@ -1,32 +0,0 @@
1
- module Euston
2
- class MessageBufferRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize environment
6
- super(environment, :message_buffer_daemon)
7
- end
8
-
9
- def before_creating_task
10
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
11
- @daemon_class = 'Euston::MessageBufferDaemon::Daemon'
12
- end
13
-
14
- def initialize_settings
15
- @logger.debug "AMQP config path: #{@amqp_config_path}"
16
- @data[:amqp_config_path] = @amqp_config_path
17
-
18
- @logger.debug "Daemon config path: #{@daemon_config_path}"
19
- @data[:daemon_config_path] = @daemon_config_path
20
-
21
- @logger.debug "Mongoid config path: #{@mongoid_config_path}"
22
- @data[:mongoid_config_path] = @mongoid_config_path
23
- end
24
-
25
- def load_environment
26
- @logger.debug "Loading environment"
27
- require 'euston-daemons/message_buffer_daemon/config/environment'
28
-
29
- Euston::MessageBufferDaemon::DaemonEnvironment.new(@data).setup
30
- end
31
- end
32
- end
@@ -1,43 +0,0 @@
1
- module Euston
2
- module SnapshotDaemon
3
- class Snapshotter < Euston::DaemonClient
4
- def initialize event_store, threshold, logger
5
- @event_store = event_store
6
- @threshold = threshold
7
- @log = logger
8
- end
9
-
10
- private
11
-
12
- def next_iteration
13
- begin
14
- stream_heads = @event_store.get_streams_to_snapshot @threshold
15
- @log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})"
16
-
17
- stream_heads.each do |stream_head|
18
- pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
19
-
20
- loader = RabbitMq::ConstantLoader.new
21
- loader.when(:hit => ->(klass) { take_snapshot klass, pair },
22
- :miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
23
-
24
- loader.load pair.stream.committed_headers[:aggregate_type]
25
- end
26
- end until stopped || stream_heads.empty?
27
- end
28
-
29
- def take_snapshot klass, pair
30
- instance = klass.hydrate pair.stream, pair.snapshot
31
- snapshot = instance.take_snapshot
32
- snapshot = EventStore::Snapshot.new pair.stream.stream_id,
33
- pair.stream.stream_revision,
34
- snapshot[:payload],
35
- :version => snapshot[:version]
36
-
37
- @log.debug "Writing snapshot: #{snapshot.inspect}"
38
-
39
- @event_store.add_snapshot snapshot
40
- end
41
- end
42
- end
43
- end
data/sample/Rakefile DELETED
@@ -1,63 +0,0 @@
1
- unless defined?(Bundler)
2
- require 'rubygems'
3
-
4
- begin
5
- require 'bundler'
6
- rescue => e
7
- STDERR.puts e.message
8
- STDERR.puts "Try running `bundle install`."
9
- exit!
10
- end
11
- end
12
-
13
- RAKE_ENV = (ENV['RAKE_ENV'] || 'development').to_sym
14
-
15
- Bundler.require :default, RAKE_ENV
16
-
17
- require 'euston'
18
- require 'euston-daemons/rake_tasks'
19
-
20
- sample_path = File.dirname __FILE__
21
- pid_path = File.join sample_path, 'pids'
22
- amqp_config_path = File.join sample_path, "amqp_config.yml"
23
- mongoid_config_path = File.join sample_path, 'mongoid_config.yml'
24
-
25
- require 'erb-yaml'
26
- amqp_config = ErbYaml.read amqp_config_path, RAKE_ENV
27
-
28
- rabbitmqadmin = RabbitMqAdminCli.new
29
- rabbitmqadmin.initialize_vhost amqp_config[:vhost]
30
- rabbitmqadmin.initialize_user amqp_config[:vhost], amqp_config[:user], amqp_config[:pass]
31
-
32
- desc "Run the command processor daemon"
33
- Euston::CommandProcessorRakeTask.new(RAKE_ENV) do |t|
34
- require File.join sample_path, 'command_handlers'
35
-
36
- t.amqp_config_path = amqp_config_path
37
- t.command_handler_namespaces = ['Euston::Daemons::Sample::CommandHandlers']
38
- t.daemon_config_path = File.join sample_path, "command_processor_daemon_config.yml"
39
- t.logger = Logger.new(STDOUT)
40
- t.mongoid_config_path = mongoid_config_path
41
- t.pid_path = pid_path
42
- end
43
-
44
- desc "Run the event processor daemon"
45
- Euston::EventProcessorRakeTask.new(RAKE_ENV) do |t|
46
- require File.join sample_path, 'event_handlers'
47
-
48
- t.amqp_config_path = amqp_config_path
49
- t.daemon_config_path = File.join sample_path, "event_processor_daemon_config.yml"
50
- t.event_handler_namespaces = ['Euston::Daemons::Sample::EventHandlers']
51
- t.logger = Logger.new(STDOUT)
52
- t.mongoid_config_path = mongoid_config_path
53
- t.pid_path = pid_path
54
- end
55
-
56
- desc "Run the message buffer daemon"
57
- Euston::MessageBufferRakeTask.new(RAKE_ENV) do |t|
58
- t.amqp_config_path = amqp_config_path
59
- t.daemon_config_path = File.join sample_path, "message_buffer_daemon_config.yml"
60
- t.logger = Logger.new(STDOUT)
61
- t.mongoid_config_path = mongoid_config_path
62
- t.pid_path = pid_path
63
- end
@@ -1,14 +0,0 @@
1
- defaults: &defaults
2
- host: localhost
3
- vhost: euston-daemons-sample
4
- user: euston-daemons-sample-user
5
- pass: password
6
-
7
- development:
8
- <<: *defaults
9
-
10
- test:
11
- <<: *defaults
12
-
13
- production:
14
- <<: *defaults
@@ -1,17 +0,0 @@
1
- module Euston
2
- module Daemons
3
- module Sample
4
- module CommandHandlers
5
- class ContactCustomer
6
- include Euston::CommandHandler
7
- version 1
8
- end
9
-
10
- class PostGoods
11
- include Euston::CommandHandler
12
- version 1
13
- end
14
- end
15
- end
16
- end
17
- end
@@ -1,9 +0,0 @@
1
- defaults: &defaults
2
- client_instances: 3
3
- mongo_db_name: euston-daemons-sample-event-store
4
-
5
- development:
6
- <<: *defaults
7
-
8
- test:
9
- <<: *defaults
@@ -1,21 +0,0 @@
1
- module Euston
2
- module Daemons
3
- module Sample
4
- module EventHandlers
5
- class SalesOffice
6
- include Euston::EventHandler
7
-
8
- consumes :customer_contacted, 1 do |headers, event|
9
- end
10
- end
11
-
12
- class Warehouse
13
- include Euston::EventHandler
14
-
15
- consumes :goods_posted, 1 do |headers, event|
16
- end
17
- end
18
- end
19
- end
20
- end
21
- end
@@ -1,8 +0,0 @@
1
- defaults: &defaults
2
- mongo_db_name: euston-daemons-sample-read-model
3
-
4
- development:
5
- <<: *defaults
6
-
7
- test:
8
- <<: *defaults
@@ -1,8 +0,0 @@
1
- defaults: &defaults
2
- debug: false
3
-
4
- development:
5
- <<: *defaults
6
-
7
- test:
8
- <<: *defaults
@@ -1,13 +0,0 @@
1
- defaults: &defaults
2
- host: localhost
3
- port: 27017
4
- safe: true
5
- use_utc: true
6
- event_store_database: euston-daemons-sample-event-store
7
- read_model_database: euston-daemons-sample-read-model
8
-
9
- development:
10
- <<: *defaults
11
-
12
- test:
13
- <<: *defaults
File without changes