euston-daemons 1.1.0-java → 1.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/euston-daemons.gemspec +27 -52
- data/lib/euston-daemons/euston/daemon.rb +7 -1
- data/lib/euston-daemons/euston/daemon_component.rb +9 -49
- data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +10 -5
- data/lib/euston-daemons/euston/exceptions.rb +9 -0
- data/lib/euston-daemons/euston/stopwatch.rb +15 -0
- data/lib/euston-daemons/pipeline/config/environment.rb +74 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
- data/lib/euston-daemons/pipeline/lib/daemon.rb +77 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
- data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
- data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
- data/lib/euston-daemons/pipeline/rake_task.rb +45 -0
- data/lib/euston-daemons/rake_task.rb +40 -19
- data/lib/euston-daemons/rake_tasks.rb +1 -3
- data/lib/euston-daemons/version.rb +1 -1
- data/lib/euston-daemons.rb +3 -1
- data/spec/daemons/{command_handler_spec.rb → command_processor_spec.rb} +3 -3
- data/spec/daemons/{event_handler_spec.rb → event_processor_spec.rb} +2 -2
- data/spec/daemons/{command_buffer_publisher_spec.rb → message_buffer_spec.rb} +19 -23
- data/spec/daemons/{snapshot_client_spec.rb → snapshotter_spec.rb} +3 -5
- data/spec/spec_helper.rb +23 -9
- data/spec/support/filters.rb +4 -5
- metadata +73 -88
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -34
- data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +0 -37
- data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +0 -35
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -22
- data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +0 -34
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -36
- data/lib/euston-daemons/euston/daemon_client.rb +0 -24
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -36
- data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +0 -42
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -24
- data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +0 -27
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -39
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -42
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +0 -15
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +0 -25
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +0 -37
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +0 -45
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +0 -21
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -17
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +0 -57
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_log.rb +0 -29
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -32
- data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +0 -43
- data/sample/Rakefile +0 -63
- data/sample/amqp_config.yml +0 -14
- data/sample/command_handlers.rb +0 -17
- data/sample/command_processor_daemon_config.yml +0 -9
- data/sample/event_handlers.rb +0 -21
- data/sample/event_processor_daemon_config.yml +0 -8
- data/sample/message_buffer_daemon_config.yml +0 -8
- data/sample/mongoid_config.yml +0 -13
- data/sample/pids/.placeholder +0 -0
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
module MongoModel
|
|
4
|
-
class MessageBuffer
|
|
5
|
-
def initialize name, mongodb
|
|
6
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
|
7
|
-
|
|
8
|
-
@name = name
|
|
9
|
-
@collection = mongodb.collection name
|
|
10
|
-
@collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
attr_reader :name
|
|
14
|
-
|
|
15
|
-
def buffer_new_message message, dispatched_at = nil
|
|
16
|
-
next_attempt = (dispatched_at || Time.now.to_f).to_f
|
|
17
|
-
|
|
18
|
-
@collection.save({ '_id' => message[:headers][:id],
|
|
19
|
-
'type' => message[:headers][:type],
|
|
20
|
-
'next_attempt' => next_attempt,
|
|
21
|
-
'next_attempt_for_humans' => Time.at(next_attempt),
|
|
22
|
-
'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def find_next_message
|
|
26
|
-
query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
|
|
27
|
-
@collection.find_one(query)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def find_due_messages
|
|
31
|
-
query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
|
|
32
|
-
order = [ 'next_attempt', Mongo::ASCENDING ]
|
|
33
|
-
|
|
34
|
-
@collection.find(query).sort(order)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def get_by_id id
|
|
38
|
-
@collection.find_one({ '_id' => id })
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def set_next_attempt message
|
|
42
|
-
next_attempt = Time.now.to_f + 10
|
|
43
|
-
|
|
44
|
-
id = { '_id' => message['_id'] }
|
|
45
|
-
doc = { '$set' => { 'next_attempt' => next_attempt,
|
|
46
|
-
'next_attempt_for_humans' => Time.at(next_attempt) } }
|
|
47
|
-
|
|
48
|
-
@collection.update id, doc, :safe => { :fsync => true }
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def remove_published_message id
|
|
52
|
-
@collection.remove({ '_id' => id }, :safe => { :fsync => true })
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
end
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
module MongoModel
|
|
4
|
-
class MessageLog
|
|
5
|
-
def initialize name, mongodb
|
|
6
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
|
7
|
-
|
|
8
|
-
@name = name
|
|
9
|
-
@collection = mongodb.collection name
|
|
10
|
-
@collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
attr_reader :name
|
|
14
|
-
|
|
15
|
-
def find_all
|
|
16
|
-
@collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def log_new_message message
|
|
20
|
-
@collection.save({ '_id' => message[:headers][:id],
|
|
21
|
-
'type' => message[:headers][:type],
|
|
22
|
-
'version' => message[:headers][:version],
|
|
23
|
-
'timestamp' => Time.now.to_f,
|
|
24
|
-
'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
class MessageBufferRakeTask < Euston::Daemons::RakeTask
|
|
3
|
-
attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
|
|
4
|
-
|
|
5
|
-
def initialize environment
|
|
6
|
-
super(environment, :message_buffer_daemon)
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def before_creating_task
|
|
10
|
-
@daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
|
|
11
|
-
@daemon_class = 'Euston::MessageBufferDaemon::Daemon'
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def initialize_settings
|
|
15
|
-
@logger.debug "AMQP config path: #{@amqp_config_path}"
|
|
16
|
-
@data[:amqp_config_path] = @amqp_config_path
|
|
17
|
-
|
|
18
|
-
@logger.debug "Daemon config path: #{@daemon_config_path}"
|
|
19
|
-
@data[:daemon_config_path] = @daemon_config_path
|
|
20
|
-
|
|
21
|
-
@logger.debug "Mongoid config path: #{@mongoid_config_path}"
|
|
22
|
-
@data[:mongoid_config_path] = @mongoid_config_path
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def load_environment
|
|
26
|
-
@logger.debug "Loading environment"
|
|
27
|
-
require 'euston-daemons/message_buffer_daemon/config/environment'
|
|
28
|
-
|
|
29
|
-
Euston::MessageBufferDaemon::DaemonEnvironment.new(@data).setup
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
end
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module SnapshotDaemon
|
|
3
|
-
class Snapshotter < Euston::DaemonClient
|
|
4
|
-
def initialize event_store, threshold, logger
|
|
5
|
-
@event_store = event_store
|
|
6
|
-
@threshold = threshold
|
|
7
|
-
@log = logger
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
private
|
|
11
|
-
|
|
12
|
-
def next_iteration
|
|
13
|
-
begin
|
|
14
|
-
stream_heads = @event_store.get_streams_to_snapshot @threshold
|
|
15
|
-
@log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})"
|
|
16
|
-
|
|
17
|
-
stream_heads.each do |stream_head|
|
|
18
|
-
pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
|
|
19
|
-
|
|
20
|
-
loader = RabbitMq::ConstantLoader.new
|
|
21
|
-
loader.when(:hit => ->(klass) { take_snapshot klass, pair },
|
|
22
|
-
:miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
|
|
23
|
-
|
|
24
|
-
loader.load pair.stream.committed_headers[:aggregate_type]
|
|
25
|
-
end
|
|
26
|
-
end until stopped || stream_heads.empty?
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def take_snapshot klass, pair
|
|
30
|
-
instance = klass.hydrate pair.stream, pair.snapshot
|
|
31
|
-
snapshot = instance.take_snapshot
|
|
32
|
-
snapshot = EventStore::Snapshot.new pair.stream.stream_id,
|
|
33
|
-
pair.stream.stream_revision,
|
|
34
|
-
snapshot[:payload],
|
|
35
|
-
:version => snapshot[:version]
|
|
36
|
-
|
|
37
|
-
@log.debug "Writing snapshot: #{snapshot.inspect}"
|
|
38
|
-
|
|
39
|
-
@event_store.add_snapshot snapshot
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
end
|
data/sample/Rakefile
DELETED
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
unless defined?(Bundler)
|
|
2
|
-
require 'rubygems'
|
|
3
|
-
|
|
4
|
-
begin
|
|
5
|
-
require 'bundler'
|
|
6
|
-
rescue => e
|
|
7
|
-
STDERR.puts e.message
|
|
8
|
-
STDERR.puts "Try running `bundle install`."
|
|
9
|
-
exit!
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
RAKE_ENV = (ENV['RAKE_ENV'] || 'development').to_sym
|
|
14
|
-
|
|
15
|
-
Bundler.require :default, RAKE_ENV
|
|
16
|
-
|
|
17
|
-
require 'euston'
|
|
18
|
-
require 'euston-daemons/rake_tasks'
|
|
19
|
-
|
|
20
|
-
sample_path = File.dirname __FILE__
|
|
21
|
-
pid_path = File.join sample_path, 'pids'
|
|
22
|
-
amqp_config_path = File.join sample_path, "amqp_config.yml"
|
|
23
|
-
mongoid_config_path = File.join sample_path, 'mongoid_config.yml'
|
|
24
|
-
|
|
25
|
-
require 'erb-yaml'
|
|
26
|
-
amqp_config = ErbYaml.read amqp_config_path, RAKE_ENV
|
|
27
|
-
|
|
28
|
-
rabbitmqadmin = RabbitMqAdminCli.new
|
|
29
|
-
rabbitmqadmin.initialize_vhost amqp_config[:vhost]
|
|
30
|
-
rabbitmqadmin.initialize_user amqp_config[:vhost], amqp_config[:user], amqp_config[:pass]
|
|
31
|
-
|
|
32
|
-
desc "Run the command processor daemon"
|
|
33
|
-
Euston::CommandProcessorRakeTask.new(RAKE_ENV) do |t|
|
|
34
|
-
require File.join sample_path, 'command_handlers'
|
|
35
|
-
|
|
36
|
-
t.amqp_config_path = amqp_config_path
|
|
37
|
-
t.command_handler_namespaces = ['Euston::Daemons::Sample::CommandHandlers']
|
|
38
|
-
t.daemon_config_path = File.join sample_path, "command_processor_daemon_config.yml"
|
|
39
|
-
t.logger = Logger.new(STDOUT)
|
|
40
|
-
t.mongoid_config_path = mongoid_config_path
|
|
41
|
-
t.pid_path = pid_path
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
desc "Run the event processor daemon"
|
|
45
|
-
Euston::EventProcessorRakeTask.new(RAKE_ENV) do |t|
|
|
46
|
-
require File.join sample_path, 'event_handlers'
|
|
47
|
-
|
|
48
|
-
t.amqp_config_path = amqp_config_path
|
|
49
|
-
t.daemon_config_path = File.join sample_path, "event_processor_daemon_config.yml"
|
|
50
|
-
t.event_handler_namespaces = ['Euston::Daemons::Sample::EventHandlers']
|
|
51
|
-
t.logger = Logger.new(STDOUT)
|
|
52
|
-
t.mongoid_config_path = mongoid_config_path
|
|
53
|
-
t.pid_path = pid_path
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
desc "Run the message buffer daemon"
|
|
57
|
-
Euston::MessageBufferRakeTask.new(RAKE_ENV) do |t|
|
|
58
|
-
t.amqp_config_path = amqp_config_path
|
|
59
|
-
t.daemon_config_path = File.join sample_path, "message_buffer_daemon_config.yml"
|
|
60
|
-
t.logger = Logger.new(STDOUT)
|
|
61
|
-
t.mongoid_config_path = mongoid_config_path
|
|
62
|
-
t.pid_path = pid_path
|
|
63
|
-
end
|
data/sample/amqp_config.yml
DELETED
data/sample/command_handlers.rb
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module Daemons
|
|
3
|
-
module Sample
|
|
4
|
-
module CommandHandlers
|
|
5
|
-
class ContactCustomer
|
|
6
|
-
include Euston::CommandHandler
|
|
7
|
-
version 1
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
class PostGoods
|
|
11
|
-
include Euston::CommandHandler
|
|
12
|
-
version 1
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
end
|
data/sample/event_handlers.rb
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module Daemons
|
|
3
|
-
module Sample
|
|
4
|
-
module EventHandlers
|
|
5
|
-
class SalesOffice
|
|
6
|
-
include Euston::EventHandler
|
|
7
|
-
|
|
8
|
-
consumes :customer_contacted, 1 do |headers, event|
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
class Warehouse
|
|
13
|
-
include Euston::EventHandler
|
|
14
|
-
|
|
15
|
-
consumes :goods_posted, 1 do |headers, event|
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
data/sample/mongoid_config.yml
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
defaults: &defaults
|
|
2
|
-
host: localhost
|
|
3
|
-
port: 27017
|
|
4
|
-
safe: true
|
|
5
|
-
use_utc: true
|
|
6
|
-
event_store_database: euston-daemons-sample-event-store
|
|
7
|
-
read_model_database: euston-daemons-sample-read-model
|
|
8
|
-
|
|
9
|
-
development:
|
|
10
|
-
<<: *defaults
|
|
11
|
-
|
|
12
|
-
test:
|
|
13
|
-
<<: *defaults
|
data/sample/pids/.placeholder
DELETED
|
File without changes
|