euston-daemons 1.1.0-java → 1.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/euston-daemons.gemspec +27 -52
- data/lib/euston-daemons/euston/daemon.rb +7 -1
- data/lib/euston-daemons/euston/daemon_component.rb +9 -49
- data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +10 -5
- data/lib/euston-daemons/euston/exceptions.rb +9 -0
- data/lib/euston-daemons/euston/stopwatch.rb +15 -0
- data/lib/euston-daemons/pipeline/config/environment.rb +74 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
- data/lib/euston-daemons/pipeline/lib/daemon.rb +77 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
- data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
- data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
- data/lib/euston-daemons/pipeline/rake_task.rb +45 -0
- data/lib/euston-daemons/rake_task.rb +40 -19
- data/lib/euston-daemons/rake_tasks.rb +1 -3
- data/lib/euston-daemons/version.rb +1 -1
- data/lib/euston-daemons.rb +3 -1
- data/spec/daemons/{command_handler_spec.rb → command_processor_spec.rb} +3 -3
- data/spec/daemons/{event_handler_spec.rb → event_processor_spec.rb} +2 -2
- data/spec/daemons/{command_buffer_publisher_spec.rb → message_buffer_spec.rb} +19 -23
- data/spec/daemons/{snapshot_client_spec.rb → snapshotter_spec.rb} +3 -5
- data/spec/spec_helper.rb +23 -9
- data/spec/support/filters.rb +4 -5
- metadata +73 -88
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -34
- data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +0 -37
- data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +0 -35
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -22
- data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +0 -34
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -36
- data/lib/euston-daemons/euston/daemon_client.rb +0 -24
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -36
- data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +0 -42
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -24
- data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +0 -27
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -39
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -42
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +0 -15
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +0 -9
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +0 -25
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +0 -37
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +0 -45
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +0 -21
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -17
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +0 -11
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +0 -57
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_log.rb +0 -29
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -32
- data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +0 -43
- data/sample/Rakefile +0 -63
- data/sample/amqp_config.yml +0 -14
- data/sample/command_handlers.rb +0 -17
- data/sample/command_processor_daemon_config.yml +0 -9
- data/sample/event_handlers.rb +0 -21
- data/sample/event_processor_daemon_config.yml +0 -8
- data/sample/message_buffer_daemon_config.yml +0 -8
- data/sample/mongoid_config.yml +0 -13
- data/sample/pids/.placeholder +0 -0
@@ -1,57 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
module MongoModel
|
4
|
-
class MessageBuffer
|
5
|
-
def initialize name, mongodb
|
6
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
7
|
-
|
8
|
-
@name = name
|
9
|
-
@collection = mongodb.collection name
|
10
|
-
@collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
|
11
|
-
end
|
12
|
-
|
13
|
-
attr_reader :name
|
14
|
-
|
15
|
-
def buffer_new_message message, dispatched_at = nil
|
16
|
-
next_attempt = (dispatched_at || Time.now.to_f).to_f
|
17
|
-
|
18
|
-
@collection.save({ '_id' => message[:headers][:id],
|
19
|
-
'type' => message[:headers][:type],
|
20
|
-
'next_attempt' => next_attempt,
|
21
|
-
'next_attempt_for_humans' => Time.at(next_attempt),
|
22
|
-
'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
23
|
-
end
|
24
|
-
|
25
|
-
def find_next_message
|
26
|
-
query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
|
27
|
-
@collection.find_one(query)
|
28
|
-
end
|
29
|
-
|
30
|
-
def find_due_messages
|
31
|
-
query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
|
32
|
-
order = [ 'next_attempt', Mongo::ASCENDING ]
|
33
|
-
|
34
|
-
@collection.find(query).sort(order)
|
35
|
-
end
|
36
|
-
|
37
|
-
def get_by_id id
|
38
|
-
@collection.find_one({ '_id' => id })
|
39
|
-
end
|
40
|
-
|
41
|
-
def set_next_attempt message
|
42
|
-
next_attempt = Time.now.to_f + 10
|
43
|
-
|
44
|
-
id = { '_id' => message['_id'] }
|
45
|
-
doc = { '$set' => { 'next_attempt' => next_attempt,
|
46
|
-
'next_attempt_for_humans' => Time.at(next_attempt) } }
|
47
|
-
|
48
|
-
@collection.update id, doc, :safe => { :fsync => true }
|
49
|
-
end
|
50
|
-
|
51
|
-
def remove_published_message id
|
52
|
-
@collection.remove({ '_id' => id }, :safe => { :fsync => true })
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,29 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
module MongoModel
|
4
|
-
class MessageLog
|
5
|
-
def initialize name, mongodb
|
6
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
7
|
-
|
8
|
-
@name = name
|
9
|
-
@collection = mongodb.collection name
|
10
|
-
@collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
|
11
|
-
end
|
12
|
-
|
13
|
-
attr_reader :name
|
14
|
-
|
15
|
-
def find_all
|
16
|
-
@collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
|
17
|
-
end
|
18
|
-
|
19
|
-
def log_new_message message
|
20
|
-
@collection.save({ '_id' => message[:headers][:id],
|
21
|
-
'type' => message[:headers][:type],
|
22
|
-
'version' => message[:headers][:version],
|
23
|
-
'timestamp' => Time.now.to_f,
|
24
|
-
'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,32 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
class MessageBufferRakeTask < Euston::Daemons::RakeTask
|
3
|
-
attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
|
4
|
-
|
5
|
-
def initialize environment
|
6
|
-
super(environment, :message_buffer_daemon)
|
7
|
-
end
|
8
|
-
|
9
|
-
def before_creating_task
|
10
|
-
@daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
|
11
|
-
@daemon_class = 'Euston::MessageBufferDaemon::Daemon'
|
12
|
-
end
|
13
|
-
|
14
|
-
def initialize_settings
|
15
|
-
@logger.debug "AMQP config path: #{@amqp_config_path}"
|
16
|
-
@data[:amqp_config_path] = @amqp_config_path
|
17
|
-
|
18
|
-
@logger.debug "Daemon config path: #{@daemon_config_path}"
|
19
|
-
@data[:daemon_config_path] = @daemon_config_path
|
20
|
-
|
21
|
-
@logger.debug "Mongoid config path: #{@mongoid_config_path}"
|
22
|
-
@data[:mongoid_config_path] = @mongoid_config_path
|
23
|
-
end
|
24
|
-
|
25
|
-
def load_environment
|
26
|
-
@logger.debug "Loading environment"
|
27
|
-
require 'euston-daemons/message_buffer_daemon/config/environment'
|
28
|
-
|
29
|
-
Euston::MessageBufferDaemon::DaemonEnvironment.new(@data).setup
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module SnapshotDaemon
|
3
|
-
class Snapshotter < Euston::DaemonClient
|
4
|
-
def initialize event_store, threshold, logger
|
5
|
-
@event_store = event_store
|
6
|
-
@threshold = threshold
|
7
|
-
@log = logger
|
8
|
-
end
|
9
|
-
|
10
|
-
private
|
11
|
-
|
12
|
-
def next_iteration
|
13
|
-
begin
|
14
|
-
stream_heads = @event_store.get_streams_to_snapshot @threshold
|
15
|
-
@log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})"
|
16
|
-
|
17
|
-
stream_heads.each do |stream_head|
|
18
|
-
pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
|
19
|
-
|
20
|
-
loader = RabbitMq::ConstantLoader.new
|
21
|
-
loader.when(:hit => ->(klass) { take_snapshot klass, pair },
|
22
|
-
:miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
|
23
|
-
|
24
|
-
loader.load pair.stream.committed_headers[:aggregate_type]
|
25
|
-
end
|
26
|
-
end until stopped || stream_heads.empty?
|
27
|
-
end
|
28
|
-
|
29
|
-
def take_snapshot klass, pair
|
30
|
-
instance = klass.hydrate pair.stream, pair.snapshot
|
31
|
-
snapshot = instance.take_snapshot
|
32
|
-
snapshot = EventStore::Snapshot.new pair.stream.stream_id,
|
33
|
-
pair.stream.stream_revision,
|
34
|
-
snapshot[:payload],
|
35
|
-
:version => snapshot[:version]
|
36
|
-
|
37
|
-
@log.debug "Writing snapshot: #{snapshot.inspect}"
|
38
|
-
|
39
|
-
@event_store.add_snapshot snapshot
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
data/sample/Rakefile
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
unless defined?(Bundler)
|
2
|
-
require 'rubygems'
|
3
|
-
|
4
|
-
begin
|
5
|
-
require 'bundler'
|
6
|
-
rescue => e
|
7
|
-
STDERR.puts e.message
|
8
|
-
STDERR.puts "Try running `bundle install`."
|
9
|
-
exit!
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
RAKE_ENV = (ENV['RAKE_ENV'] || 'development').to_sym
|
14
|
-
|
15
|
-
Bundler.require :default, RAKE_ENV
|
16
|
-
|
17
|
-
require 'euston'
|
18
|
-
require 'euston-daemons/rake_tasks'
|
19
|
-
|
20
|
-
sample_path = File.dirname __FILE__
|
21
|
-
pid_path = File.join sample_path, 'pids'
|
22
|
-
amqp_config_path = File.join sample_path, "amqp_config.yml"
|
23
|
-
mongoid_config_path = File.join sample_path, 'mongoid_config.yml'
|
24
|
-
|
25
|
-
require 'erb-yaml'
|
26
|
-
amqp_config = ErbYaml.read amqp_config_path, RAKE_ENV
|
27
|
-
|
28
|
-
rabbitmqadmin = RabbitMqAdminCli.new
|
29
|
-
rabbitmqadmin.initialize_vhost amqp_config[:vhost]
|
30
|
-
rabbitmqadmin.initialize_user amqp_config[:vhost], amqp_config[:user], amqp_config[:pass]
|
31
|
-
|
32
|
-
desc "Run the command processor daemon"
|
33
|
-
Euston::CommandProcessorRakeTask.new(RAKE_ENV) do |t|
|
34
|
-
require File.join sample_path, 'command_handlers'
|
35
|
-
|
36
|
-
t.amqp_config_path = amqp_config_path
|
37
|
-
t.command_handler_namespaces = ['Euston::Daemons::Sample::CommandHandlers']
|
38
|
-
t.daemon_config_path = File.join sample_path, "command_processor_daemon_config.yml"
|
39
|
-
t.logger = Logger.new(STDOUT)
|
40
|
-
t.mongoid_config_path = mongoid_config_path
|
41
|
-
t.pid_path = pid_path
|
42
|
-
end
|
43
|
-
|
44
|
-
desc "Run the event processor daemon"
|
45
|
-
Euston::EventProcessorRakeTask.new(RAKE_ENV) do |t|
|
46
|
-
require File.join sample_path, 'event_handlers'
|
47
|
-
|
48
|
-
t.amqp_config_path = amqp_config_path
|
49
|
-
t.daemon_config_path = File.join sample_path, "event_processor_daemon_config.yml"
|
50
|
-
t.event_handler_namespaces = ['Euston::Daemons::Sample::EventHandlers']
|
51
|
-
t.logger = Logger.new(STDOUT)
|
52
|
-
t.mongoid_config_path = mongoid_config_path
|
53
|
-
t.pid_path = pid_path
|
54
|
-
end
|
55
|
-
|
56
|
-
desc "Run the message buffer daemon"
|
57
|
-
Euston::MessageBufferRakeTask.new(RAKE_ENV) do |t|
|
58
|
-
t.amqp_config_path = amqp_config_path
|
59
|
-
t.daemon_config_path = File.join sample_path, "message_buffer_daemon_config.yml"
|
60
|
-
t.logger = Logger.new(STDOUT)
|
61
|
-
t.mongoid_config_path = mongoid_config_path
|
62
|
-
t.pid_path = pid_path
|
63
|
-
end
|
data/sample/amqp_config.yml
DELETED
data/sample/command_handlers.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module Daemons
|
3
|
-
module Sample
|
4
|
-
module CommandHandlers
|
5
|
-
class ContactCustomer
|
6
|
-
include Euston::CommandHandler
|
7
|
-
version 1
|
8
|
-
end
|
9
|
-
|
10
|
-
class PostGoods
|
11
|
-
include Euston::CommandHandler
|
12
|
-
version 1
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
data/sample/event_handlers.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module Daemons
|
3
|
-
module Sample
|
4
|
-
module EventHandlers
|
5
|
-
class SalesOffice
|
6
|
-
include Euston::EventHandler
|
7
|
-
|
8
|
-
consumes :customer_contacted, 1 do |headers, event|
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
class Warehouse
|
13
|
-
include Euston::EventHandler
|
14
|
-
|
15
|
-
consumes :goods_posted, 1 do |headers, event|
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
data/sample/mongoid_config.yml
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
defaults: &defaults
|
2
|
-
host: localhost
|
3
|
-
port: 27017
|
4
|
-
safe: true
|
5
|
-
use_utc: true
|
6
|
-
event_store_database: euston-daemons-sample-event-store
|
7
|
-
read_model_database: euston-daemons-sample-read-model
|
8
|
-
|
9
|
-
development:
|
10
|
-
<<: *defaults
|
11
|
-
|
12
|
-
test:
|
13
|
-
<<: *defaults
|
data/sample/pids/.placeholder
DELETED
File without changes
|