euston-daemons 1.0.5 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -1
- data/Rakefile +2 -3
- data/euston-daemons.gemspec +50 -39
- data/lib/euston-daemons.rb +14 -1
- data/lib/euston-daemons/euston/daemon.rb +99 -0
- data/lib/euston-daemons/euston/daemon_component.rb +25 -0
- data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
- data/lib/euston-daemons/euston/exceptions.rb +9 -0
- data/lib/euston-daemons/euston/stopwatch.rb +15 -0
- data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
- data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
- data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
- data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
- data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
- data/lib/euston-daemons/rake_task.rb +63 -66
- data/lib/euston-daemons/rake_tasks.rb +3 -5
- data/lib/euston-daemons/version.rb +1 -1
- data/spec/daemons/command_processor_spec.rb +48 -0
- data/spec/daemons/event_processor_spec.rb +55 -0
- data/spec/daemons/message_buffer_spec.rb +106 -0
- data/spec/daemons/snapshotter_spec.rb +96 -0
- data/spec/spec_helper.rb +91 -0
- data/spec/support/factories/commands.rb +16 -0
- data/spec/support/factories/commit.rb +7 -0
- data/spec/support/factories/event_message.rb +12 -0
- data/spec/support/factories/events.rb +8 -0
- data/spec/support/filters.rb +13 -0
- data/spec/support/sample_model/commands.rb +14 -0
- data/spec/support/sample_model/counter.rb +36 -0
- data/spec/support/sample_model/counter2.rb +46 -0
- data/spec/support/stub_retrying_subscription.rb +9 -0
- metadata +131 -67
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
- data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
- data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
- data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
- data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
- data/lib/euston-daemons/framework/basic_component.rb +0 -33
- data/lib/euston-daemons/framework/channel_thread.rb +0 -22
- data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
- data/lib/euston-daemons/framework/daemon.rb +0 -27
- data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
- data/lib/euston-daemons/framework/queue.rb +0 -71
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
- data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
- data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
- data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
- data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
- data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
- data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
- data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,56 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Publisher
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
|
16
|
-
def initialize channel, exchange_name
|
17
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
18
|
-
|
19
|
-
@channel = channel
|
20
|
-
@exchange = get_exchange @channel, exchange_name
|
21
|
-
end
|
22
|
-
|
23
|
-
def disconnect
|
24
|
-
@channel.disconnect
|
25
|
-
end
|
26
|
-
|
27
|
-
def dispatch_one_due_message
|
28
|
-
if (message = @read_model.find_next_message)
|
29
|
-
@read_model.set_next_attempt message
|
30
|
-
protected_publish(message)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def dispatch_due_messages
|
35
|
-
dispatched = 0
|
36
|
-
@read_model.find_due_messages.each do |message|
|
37
|
-
@read_model.set_next_attempt message
|
38
|
-
break unless protected_publish(message)
|
39
|
-
dispatched += 1
|
40
|
-
end
|
41
|
-
dispatched
|
42
|
-
end
|
43
|
-
|
44
|
-
def protected_publish(message)
|
45
|
-
ret = true
|
46
|
-
begin
|
47
|
-
@exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
|
48
|
-
rescue NativeException => e
|
49
|
-
Thread.current[:exception] = e
|
50
|
-
ret = false
|
51
|
-
end
|
52
|
-
ret
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
module ReadModel
|
4
|
-
class MessageLog
|
5
|
-
class << self
|
6
|
-
def commands
|
7
|
-
self.new "command_log", Euston::RabbitMq.event_store_mongodb
|
8
|
-
end
|
9
|
-
|
10
|
-
def events
|
11
|
-
self.new "event_log", Euston::RabbitMq.event_store_mongodb
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def initialize name, mongodb
|
16
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
17
|
-
|
18
|
-
@collection = mongodb.collection name
|
19
|
-
@collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
|
20
|
-
end
|
21
|
-
|
22
|
-
def find_all
|
23
|
-
@collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
|
24
|
-
end
|
25
|
-
|
26
|
-
def log_new_message message
|
27
|
-
@collection.save({ '_id' => message[:headers][:id],
|
28
|
-
'type' => message[:headers][:type],
|
29
|
-
'version' => message[:headers][:version],
|
30
|
-
'timestamp' => Time.now.to_f,
|
31
|
-
'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Subscriber
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
include Euston::RabbitMq::Queues
|
16
|
-
include Hollywood
|
17
|
-
|
18
|
-
def initialize channel, exchange_name
|
19
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
20
|
-
|
21
|
-
@channel = channel
|
22
|
-
@channel.prefetch(1)
|
23
|
-
@exchange = get_exchange @channel, exchange_name
|
24
|
-
@queue = get_queue @channel, "#{exchange_name}_buffer"
|
25
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
26
|
-
|
27
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
28
|
-
:message_failed => method(:message_failed),
|
29
|
-
:message_received => method(:remove_message_from_buffer))
|
30
|
-
|
31
|
-
@consumer = @queue.consumer
|
32
|
-
end
|
33
|
-
|
34
|
-
def disconnect
|
35
|
-
@channel.disconnect
|
36
|
-
end
|
37
|
-
|
38
|
-
def dequeue_buffered_messages(timeout) #will block until message or timeout
|
39
|
-
@queue.safe_subscribe_with_timeout(@consumer, timeout)
|
40
|
-
end
|
41
|
-
|
42
|
-
def log_failure message, error
|
43
|
-
text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
45
|
-
err.set_backtrace error.backtrace
|
46
|
-
|
47
|
-
Safely.report! err
|
48
|
-
end
|
49
|
-
|
50
|
-
def message_failed(message, error, header)
|
51
|
-
header.ack!
|
52
|
-
log_failure message, error
|
53
|
-
end
|
54
|
-
|
55
|
-
def remove_message_from_buffer message
|
56
|
-
@read_model.remove_published_message message[:headers][:id]
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
class MessageBufferRakeTask < Euston::Daemons::RakeTask
|
3
|
-
attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
|
4
|
-
|
5
|
-
def initialize
|
6
|
-
super(:message_buffer_daemon)
|
7
|
-
end
|
8
|
-
|
9
|
-
def before_creating_task
|
10
|
-
@daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
|
11
|
-
@daemon_class = 'Euston::MessageBufferDaemon::Daemon'
|
12
|
-
end
|
13
|
-
|
14
|
-
def initialize_paths
|
15
|
-
EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
|
16
|
-
Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
|
17
|
-
|
18
|
-
EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
|
19
|
-
Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
|
20
|
-
|
21
|
-
EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
|
22
|
-
Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
|
23
|
-
end
|
24
|
-
|
25
|
-
def load_environment
|
26
|
-
EUSTON_LOG.debug "Loading environment"
|
27
|
-
require_rel 'config/environment.rb'
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|