euston-daemons 1.0.5 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +4 -1
- data/Rakefile +2 -3
- data/euston-daemons.gemspec +50 -39
- data/lib/euston-daemons.rb +14 -1
- data/lib/euston-daemons/euston/daemon.rb +99 -0
- data/lib/euston-daemons/euston/daemon_component.rb +25 -0
- data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
- data/lib/euston-daemons/euston/exceptions.rb +9 -0
- data/lib/euston-daemons/euston/stopwatch.rb +15 -0
- data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
- data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
- data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
- data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
- data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
- data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
- data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
- data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
- data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
- data/lib/euston-daemons/rake_task.rb +63 -66
- data/lib/euston-daemons/rake_tasks.rb +3 -5
- data/lib/euston-daemons/version.rb +1 -1
- data/spec/daemons/command_processor_spec.rb +48 -0
- data/spec/daemons/event_processor_spec.rb +55 -0
- data/spec/daemons/message_buffer_spec.rb +106 -0
- data/spec/daemons/snapshotter_spec.rb +96 -0
- data/spec/spec_helper.rb +91 -0
- data/spec/support/factories/commands.rb +16 -0
- data/spec/support/factories/commit.rb +7 -0
- data/spec/support/factories/event_message.rb +12 -0
- data/spec/support/factories/events.rb +8 -0
- data/spec/support/filters.rb +13 -0
- data/spec/support/sample_model/commands.rb +14 -0
- data/spec/support/sample_model/counter.rb +36 -0
- data/spec/support/sample_model/counter2.rb +46 -0
- data/spec/support/stub_retrying_subscription.rb +9 -0
- metadata +131 -67
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
- data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
- data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
- data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
- data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
- data/lib/euston-daemons/framework/basic_component.rb +0 -33
- data/lib/euston-daemons/framework/channel_thread.rb +0 -22
- data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
- data/lib/euston-daemons/framework/daemon.rb +0 -27
- data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
- data/lib/euston-daemons/framework/queue.rb +0 -71
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
- data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
- data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
- data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
- data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
- data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
- data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
- data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,56 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Publisher
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
|
16
|
-
def initialize channel, exchange_name
|
17
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
18
|
-
|
19
|
-
@channel = channel
|
20
|
-
@exchange = get_exchange @channel, exchange_name
|
21
|
-
end
|
22
|
-
|
23
|
-
def disconnect
|
24
|
-
@channel.disconnect
|
25
|
-
end
|
26
|
-
|
27
|
-
def dispatch_one_due_message
|
28
|
-
if (message = @read_model.find_next_message)
|
29
|
-
@read_model.set_next_attempt message
|
30
|
-
protected_publish(message)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def dispatch_due_messages
|
35
|
-
dispatched = 0
|
36
|
-
@read_model.find_due_messages.each do |message|
|
37
|
-
@read_model.set_next_attempt message
|
38
|
-
break unless protected_publish(message)
|
39
|
-
dispatched += 1
|
40
|
-
end
|
41
|
-
dispatched
|
42
|
-
end
|
43
|
-
|
44
|
-
def protected_publish(message)
|
45
|
-
ret = true
|
46
|
-
begin
|
47
|
-
@exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
|
48
|
-
rescue NativeException => e
|
49
|
-
Thread.current[:exception] = e
|
50
|
-
ret = false
|
51
|
-
end
|
52
|
-
ret
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
module ReadModel
|
4
|
-
class MessageLog
|
5
|
-
class << self
|
6
|
-
def commands
|
7
|
-
self.new "command_log", Euston::RabbitMq.event_store_mongodb
|
8
|
-
end
|
9
|
-
|
10
|
-
def events
|
11
|
-
self.new "event_log", Euston::RabbitMq.event_store_mongodb
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def initialize name, mongodb
|
16
|
-
mongodb.create_collection name unless mongodb.collection_names.include? name
|
17
|
-
|
18
|
-
@collection = mongodb.collection name
|
19
|
-
@collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
|
20
|
-
end
|
21
|
-
|
22
|
-
def find_all
|
23
|
-
@collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
|
24
|
-
end
|
25
|
-
|
26
|
-
def log_new_message message
|
27
|
-
@collection.save({ '_id' => message[:headers][:id],
|
28
|
-
'type' => message[:headers][:type],
|
29
|
-
'version' => message[:headers][:version],
|
30
|
-
'timestamp' => Time.now.to_f,
|
31
|
-
'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Subscriber
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
include Euston::RabbitMq::Queues
|
16
|
-
include Hollywood
|
17
|
-
|
18
|
-
def initialize channel, exchange_name
|
19
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
20
|
-
|
21
|
-
@channel = channel
|
22
|
-
@channel.prefetch(1)
|
23
|
-
@exchange = get_exchange @channel, exchange_name
|
24
|
-
@queue = get_queue @channel, "#{exchange_name}_buffer"
|
25
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
26
|
-
|
27
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
28
|
-
:message_failed => method(:message_failed),
|
29
|
-
:message_received => method(:remove_message_from_buffer))
|
30
|
-
|
31
|
-
@consumer = @queue.consumer
|
32
|
-
end
|
33
|
-
|
34
|
-
def disconnect
|
35
|
-
@channel.disconnect
|
36
|
-
end
|
37
|
-
|
38
|
-
def dequeue_buffered_messages(timeout) #will block until message or timeout
|
39
|
-
@queue.safe_subscribe_with_timeout(@consumer, timeout)
|
40
|
-
end
|
41
|
-
|
42
|
-
def log_failure message, error
|
43
|
-
text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
45
|
-
err.set_backtrace error.backtrace
|
46
|
-
|
47
|
-
Safely.report! err
|
48
|
-
end
|
49
|
-
|
50
|
-
def message_failed(message, error, header)
|
51
|
-
header.ack!
|
52
|
-
log_failure message, error
|
53
|
-
end
|
54
|
-
|
55
|
-
def remove_message_from_buffer message
|
56
|
-
@read_model.remove_published_message message[:headers][:id]
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
class MessageBufferRakeTask < Euston::Daemons::RakeTask
|
3
|
-
attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
|
4
|
-
|
5
|
-
def initialize
|
6
|
-
super(:message_buffer_daemon)
|
7
|
-
end
|
8
|
-
|
9
|
-
def before_creating_task
|
10
|
-
@daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
|
11
|
-
@daemon_class = 'Euston::MessageBufferDaemon::Daemon'
|
12
|
-
end
|
13
|
-
|
14
|
-
def initialize_paths
|
15
|
-
EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
|
16
|
-
Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
|
17
|
-
|
18
|
-
EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
|
19
|
-
Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
|
20
|
-
|
21
|
-
EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
|
22
|
-
Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
|
23
|
-
end
|
24
|
-
|
25
|
-
def load_environment
|
26
|
-
EUSTON_LOG.debug "Loading environment"
|
27
|
-
require_rel 'config/environment.rb'
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|