isimud 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/.yardoc/checksums +15 -0
- data/.yardoc/object_types +0 -0
- data/.yardoc/objects/root.dat +0 -0
- data/.yardoc/proxy_types +0 -0
- data/Gemfile +23 -0
- data/Gemfile.lock +123 -0
- data/README.md +218 -0
- data/Rakefile +2 -0
- data/config.ru +7 -0
- data/config/tddium.yml +11 -0
- data/doc/Isimud.html +1696 -0
- data/doc/Isimud/BunnyClient.html +1004 -0
- data/doc/Isimud/Client.html +812 -0
- data/doc/Isimud/Event.html +1500 -0
- data/doc/Isimud/EventListener.html +1217 -0
- data/doc/Isimud/EventObserver.html +367 -0
- data/doc/Isimud/EventObserver/ClassMethods.html +292 -0
- data/doc/Isimud/Generators.html +117 -0
- data/doc/Isimud/Generators/ConfigGenerator.html +192 -0
- data/doc/Isimud/Generators/InitializerGenerator.html +192 -0
- data/doc/Isimud/Logging.html +230 -0
- data/doc/Isimud/ModelWatcher.html +312 -0
- data/doc/Isimud/ModelWatcher/ClassMethods.html +511 -0
- data/doc/Isimud/Railtie.html +123 -0
- data/doc/Isimud/TestClient.html +1003 -0
- data/doc/Isimud/TestClient/Queue.html +556 -0
- data/doc/_index.html +290 -0
- data/doc/class_list.html +58 -0
- data/doc/css/common.css +1 -0
- data/doc/css/full_list.css +57 -0
- data/doc/css/style.css +339 -0
- data/doc/file.README.html +338 -0
- data/doc/file_list.html +60 -0
- data/doc/frames.html +26 -0
- data/doc/index.html +338 -0
- data/doc/js/app.js +219 -0
- data/doc/js/full_list.js +181 -0
- data/doc/js/jquery.js +4 -0
- data/doc/method_list.html +711 -0
- data/doc/top-level-namespace.html +112 -0
- data/isimud.gemspec +25 -0
- data/lib/isimud.rb +91 -0
- data/lib/isimud/bunny_client.rb +95 -0
- data/lib/isimud/client.rb +48 -0
- data/lib/isimud/event.rb +112 -0
- data/lib/isimud/event_listener.rb +200 -0
- data/lib/isimud/event_observer.rb +81 -0
- data/lib/isimud/logging.rb +11 -0
- data/lib/isimud/model_watcher.rb +144 -0
- data/lib/isimud/railtie.rb +9 -0
- data/lib/isimud/tasks.rb +20 -0
- data/lib/isimud/test_client.rb +89 -0
- data/lib/isimud/version.rb +3 -0
- data/lib/rails/generators/isimud/config_generator.rb +12 -0
- data/lib/rails/generators/isimud/initializer_generator.rb +12 -0
- data/lib/rails/generators/isimud/templates/initializer.rb +17 -0
- data/lib/rails/generators/isimud/templates/isimud.yml +20 -0
- data/spec/internal/app/models/admin.rb +2 -0
- data/spec/internal/app/models/company.rb +34 -0
- data/spec/internal/app/models/user.rb +27 -0
- data/spec/internal/config/database.yml +3 -0
- data/spec/internal/config/routes.rb +3 -0
- data/spec/internal/db/schema.rb +22 -0
- data/spec/internal/log/.gitignore +1 -0
- data/spec/internal/public/favicon.ico +0 -0
- data/spec/isimud/bunny_client_spec.rb +125 -0
- data/spec/isimud/event_listener_spec.rb +86 -0
- data/spec/isimud/event_observer_spec.rb +32 -0
- data/spec/isimud/event_spec.rb +74 -0
- data/spec/isimud/model_watcher_spec.rb +189 -0
- data/spec/isimud/test_client_spec.rb +28 -0
- data/spec/isimud_spec.rb +49 -0
- data/spec/spec_helper.rb +55 -0
- metadata +195 -0
@@ -0,0 +1,200 @@
|
|
1
|
+
require 'isimud'
|
2
|
+
require 'thread'
|
3
|
+
|
4
|
+
module Isimud
|
5
|
+
class EventListener
|
6
|
+
include Logging
|
7
|
+
attr_reader :error_count, :error_interval, :error_limit, :name, :queues, :events_exchange, :models_exchange,
|
8
|
+
:running
|
9
|
+
|
10
|
+
DEFAULT_ERROR_LIMIT = 10
|
11
|
+
DEFAULT_ERROR_INTERVAL = 3600
|
12
|
+
|
13
|
+
DEFAULT_EVENTS_EXCHANGE = 'events'
|
14
|
+
DEFAULT_MODELS_EXCHANGE = 'models'
|
15
|
+
|
16
|
+
def initialize(options = {})
|
17
|
+
default_options = {
|
18
|
+
error_limit: Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT,
|
19
|
+
error_interval: DEFAULT_ERROR_INTERVAL,
|
20
|
+
events_exchange: Isimud.events_exchange || DEFAULT_EVENTS_EXCHANGE,
|
21
|
+
models_exchange: Isimud.model_watcher_exchange || DEFAULT_MODELS_EXCHANGE,
|
22
|
+
name: "#{Rails.application.class.parent_name.downcase}-listener"
|
23
|
+
}
|
24
|
+
options.reverse_merge!(default_options)
|
25
|
+
@error_count = 0
|
26
|
+
@observers = Hash.new
|
27
|
+
@observed_models = Set.new
|
28
|
+
@error_limit = options[:error_limit]
|
29
|
+
@error_interval = options[:error_interval]
|
30
|
+
@events_exchange = options[:events_exchange]
|
31
|
+
@models_exchange = options[:models_exchange]
|
32
|
+
@name = options[:name]
|
33
|
+
@observer_mutex = Mutex.new
|
34
|
+
@error_counter_mutex = Mutex.new
|
35
|
+
@running = false
|
36
|
+
end
|
37
|
+
|
38
|
+
def max_errors
|
39
|
+
Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_env?
|
43
|
+
['cucumber', 'test'].include?(Rails.env)
|
44
|
+
end
|
45
|
+
|
46
|
+
def run
|
47
|
+
@running = true
|
48
|
+
bind_queues and return if test_env?
|
49
|
+
start_shutdown_thread
|
50
|
+
start_error_counter_thread
|
51
|
+
client.on_exception do |e|
|
52
|
+
count_error(e)
|
53
|
+
end
|
54
|
+
client.connect
|
55
|
+
start_event_thread
|
56
|
+
|
57
|
+
puts 'EventListener started. Hit Ctrl-C to exit'
|
58
|
+
Thread.stop
|
59
|
+
puts 'Main thread wakeup - exiting.'
|
60
|
+
client.close
|
61
|
+
end
|
62
|
+
|
63
|
+
# Override this method to set up message observers
|
64
|
+
def bind_queues
|
65
|
+
Isimud::EventObserver.observed_models.each do |model_class|
|
66
|
+
log "EventListener: registering observers for #{model_class}"
|
67
|
+
register_observer_class(model_class)
|
68
|
+
count = 0
|
69
|
+
model_class.find_active_observers.each do |model|
|
70
|
+
register_observer(model)
|
71
|
+
count += 1
|
72
|
+
end
|
73
|
+
log "EventListener: registered #{count} observers for #{model_class}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def has_observer?(observer)
|
78
|
+
@observers.has_key?(observer_key_for(observer.class, observer.id))
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def start_shutdown_thread
|
84
|
+
shutdown_thread = Thread.new do
|
85
|
+
Thread.stop # wait until we get a TERM or INT signal.
|
86
|
+
log 'EventListener: shutdown requested. Shutting down AMQP...', :info
|
87
|
+
@running = false
|
88
|
+
Thread.main.run
|
89
|
+
end
|
90
|
+
%w(SIGINT SIGTERM).each { |sig| trap(sig) { shutdown_thread.wakeup } }
|
91
|
+
end
|
92
|
+
|
93
|
+
def client
|
94
|
+
Isimud.client
|
95
|
+
end
|
96
|
+
|
97
|
+
def start_event_thread
|
98
|
+
Thread.new do
|
99
|
+
log 'EventListener: starting event_thread'
|
100
|
+
while @running
|
101
|
+
begin
|
102
|
+
bind_queues
|
103
|
+
Rails.logger.info 'EventListener: event_thread finished'
|
104
|
+
Thread.stop
|
105
|
+
rescue Bunny::Exception => e
|
106
|
+
count_error(e)
|
107
|
+
Rails.logger.warn 'EventListener: resetting queues'
|
108
|
+
client.reset
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def count_error(exception)
|
115
|
+
@error_counter_mutex.synchronize do
|
116
|
+
@error_count += 1
|
117
|
+
log "EventListener#count_error count = #{@error_count} limit=#{error_limit}", :warn
|
118
|
+
if (@error_count >= error_limit)
|
119
|
+
log 'EventListener: too many errors, exiting', :fatal
|
120
|
+
@running = false
|
121
|
+
Thread.main.run unless test_env?
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# start an error counter thread that clears the error count once per hour
|
127
|
+
def start_error_counter_thread
|
128
|
+
log 'EventListener: starting error counter'
|
129
|
+
@error_count = 0
|
130
|
+
Thread.new do
|
131
|
+
while true
|
132
|
+
sleep(error_interval)
|
133
|
+
@error_counter_mutex.synchronize do
|
134
|
+
log('EventListener: resetting error counter') if @error_count > 0
|
135
|
+
@error_count = 0
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def handle_observer_event(payload)
|
142
|
+
event = JSON.parse(payload).with_indifferent_access
|
143
|
+
log "EventListener: received observer model message: #{payload.inspect}"
|
144
|
+
if %w(update destroy).include?(event[:action])
|
145
|
+
unregister_observer(event[:type], event[:id])
|
146
|
+
end
|
147
|
+
unless event[:action] == 'destroy'
|
148
|
+
observer = event[:type].constantize.find(event[:id])
|
149
|
+
register_observer(observer) if observer.enable_listener?
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
# Create and bind a queue for the observer. Also ensure that we are listening for observer class update events
|
154
|
+
def register_observer(observer)
|
155
|
+
@observer_mutex.synchronize do
|
156
|
+
log "EventListener: registering observer #{observer.class} #{observer.id}"
|
157
|
+
observer.observe_events(client, events_exchange)
|
158
|
+
@observers[observer_key_for(observer.class, observer.id)] = observer
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
# Delete a queue for an observer. This also purges all messages associated with it
|
163
|
+
def unregister_observer(observer_class, observer_id)
|
164
|
+
@observer_mutex.synchronize do
|
165
|
+
if (observer = @observers.delete(observer_key_for(observer_class, observer_id)))
|
166
|
+
begin
|
167
|
+
log "EventListener: unregistering #{observer.class} #{observer.id}"
|
168
|
+
queue_name = observer_class.constantize.event_queue_name(observer_id)
|
169
|
+
client.delete_queue(queue_name)
|
170
|
+
rescue => e
|
171
|
+
log "EventListener: error unregistering #{observer_class} #{observer_id}: #{e.message}", :warn
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
# Create or return the observer queue which listens for ModelWatcher events
|
178
|
+
def observer_queue
|
179
|
+
@observer_queue ||= client.create_queue("#{name}.listener.#{Process.pid}", models_exchange,
|
180
|
+
queue_options: {exclusive: true},
|
181
|
+
subscribe_options: {manual_ack: true}, &method(:handle_observer_event))
|
182
|
+
end
|
183
|
+
|
184
|
+
# Register the observer class watcher
|
185
|
+
def register_observer_class(observer_class)
|
186
|
+
@observer_mutex.synchronize do
|
187
|
+
return if @observed_models.include?(observer_class)
|
188
|
+
@observed_models << observer_class
|
189
|
+
log "EventListener: registering observer class #{observer_class}"
|
190
|
+
observer_queue.bind(models_exchange, routing_key: "#{Isimud.model_watcher_schema}.#{observer_class.base_class.name}.*")
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def observer_key_for(type, id)
|
195
|
+
[type.to_s, id.to_s].join(':')
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support'
|
3
|
+
require 'active_support/concern'
|
4
|
+
require 'active_support/core_ext/module/attribute_accessors'
|
5
|
+
|
6
|
+
# Module for attaching and listening to events
|
7
|
+
module Isimud
|
8
|
+
module EventObserver
|
9
|
+
extend ::ActiveSupport::Concern
|
10
|
+
include Isimud::Logging
|
11
|
+
|
12
|
+
mattr_accessor :observed_models do
|
13
|
+
Array.new
|
14
|
+
end
|
15
|
+
|
16
|
+
mattr_accessor :observed_mutex do
|
17
|
+
Mutex.new
|
18
|
+
end
|
19
|
+
|
20
|
+
# Event handling hook. Override in your class.
|
21
|
+
def handle_event(event)
|
22
|
+
Rails.logger.warn("Isimud::EventObserver#handle_event not implemented for #{event_queue_name}")
|
23
|
+
end
|
24
|
+
|
25
|
+
# Routing keys that are bound to the event queue. Override in your subclass
|
26
|
+
def routing_keys
|
27
|
+
[]
|
28
|
+
end
|
29
|
+
|
30
|
+
# Returns true if this instance is enabled for listening to events. Override in your subclass.
|
31
|
+
def enable_listener?
|
32
|
+
true
|
33
|
+
end
|
34
|
+
|
35
|
+
# Exchange used for listening to events. Override in your subclass if you want to specify an alternative exchange for
|
36
|
+
# events. Otherwise
|
37
|
+
def observed_exchange
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
|
41
|
+
# Create or attach to a queue on the specified exchange. When an event message that matches the observer's routing keys
|
42
|
+
# is received, parse the event and call handle_event on same.
|
43
|
+
def observe_events(client, default_exchange)
|
44
|
+
client.bind(self.class.event_queue_name(id), observed_exchange || default_exchange, *routing_keys) do |message|
|
45
|
+
event = Event.parse(message)
|
46
|
+
handle_event(event)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
module ClassMethods
|
51
|
+
# Method used to retrieve active observers. Override in your EventObserver class
|
52
|
+
def find_active_observers
|
53
|
+
[]
|
54
|
+
end
|
55
|
+
|
56
|
+
def queue_prefix
|
57
|
+
Rails.application.class.parent_name.downcase
|
58
|
+
end
|
59
|
+
|
60
|
+
def event_queue_name(id)
|
61
|
+
[queue_prefix, base_class.name.underscore, id].join('.')
|
62
|
+
end
|
63
|
+
|
64
|
+
protected
|
65
|
+
|
66
|
+
def register_class
|
67
|
+
Isimud::EventObserver.observed_mutex.synchronize do
|
68
|
+
unless Isimud::EventObserver.observed_models.include?(self.base_class)
|
69
|
+
Rails.logger.info("Isimud::EventObserver: registering #{self.base_class}")
|
70
|
+
Isimud::EventObserver.observed_models << self.base_class
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
included do
|
77
|
+
include Isimud::ModelWatcher unless self.include?(Isimud::ModelWatcher)
|
78
|
+
register_class
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support'
|
3
|
+
require 'active_support/concern'
|
4
|
+
require 'active_support/core_ext/module/attribute_accessors'
|
5
|
+
|
6
|
+
module Isimud
|
7
|
+
# ActiveModel mixin for sending model updates to a message server.
|
8
|
+
module ModelWatcher
|
9
|
+
extend ::ActiveSupport::Concern
|
10
|
+
include Isimud::Logging
|
11
|
+
|
12
|
+
mattr_accessor :watched_models
|
13
|
+
|
14
|
+
DEFAULT_EXCHANGE = 'models'
|
15
|
+
IGNORED_COLUMNS = %w{id}
|
16
|
+
|
17
|
+
included do
|
18
|
+
ModelWatcher.watched_models ||= Array.new
|
19
|
+
ModelWatcher.watched_models << self.name
|
20
|
+
cattr_accessor :isimud_watch_attributes
|
21
|
+
cattr_accessor :sync_includes
|
22
|
+
|
23
|
+
after_commit :isimud_notify_created, on: :create
|
24
|
+
after_commit :isimud_notify_updated, on: :update
|
25
|
+
after_commit :isimud_notify_destroyed, on: :destroy
|
26
|
+
end
|
27
|
+
|
28
|
+
module ClassMethods
|
29
|
+
# Set attributes to observe and include in messages. Any property method with a return value may be included
|
30
|
+
# in the list of attributes.
|
31
|
+
# @param [Array<String,Symbol>] attributes list of attributes / properties
|
32
|
+
def watch_attributes(*attributes)
|
33
|
+
self.isimud_watch_attributes = attributes.flatten.map(&:to_s) if attributes.present?
|
34
|
+
end
|
35
|
+
|
36
|
+
# Include the following tables when fetching records for synchronization
|
37
|
+
def sync_include(_sync_includes)
|
38
|
+
self.sync_includes = _sync_includes
|
39
|
+
end
|
40
|
+
|
41
|
+
# Synchronize instances of this model with the data warehouse. This is accomplished by calling
|
42
|
+
# isimud_notify_updated() on each instance fetched from the database.
|
43
|
+
# @param [Hash] options synchronize options
|
44
|
+
# @option options [ActiveRecord::Relation] :where where_clause filter for limiting records to sync. By default, all records are synchronized.
|
45
|
+
# @option options [IO] :output optional stream for writing progress. A '.' is printed for every 100 records synchronized.
|
46
|
+
# @return [Integer] number of records synchronized
|
47
|
+
def synchronize(options = {})
|
48
|
+
where_clause = options[:where] || {}
|
49
|
+
output = options[:output] || nil
|
50
|
+
count = 0
|
51
|
+
query = self.where(where_clause)
|
52
|
+
query = query.includes(sync_includes) if sync_includes
|
53
|
+
query.find_each do |m|
|
54
|
+
next unless m.isimud_synchronize?
|
55
|
+
begin
|
56
|
+
m.isimud_sync
|
57
|
+
rescue Bunny::ClientTimeout, Timeout::Error => e
|
58
|
+
output && output.print("\n#{e}, sleeping for 10 seconds")
|
59
|
+
sleep(10)
|
60
|
+
m.isimud_sync
|
61
|
+
end
|
62
|
+
if (count += 1) % 100 == 0
|
63
|
+
output && output.print('.')
|
64
|
+
end
|
65
|
+
if (count % 1000) == 0
|
66
|
+
GC.start
|
67
|
+
end
|
68
|
+
end
|
69
|
+
count
|
70
|
+
end
|
71
|
+
|
72
|
+
def isimud_model_watcher_type
|
73
|
+
(respond_to?(:base_class) ? base_class.name : name).demodulize
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Override to set conditions for synchronizing this instance with the server (default is always)
|
78
|
+
def isimud_synchronize?
|
79
|
+
true
|
80
|
+
end
|
81
|
+
|
82
|
+
def isimud_sync
|
83
|
+
isimud_send_action_message(:update)
|
84
|
+
end
|
85
|
+
|
86
|
+
protected
|
87
|
+
|
88
|
+
def isimud_notify_created
|
89
|
+
isimud_send_action_message(:create)
|
90
|
+
end
|
91
|
+
|
92
|
+
def isimud_notify_updated
|
93
|
+
changed_attrs = previous_changes.keys
|
94
|
+
attributes = isimud_watch_attributes || isimud_default_attributes
|
95
|
+
isimud_send_action_message(:update) if (changed_attrs & attributes).any?
|
96
|
+
end
|
97
|
+
|
98
|
+
def isimud_notify_destroyed
|
99
|
+
isimud_send_action_message(:destroy)
|
100
|
+
end
|
101
|
+
|
102
|
+
def isimud_default_attributes
|
103
|
+
self.class.column_names - IGNORED_COLUMNS
|
104
|
+
end
|
105
|
+
|
106
|
+
def isimud_attribute_data
|
107
|
+
attributes = isimud_watch_attributes || isimud_default_attributes
|
108
|
+
attributes.inject(Hash.new) { |hsh, attr| hsh[attr] = send(attr); hsh }
|
109
|
+
end
|
110
|
+
|
111
|
+
def isimud_model_watcher_schema
|
112
|
+
Isimud.model_watcher_schema || if defined?(Rails)
|
113
|
+
Rails.configuration.database_configuration[Rails.env]['database']
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def isimud_model_watcher_exchange
|
118
|
+
Isimud.model_watcher_exchange
|
119
|
+
end
|
120
|
+
|
121
|
+
def isimud_model_watcher_type
|
122
|
+
self.class.isimud_model_watcher_type
|
123
|
+
end
|
124
|
+
|
125
|
+
def isimud_model_watcher_routing_key(action)
|
126
|
+
[isimud_model_watcher_schema, isimud_model_watcher_type, action].join('.')
|
127
|
+
end
|
128
|
+
|
129
|
+
def isimud_send_action_message(action)
|
130
|
+
return unless Isimud.model_watcher_enabled? && isimud_synchronize?
|
131
|
+
payload = {
|
132
|
+
schema: isimud_model_watcher_schema,
|
133
|
+
type: isimud_model_watcher_type,
|
134
|
+
action: action,
|
135
|
+
id: id,
|
136
|
+
timestamp: (updated_at || Time.now).utc
|
137
|
+
}
|
138
|
+
payload[:attributes] = isimud_attribute_data unless action == :destroy
|
139
|
+
routing_key = isimud_model_watcher_routing_key(action)
|
140
|
+
log "Isimud::ModelWatcher#publish: exchange #{isimud_model_watcher_exchange} routing_key #{routing_key} payload #{payload.inspect}"
|
141
|
+
Isimud.client.publish(isimud_model_watcher_exchange, routing_key, payload.to_json)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|