isimud 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +18 -0
  3. data/.rspec +2 -0
  4. data/.ruby-version +1 -0
  5. data/.yardoc/checksums +15 -0
  6. data/.yardoc/object_types +0 -0
  7. data/.yardoc/objects/root.dat +0 -0
  8. data/.yardoc/proxy_types +0 -0
  9. data/Gemfile +23 -0
  10. data/Gemfile.lock +123 -0
  11. data/README.md +218 -0
  12. data/Rakefile +2 -0
  13. data/config.ru +7 -0
  14. data/config/tddium.yml +11 -0
  15. data/doc/Isimud.html +1696 -0
  16. data/doc/Isimud/BunnyClient.html +1004 -0
  17. data/doc/Isimud/Client.html +812 -0
  18. data/doc/Isimud/Event.html +1500 -0
  19. data/doc/Isimud/EventListener.html +1217 -0
  20. data/doc/Isimud/EventObserver.html +367 -0
  21. data/doc/Isimud/EventObserver/ClassMethods.html +292 -0
  22. data/doc/Isimud/Generators.html +117 -0
  23. data/doc/Isimud/Generators/ConfigGenerator.html +192 -0
  24. data/doc/Isimud/Generators/InitializerGenerator.html +192 -0
  25. data/doc/Isimud/Logging.html +230 -0
  26. data/doc/Isimud/ModelWatcher.html +312 -0
  27. data/doc/Isimud/ModelWatcher/ClassMethods.html +511 -0
  28. data/doc/Isimud/Railtie.html +123 -0
  29. data/doc/Isimud/TestClient.html +1003 -0
  30. data/doc/Isimud/TestClient/Queue.html +556 -0
  31. data/doc/_index.html +290 -0
  32. data/doc/class_list.html +58 -0
  33. data/doc/css/common.css +1 -0
  34. data/doc/css/full_list.css +57 -0
  35. data/doc/css/style.css +339 -0
  36. data/doc/file.README.html +338 -0
  37. data/doc/file_list.html +60 -0
  38. data/doc/frames.html +26 -0
  39. data/doc/index.html +338 -0
  40. data/doc/js/app.js +219 -0
  41. data/doc/js/full_list.js +181 -0
  42. data/doc/js/jquery.js +4 -0
  43. data/doc/method_list.html +711 -0
  44. data/doc/top-level-namespace.html +112 -0
  45. data/isimud.gemspec +25 -0
  46. data/lib/isimud.rb +91 -0
  47. data/lib/isimud/bunny_client.rb +95 -0
  48. data/lib/isimud/client.rb +48 -0
  49. data/lib/isimud/event.rb +112 -0
  50. data/lib/isimud/event_listener.rb +200 -0
  51. data/lib/isimud/event_observer.rb +81 -0
  52. data/lib/isimud/logging.rb +11 -0
  53. data/lib/isimud/model_watcher.rb +144 -0
  54. data/lib/isimud/railtie.rb +9 -0
  55. data/lib/isimud/tasks.rb +20 -0
  56. data/lib/isimud/test_client.rb +89 -0
  57. data/lib/isimud/version.rb +3 -0
  58. data/lib/rails/generators/isimud/config_generator.rb +12 -0
  59. data/lib/rails/generators/isimud/initializer_generator.rb +12 -0
  60. data/lib/rails/generators/isimud/templates/initializer.rb +17 -0
  61. data/lib/rails/generators/isimud/templates/isimud.yml +20 -0
  62. data/spec/internal/app/models/admin.rb +2 -0
  63. data/spec/internal/app/models/company.rb +34 -0
  64. data/spec/internal/app/models/user.rb +27 -0
  65. data/spec/internal/config/database.yml +3 -0
  66. data/spec/internal/config/routes.rb +3 -0
  67. data/spec/internal/db/schema.rb +22 -0
  68. data/spec/internal/log/.gitignore +1 -0
  69. data/spec/internal/public/favicon.ico +0 -0
  70. data/spec/isimud/bunny_client_spec.rb +125 -0
  71. data/spec/isimud/event_listener_spec.rb +86 -0
  72. data/spec/isimud/event_observer_spec.rb +32 -0
  73. data/spec/isimud/event_spec.rb +74 -0
  74. data/spec/isimud/model_watcher_spec.rb +189 -0
  75. data/spec/isimud/test_client_spec.rb +28 -0
  76. data/spec/isimud_spec.rb +49 -0
  77. data/spec/spec_helper.rb +55 -0
  78. metadata +195 -0
@@ -0,0 +1,200 @@
1
+ require 'isimud'
2
+ require 'thread'
3
+
4
+ module Isimud
5
+ class EventListener
6
+ include Logging
7
+ attr_reader :error_count, :error_interval, :error_limit, :name, :queues, :events_exchange, :models_exchange,
8
+ :running
9
+
10
+ DEFAULT_ERROR_LIMIT = 10
11
+ DEFAULT_ERROR_INTERVAL = 3600
12
+
13
+ DEFAULT_EVENTS_EXCHANGE = 'events'
14
+ DEFAULT_MODELS_EXCHANGE = 'models'
15
+
16
+ def initialize(options = {})
17
+ default_options = {
18
+ error_limit: Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT,
19
+ error_interval: DEFAULT_ERROR_INTERVAL,
20
+ events_exchange: Isimud.events_exchange || DEFAULT_EVENTS_EXCHANGE,
21
+ models_exchange: Isimud.model_watcher_exchange || DEFAULT_MODELS_EXCHANGE,
22
+ name: "#{Rails.application.class.parent_name.downcase}-listener"
23
+ }
24
+ options.reverse_merge!(default_options)
25
+ @error_count = 0
26
+ @observers = Hash.new
27
+ @observed_models = Set.new
28
+ @error_limit = options[:error_limit]
29
+ @error_interval = options[:error_interval]
30
+ @events_exchange = options[:events_exchange]
31
+ @models_exchange = options[:models_exchange]
32
+ @name = options[:name]
33
+ @observer_mutex = Mutex.new
34
+ @error_counter_mutex = Mutex.new
35
+ @running = false
36
+ end
37
+
38
+ def max_errors
39
+ Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT
40
+ end
41
+
42
+ def test_env?
43
+ ['cucumber', 'test'].include?(Rails.env)
44
+ end
45
+
46
+ def run
47
+ @running = true
48
+ bind_queues and return if test_env?
49
+ start_shutdown_thread
50
+ start_error_counter_thread
51
+ client.on_exception do |e|
52
+ count_error(e)
53
+ end
54
+ client.connect
55
+ start_event_thread
56
+
57
+ puts 'EventListener started. Hit Ctrl-C to exit'
58
+ Thread.stop
59
+ puts 'Main thread wakeup - exiting.'
60
+ client.close
61
+ end
62
+
63
+ # Override this method to set up message observers
64
+ def bind_queues
65
+ Isimud::EventObserver.observed_models.each do |model_class|
66
+ log "EventListener: registering observers for #{model_class}"
67
+ register_observer_class(model_class)
68
+ count = 0
69
+ model_class.find_active_observers.each do |model|
70
+ register_observer(model)
71
+ count += 1
72
+ end
73
+ log "EventListener: registered #{count} observers for #{model_class}"
74
+ end
75
+ end
76
+
77
+ def has_observer?(observer)
78
+ @observers.has_key?(observer_key_for(observer.class, observer.id))
79
+ end
80
+
81
+ private
82
+
83
+ def start_shutdown_thread
84
+ shutdown_thread = Thread.new do
85
+ Thread.stop # wait until we get a TERM or INT signal.
86
+ log 'EventListener: shutdown requested. Shutting down AMQP...', :info
87
+ @running = false
88
+ Thread.main.run
89
+ end
90
+ %w(SIGINT SIGTERM).each { |sig| trap(sig) { shutdown_thread.wakeup } }
91
+ end
92
+
93
+ def client
94
+ Isimud.client
95
+ end
96
+
97
+ def start_event_thread
98
+ Thread.new do
99
+ log 'EventListener: starting event_thread'
100
+ while @running
101
+ begin
102
+ bind_queues
103
+ Rails.logger.info 'EventListener: event_thread finished'
104
+ Thread.stop
105
+ rescue Bunny::Exception => e
106
+ count_error(e)
107
+ Rails.logger.warn 'EventListener: resetting queues'
108
+ client.reset
109
+ end
110
+ end
111
+ end
112
+ end
113
+
114
+ def count_error(exception)
115
+ @error_counter_mutex.synchronize do
116
+ @error_count += 1
117
+ log "EventListener#count_error count = #{@error_count} limit=#{error_limit}", :warn
118
+ if (@error_count >= error_limit)
119
+ log 'EventListener: too many errors, exiting', :fatal
120
+ @running = false
121
+ Thread.main.run unless test_env?
122
+ end
123
+ end
124
+ end
125
+
126
+ # start an error counter thread that clears the error count once per hour
127
+ def start_error_counter_thread
128
+ log 'EventListener: starting error counter'
129
+ @error_count = 0
130
+ Thread.new do
131
+ while true
132
+ sleep(error_interval)
133
+ @error_counter_mutex.synchronize do
134
+ log('EventListener: resetting error counter') if @error_count > 0
135
+ @error_count = 0
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ def handle_observer_event(payload)
142
+ event = JSON.parse(payload).with_indifferent_access
143
+ log "EventListener: received observer model message: #{payload.inspect}"
144
+ if %w(update destroy).include?(event[:action])
145
+ unregister_observer(event[:type], event[:id])
146
+ end
147
+ unless event[:action] == 'destroy'
148
+ observer = event[:type].constantize.find(event[:id])
149
+ register_observer(observer) if observer.enable_listener?
150
+ end
151
+ end
152
+
153
+ # Create and bind a queue for the observer. Also ensure that we are listening for observer class update events
154
+ def register_observer(observer)
155
+ @observer_mutex.synchronize do
156
+ log "EventListener: registering observer #{observer.class} #{observer.id}"
157
+ observer.observe_events(client, events_exchange)
158
+ @observers[observer_key_for(observer.class, observer.id)] = observer
159
+ end
160
+ end
161
+
162
+ # Delete a queue for an observer. This also purges all messages associated with it
163
+ def unregister_observer(observer_class, observer_id)
164
+ @observer_mutex.synchronize do
165
+ if (observer = @observers.delete(observer_key_for(observer_class, observer_id)))
166
+ begin
167
+ log "EventListener: unregistering #{observer.class} #{observer.id}"
168
+ queue_name = observer_class.constantize.event_queue_name(observer_id)
169
+ client.delete_queue(queue_name)
170
+ rescue => e
171
+ log "EventListener: error unregistering #{observer_class} #{observer_id}: #{e.message}", :warn
172
+ end
173
+ end
174
+ end
175
+ end
176
+
177
+ # Create or return the observer queue which listens for ModelWatcher events
178
+ def observer_queue
179
+ @observer_queue ||= client.create_queue("#{name}.listener.#{Process.pid}", models_exchange,
180
+ queue_options: {exclusive: true},
181
+ subscribe_options: {manual_ack: true}, &method(:handle_observer_event))
182
+ end
183
+
184
+ # Register the observer class watcher
185
+ def register_observer_class(observer_class)
186
+ @observer_mutex.synchronize do
187
+ return if @observed_models.include?(observer_class)
188
+ @observed_models << observer_class
189
+ log "EventListener: registering observer class #{observer_class}"
190
+ observer_queue.bind(models_exchange, routing_key: "#{Isimud.model_watcher_schema}.#{observer_class.base_class.name}.*")
191
+ end
192
+ end
193
+
194
+ def observer_key_for(type, id)
195
+ [type.to_s, id.to_s].join(':')
196
+ end
197
+ end
198
+ end
199
+
200
+
@@ -0,0 +1,81 @@
1
+ require 'active_record'
2
+ require 'active_support'
3
+ require 'active_support/concern'
4
+ require 'active_support/core_ext/module/attribute_accessors'
5
+
6
+ # Module for attaching and listening to events
7
+ module Isimud
8
+ module EventObserver
9
+ extend ::ActiveSupport::Concern
10
+ include Isimud::Logging
11
+
12
+ mattr_accessor :observed_models do
13
+ Array.new
14
+ end
15
+
16
+ mattr_accessor :observed_mutex do
17
+ Mutex.new
18
+ end
19
+
20
+ # Event handling hook. Override in your class.
21
+ def handle_event(event)
22
+ Rails.logger.warn("Isimud::EventObserver#handle_event not implemented for #{event_queue_name}")
23
+ end
24
+
25
+ # Routing keys that are bound to the event queue. Override in your subclass
26
+ def routing_keys
27
+ []
28
+ end
29
+
30
+ # Returns true if this instance is enabled for listening to events. Override in your subclass.
31
+ def enable_listener?
32
+ true
33
+ end
34
+
35
+ # Exchange used for listening to events. Override in your subclass if you want to specify an alternative exchange for
36
+ # events. Otherwise
37
+ def observed_exchange
38
+ nil
39
+ end
40
+
41
+ # Create or attach to a queue on the specified exchange. When an event message that matches the observer's routing keys
42
+ # is received, parse the event and call handle_event on same.
43
+ def observe_events(client, default_exchange)
44
+ client.bind(self.class.event_queue_name(id), observed_exchange || default_exchange, *routing_keys) do |message|
45
+ event = Event.parse(message)
46
+ handle_event(event)
47
+ end
48
+ end
49
+
50
+ module ClassMethods
51
+ # Method used to retrieve active observers. Override in your EventObserver class
52
+ def find_active_observers
53
+ []
54
+ end
55
+
56
+ def queue_prefix
57
+ Rails.application.class.parent_name.downcase
58
+ end
59
+
60
+ def event_queue_name(id)
61
+ [queue_prefix, base_class.name.underscore, id].join('.')
62
+ end
63
+
64
+ protected
65
+
66
+ def register_class
67
+ Isimud::EventObserver.observed_mutex.synchronize do
68
+ unless Isimud::EventObserver.observed_models.include?(self.base_class)
69
+ Rails.logger.info("Isimud::EventObserver: registering #{self.base_class}")
70
+ Isimud::EventObserver.observed_models << self.base_class
71
+ end
72
+ end
73
+ end
74
+ end
75
+
76
+ included do
77
+ include Isimud::ModelWatcher unless self.include?(Isimud::ModelWatcher)
78
+ register_class
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,11 @@
1
+ module Isimud
2
+ module Logging
3
+ def log(message, level = Isimud.log_level)
4
+ logger.send (level || :debug).to_sym , message
5
+ end
6
+
7
+ def logger
8
+ Isimud.logger
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,144 @@
1
+ require 'active_record'
2
+ require 'active_support'
3
+ require 'active_support/concern'
4
+ require 'active_support/core_ext/module/attribute_accessors'
5
+
6
+ module Isimud
7
+ # ActiveModel mixin for sending model updates to a message server.
8
+ module ModelWatcher
9
+ extend ::ActiveSupport::Concern
10
+ include Isimud::Logging
11
+
12
+ mattr_accessor :watched_models
13
+
14
+ DEFAULT_EXCHANGE = 'models'
15
+ IGNORED_COLUMNS = %w{id}
16
+
17
+ included do
18
+ ModelWatcher.watched_models ||= Array.new
19
+ ModelWatcher.watched_models << self.name
20
+ cattr_accessor :isimud_watch_attributes
21
+ cattr_accessor :sync_includes
22
+
23
+ after_commit :isimud_notify_created, on: :create
24
+ after_commit :isimud_notify_updated, on: :update
25
+ after_commit :isimud_notify_destroyed, on: :destroy
26
+ end
27
+
28
+ module ClassMethods
29
+ # Set attributes to observe and include in messages. Any property method with a return value may be included
30
+ # in the list of attributes.
31
+ # @param [Array<String,Symbol>] attributes list of attributes / properties
32
+ def watch_attributes(*attributes)
33
+ self.isimud_watch_attributes = attributes.flatten.map(&:to_s) if attributes.present?
34
+ end
35
+
36
+ # Include the following tables when fetching records for synchronization
37
+ def sync_include(_sync_includes)
38
+ self.sync_includes = _sync_includes
39
+ end
40
+
41
+ # Synchronize instances of this model with the data warehouse. This is accomplished by calling
42
+ # isimud_notify_updated() on each instance fetched from the database.
43
+ # @param [Hash] options synchronize options
44
+ # @option options [ActiveRecord::Relation] :where where_clause filter for limiting records to sync. By default, all records are synchronized.
45
+ # @option options [IO] :output optional stream for writing progress. A '.' is printed for every 100 records synchronized.
46
+ # @return [Integer] number of records synchronized
47
+ def synchronize(options = {})
48
+ where_clause = options[:where] || {}
49
+ output = options[:output] || nil
50
+ count = 0
51
+ query = self.where(where_clause)
52
+ query = query.includes(sync_includes) if sync_includes
53
+ query.find_each do |m|
54
+ next unless m.isimud_synchronize?
55
+ begin
56
+ m.isimud_sync
57
+ rescue Bunny::ClientTimeout, Timeout::Error => e
58
+ output && output.print("\n#{e}, sleeping for 10 seconds")
59
+ sleep(10)
60
+ m.isimud_sync
61
+ end
62
+ if (count += 1) % 100 == 0
63
+ output && output.print('.')
64
+ end
65
+ if (count % 1000) == 0
66
+ GC.start
67
+ end
68
+ end
69
+ count
70
+ end
71
+
72
+ def isimud_model_watcher_type
73
+ (respond_to?(:base_class) ? base_class.name : name).demodulize
74
+ end
75
+ end
76
+
77
+ # Override to set conditions for synchronizing this instance with the server (default is always)
78
+ def isimud_synchronize?
79
+ true
80
+ end
81
+
82
+ def isimud_sync
83
+ isimud_send_action_message(:update)
84
+ end
85
+
86
+ protected
87
+
88
+ def isimud_notify_created
89
+ isimud_send_action_message(:create)
90
+ end
91
+
92
+ def isimud_notify_updated
93
+ changed_attrs = previous_changes.keys
94
+ attributes = isimud_watch_attributes || isimud_default_attributes
95
+ isimud_send_action_message(:update) if (changed_attrs & attributes).any?
96
+ end
97
+
98
+ def isimud_notify_destroyed
99
+ isimud_send_action_message(:destroy)
100
+ end
101
+
102
+ def isimud_default_attributes
103
+ self.class.column_names - IGNORED_COLUMNS
104
+ end
105
+
106
+ def isimud_attribute_data
107
+ attributes = isimud_watch_attributes || isimud_default_attributes
108
+ attributes.inject(Hash.new) { |hsh, attr| hsh[attr] = send(attr); hsh }
109
+ end
110
+
111
+ def isimud_model_watcher_schema
112
+ Isimud.model_watcher_schema || if defined?(Rails)
113
+ Rails.configuration.database_configuration[Rails.env]['database']
114
+ end
115
+ end
116
+
117
+ def isimud_model_watcher_exchange
118
+ Isimud.model_watcher_exchange
119
+ end
120
+
121
+ def isimud_model_watcher_type
122
+ self.class.isimud_model_watcher_type
123
+ end
124
+
125
+ def isimud_model_watcher_routing_key(action)
126
+ [isimud_model_watcher_schema, isimud_model_watcher_type, action].join('.')
127
+ end
128
+
129
+ def isimud_send_action_message(action)
130
+ return unless Isimud.model_watcher_enabled? && isimud_synchronize?
131
+ payload = {
132
+ schema: isimud_model_watcher_schema,
133
+ type: isimud_model_watcher_type,
134
+ action: action,
135
+ id: id,
136
+ timestamp: (updated_at || Time.now).utc
137
+ }
138
+ payload[:attributes] = isimud_attribute_data unless action == :destroy
139
+ routing_key = isimud_model_watcher_routing_key(action)
140
+ log "Isimud::ModelWatcher#publish: exchange #{isimud_model_watcher_exchange} routing_key #{routing_key} payload #{payload.inspect}"
141
+ Isimud.client.publish(isimud_model_watcher_exchange, routing_key, payload.to_json)
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,9 @@
1
+ require 'rails'
2
+
3
+ module Isimud
4
+ class Railtie < Rails::Railtie
5
+ generators do
6
+ require 'isimud'
7
+ end
8
+ end
9
+ end