isimud 0.5.2

data/lib/isimud/event_listener.rb

@@ -0,0 +1,200 @@
+require 'isimud'
+require 'thread'
+
+module Isimud
+  class EventListener
+    include Logging
+    attr_reader :error_count, :error_interval, :error_limit, :name, :queues, :events_exchange, :models_exchange,
+                :running
+
+    DEFAULT_ERROR_LIMIT    = 10
+    DEFAULT_ERROR_INTERVAL = 3600
+
+    DEFAULT_EVENTS_EXCHANGE = 'events'
+    DEFAULT_MODELS_EXCHANGE = 'models'
+
+    def initialize(options = {})
+      default_options = {
+          error_limit:     Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT,
+          error_interval:  DEFAULT_ERROR_INTERVAL,
+          events_exchange: Isimud.events_exchange || DEFAULT_EVENTS_EXCHANGE,
+          models_exchange: Isimud.model_watcher_exchange || DEFAULT_MODELS_EXCHANGE,
+          name:            "#{Rails.application.class.parent_name.downcase}-listener"
+      }
+      options.reverse_merge!(default_options)
+      @error_count         = 0
+      @observers           = Hash.new
+      @observed_models     = Set.new
+      @error_limit         = options[:error_limit]
+      @error_interval      = options[:error_interval]
+      @events_exchange     = options[:events_exchange]
+      @models_exchange     = options[:models_exchange]
+      @name                = options[:name]
+      @observer_mutex      = Mutex.new
+      @error_counter_mutex = Mutex.new
+      @running             = false
+    end
+
+    def max_errors
+      Isimud.listener_error_limit || DEFAULT_ERROR_LIMIT
+    end
+
+    def test_env?
+      ['cucumber', 'test'].include?(Rails.env)
+    end
+
+    def run
+      @running = true
+      bind_queues and return if test_env?
+      start_shutdown_thread
+      start_error_counter_thread
+      client.on_exception do |e|
+        count_error(e)
+      end
+      client.connect
+      start_event_thread
+
+      puts 'EventListener started. Hit Ctrl-C to exit'
+      Thread.stop
+      puts 'Main thread wakeup - exiting.'
+      client.close
+    end
+
+    # Override this method to set up message observers
+    def bind_queues
+      Isimud::EventObserver.observed_models.each do |model_class|
+        log "EventListener: registering observers for #{model_class}"
+        register_observer_class(model_class)
+        count = 0
+        model_class.find_active_observers.each do |model|
+          register_observer(model)
+          count += 1
+        end
+        log "EventListener: registered #{count} observers for #{model_class}"
+      end
+    end
+
+    def has_observer?(observer)
+      @observers.has_key?(observer_key_for(observer.class, observer.id))
+    end
+
+    private
+
+    def start_shutdown_thread
+      shutdown_thread = Thread.new do
+        Thread.stop # wait until we get a TERM or INT signal.
+        log 'EventListener: shutdown requested.  Shutting down AMQP...', :info
+        @running = false
+        Thread.main.run
+      end
+      %w(SIGINT SIGTERM).each { |sig| trap(sig) { shutdown_thread.wakeup } }
+    end
+
+    def client
+      Isimud.client
+    end
+
+    def start_event_thread
+      Thread.new do
+        log 'EventListener: starting event_thread'
+        while @running
+          begin
+            bind_queues
+            Rails.logger.info 'EventListener: event_thread finished'
+            Thread.stop
+          rescue Bunny::Exception => e
+            count_error(e)
+            Rails.logger.warn 'EventListener: resetting queues'
+            client.reset
+          end
+        end
+      end
+    end
+
+    def count_error(exception)
+      @error_counter_mutex.synchronize do
+        @error_count += 1
+        log "EventListener#count_error count = #{@error_count} limit=#{error_limit}", :warn
+        if (@error_count >= error_limit)
+          log 'EventListener: too many errors, exiting', :fatal
+          @running = false
+          Thread.main.run unless test_env?
+        end
+      end
+    end
+
+    # start an error counter thread that clears the error count once per hour
+    def start_error_counter_thread
+      log 'EventListener: starting error counter'
+      @error_count = 0
+      Thread.new do
+        while true
+          sleep(error_interval)
+          @error_counter_mutex.synchronize do
+            log('EventListener: resetting error counter') if @error_count > 0
+            @error_count = 0
+          end
+        end
+      end
+    end
+
+    def handle_observer_event(payload)
+      event = JSON.parse(payload).with_indifferent_access
+      log "EventListener: received observer model message: #{payload.inspect}"
+      if %w(update destroy).include?(event[:action])
+        unregister_observer(event[:type], event[:id])
+      end
+      unless event[:action] == 'destroy'
+        observer = event[:type].constantize.find(event[:id])
+        register_observer(observer) if observer.enable_listener?
+      end
+    end
+
+    # Create and bind a queue for the observer. Also ensure that we are listening for observer class update events
+    def register_observer(observer)
+      @observer_mutex.synchronize do
+        log "EventListener: registering observer #{observer.class} #{observer.id}"
+        observer.observe_events(client, events_exchange)
+        @observers[observer_key_for(observer.class, observer.id)] = observer
+      end
+    end
+
+    # Delete a queue for an observer. This also purges all messages associated with it
+    def unregister_observer(observer_class, observer_id)
+      @observer_mutex.synchronize do
+        if (observer = @observers.delete(observer_key_for(observer_class, observer_id)))
+          begin
+            log "EventListener: unregistering #{observer.class} #{observer.id}"
+            queue_name = observer_class.constantize.event_queue_name(observer_id)
+            client.delete_queue(queue_name)
+          rescue => e
+            log "EventListener: error unregistering #{observer_class} #{observer_id}: #{e.message}", :warn
+          end
+        end
+      end
+    end
+
+    # Create or return the observer queue which listens for ModelWatcher events
+    def observer_queue
+      @observer_queue ||= client.create_queue("#{name}.listener.#{Process.pid}", models_exchange,
+                                              queue_options:     {exclusive: true},
+                                              subscribe_options: {manual_ack: true}, &method(:handle_observer_event))
+    end
+
+    # Register the observer class watcher
+    def register_observer_class(observer_class)
+      @observer_mutex.synchronize do
+        return if @observed_models.include?(observer_class)
+        @observed_models << observer_class
+        log "EventListener: registering observer class #{observer_class}"
+        observer_queue.bind(models_exchange, routing_key: "#{Isimud.model_watcher_schema}.#{observer_class.base_class.name}.*")
+      end
+    end
+
+    def observer_key_for(type, id)
+      [type.to_s, id.to_s].join(':')
+    end
+  end
+end
+
+

data/lib/isimud/event_observer.rb

@@ -0,0 +1,81 @@
+require 'active_record'
+require 'active_support'
+require 'active_support/concern'
+require 'active_support/core_ext/module/attribute_accessors'
+
+# Module for attaching and listening to events
+module Isimud
+  module EventObserver
+    extend ::ActiveSupport::Concern
+    include Isimud::Logging
+
+    mattr_accessor :observed_models do
+      Array.new
+    end
+
+    mattr_accessor :observed_mutex do
+      Mutex.new
+    end
+
+    # Event handling hook. Override in your class.
+    def handle_event(event)
+      Rails.logger.warn("Isimud::EventObserver#handle_event not implemented for #{event_queue_name}")
+    end
+
+    # Routing keys that are bound to the event queue. Override in your subclass
+    def routing_keys
+      []
+    end
+
+    # Returns true if this instance is enabled for listening to events. Override in your subclass.
+    def enable_listener?
+      true
+    end
+
+    # Exchange used for listening to events. Override in your subclass if you want to specify an alternative exchange for
+    # events. Otherwise
+    def observed_exchange
+      nil
+    end
+
+    # Create or attach to a queue on the specified exchange. When an event message that matches the observer's routing keys
+    # is received, parse the event and call handle_event on same.
+    def observe_events(client, default_exchange)
+      client.bind(self.class.event_queue_name(id), observed_exchange || default_exchange, *routing_keys) do |message|
+        event = Event.parse(message)
+        handle_event(event)
+      end
+    end
+
+    module ClassMethods
+      # Method used to retrieve active observers. Override in your EventObserver class
+      def find_active_observers
+        []
+      end
+
+      def queue_prefix
+        Rails.application.class.parent_name.downcase
+      end
+
+      def event_queue_name(id)
+        [queue_prefix, base_class.name.underscore, id].join('.')
+      end
+
+      protected
+
+      def register_class
+        Isimud::EventObserver.observed_mutex.synchronize do
+          unless Isimud::EventObserver.observed_models.include?(self.base_class)
+            Rails.logger.info("Isimud::EventObserver: registering #{self.base_class}")
+            Isimud::EventObserver.observed_models << self.base_class
+          end
+        end
+      end
+    end
+
+    included do
+      include Isimud::ModelWatcher unless self.include?(Isimud::ModelWatcher)
+      register_class
+    end
+  end
+end

data/lib/isimud/logging.rb

@@ -0,0 +1,11 @@
+module Isimud
+  module Logging
+    def log(message, level = Isimud.log_level)
+      logger.send (level || :debug).to_sym , message
+    end
+
+    def logger
+      Isimud.logger
+    end
+  end
+end

data/lib/isimud/model_watcher.rb

@@ -0,0 +1,144 @@
+require 'active_record'
+require 'active_support'
+require 'active_support/concern'
+require 'active_support/core_ext/module/attribute_accessors'
+
+module Isimud
+  # ActiveModel mixin for sending model updates to a message server.
+  module ModelWatcher
+    extend ::ActiveSupport::Concern
+    include Isimud::Logging
+
+    mattr_accessor :watched_models
+
+    DEFAULT_EXCHANGE = 'models'
+    IGNORED_COLUMNS  = %w{id}
+
+    included do
+      ModelWatcher.watched_models ||= Array.new
+      ModelWatcher.watched_models << self.name
+      cattr_accessor :isimud_watch_attributes
+      cattr_accessor :sync_includes
+
+      after_commit :isimud_notify_created, on: :create
+      after_commit :isimud_notify_updated, on: :update
+      after_commit :isimud_notify_destroyed, on: :destroy
+    end
+
+    module ClassMethods
+      # Set attributes to observe and include in messages. Any property method with a return value may be included
+      # in the list of attributes.
+      # @param [Array<String,Symbol>] attributes list of attributes / properties
+      def watch_attributes(*attributes)
+        self.isimud_watch_attributes = attributes.flatten.map(&:to_s) if attributes.present?
+      end
+
+      # Include the following tables when fetching records for synchronization
+      def sync_include(_sync_includes)
+        self.sync_includes = _sync_includes
+      end
+
+      # Synchronize instances of this model with the data warehouse. This is accomplished by calling
+      # isimud_notify_updated() on each instance fetched from the database.
+      # @param [Hash] options synchronize options
+      # @option options [ActiveRecord::Relation] :where where_clause filter for limiting records to sync. By default, all records are synchronized.
+      # @option options [IO] :output optional stream for writing progress. A '.' is printed for every 100 records synchronized.
+      # @return [Integer] number of records synchronized
+      def synchronize(options = {})
+        where_clause = options[:where] || {}
+        output       = options[:output] || nil
+        count        = 0
+        query        = self.where(where_clause)
+        query        = query.includes(sync_includes) if sync_includes
+        query.find_each do |m|
+          next unless m.isimud_synchronize?
+          begin
+            m.isimud_sync
+          rescue Bunny::ClientTimeout, Timeout::Error => e
+            output && output.print("\n#{e}, sleeping for 10 seconds")
+            sleep(10)
+            m.isimud_sync
+          end
+          if (count += 1) % 100 == 0
+            output && output.print('.')
+          end
+          if (count % 1000) == 0
+            GC.start
+          end
+        end
+        count
+      end
+
+      def isimud_model_watcher_type
+        (respond_to?(:base_class) ? base_class.name : name).demodulize
+      end
+    end
+
+    # Override to set conditions for synchronizing this instance with the server (default is always)
+    def isimud_synchronize?
+      true
+    end
+
+    def isimud_sync
+      isimud_send_action_message(:update)
+    end
+
+    protected
+
+    def isimud_notify_created
+      isimud_send_action_message(:create)
+    end
+
+    def isimud_notify_updated
+      changed_attrs = previous_changes.keys
+      attributes    = isimud_watch_attributes || isimud_default_attributes
+      isimud_send_action_message(:update) if (changed_attrs & attributes).any?
+    end
+
+    def isimud_notify_destroyed
+      isimud_send_action_message(:destroy)
+    end
+
+    def isimud_default_attributes
+      self.class.column_names - IGNORED_COLUMNS
+    end
+
+    def isimud_attribute_data
+      attributes = isimud_watch_attributes || isimud_default_attributes
+      attributes.inject(Hash.new) { |hsh, attr| hsh[attr] = send(attr); hsh }
+    end
+
+    def isimud_model_watcher_schema
+      Isimud.model_watcher_schema || if defined?(Rails)
+                                              Rails.configuration.database_configuration[Rails.env]['database']
+                                            end
+    end
+
+    def isimud_model_watcher_exchange
+      Isimud.model_watcher_exchange
+    end
+
+    def isimud_model_watcher_type
+      self.class.isimud_model_watcher_type
+    end
+
+    def isimud_model_watcher_routing_key(action)
+      [isimud_model_watcher_schema, isimud_model_watcher_type, action].join('.')
+    end
+
+    def isimud_send_action_message(action)
+      return unless Isimud.model_watcher_enabled? && isimud_synchronize?
+      payload              = {
+          schema:    isimud_model_watcher_schema,
+          type:      isimud_model_watcher_type,
+          action:    action,
+          id:        id,
+          timestamp: (updated_at || Time.now).utc
+      }
+      payload[:attributes] = isimud_attribute_data unless action == :destroy
+      routing_key          = isimud_model_watcher_routing_key(action)
+      log "Isimud::ModelWatcher#publish: exchange #{isimud_model_watcher_exchange} routing_key #{routing_key} payload #{payload.inspect}"
+      Isimud.client.publish(isimud_model_watcher_exchange, routing_key, payload.to_json)
+    end
+  end
+end

data/lib/isimud/railtie.rb

@@ -0,0 +1,9 @@
+require 'rails'
+
+module Isimud
+  class Railtie < Rails::Railtie
+    generators do
+      require 'isimud'
+    end
+  end
+end