promiscuous 0.25 → 0.31.0

data/lib/promiscuous/subscriber/worker/message.rb

@@ -0,0 +1,46 @@
+class Promiscuous::Subscriber::Worker::Message
+  attr_accessor :worker, :metadata, :payload, :parsed_payload
+
+  def initialize(worker, metadata, payload)
+    self.worker = worker
+    self.metadata = metadata
+    self.payload = payload
+  end
+
+  def parsed_payload
+    @parsed_payload ||= JSON.parse(payload)
+  end
+
+  def queue_name
+    parsed_payload['__amqp__']
+  end
+
+  def has_version?
+    !!version
+  end
+
+  def version
+    @version ||= parsed_payload['version'].try(:symbolize_keys)
+  end
+
+  def update_dependencies
+    global_key = Promiscuous::Redis.sub_key('global')
+    global_version = Promiscuous::Redis.incr global_key
+    Promiscuous::Redis.publish(global_key, global_version)
+  end
+
+  def process
+    return if worker.stopped?
+
+    Promiscuous.debug "[receive] #{payload}"
+
+    worker.unit_of_work(queue_name) { Promiscuous::Subscriber.process(parsed_payload) }
+
+    update_dependencies
+    metadata.ack
+  rescue Exception => e
+    e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
+    Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.try(:call, e)
+  end
+end

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -0,0 +1,168 @@
+class Promiscuous::Subscriber::Worker::MessageSynchronizer
+  include Celluloid::IO
+
+  attr_accessor :worker, :redis
+
+  def initialize(worker)
+    self.worker = worker
+    @subscriptions = {}
+  end
+
+  def resume
+    self.redis = Promiscuous::Redis.new_celluloid_connection
+    main_loop!
+  end
+
+  def stop
+    terminate
+  end
+
+  def finalize
+    self.redis.client.connection.disconnect if self.redis
+  rescue
+  end
+
+  def main_loop
+    loop do
+      reply = redis.client.read
+      raise reply if reply.is_a?(Redis::CommandError)
+      type, subscription, arg = reply
+
+      case type
+      when 'subscribe'
+        find_subscription(subscription).finalize_subscription
+      when 'unsubscribe'
+      when 'message'
+        find_subscription(subscription).maybe_perform_callbacks(arg)
+      end
+    end
+  rescue Celluloid::Task::TerminatedError
+  rescue Exception => e
+    Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
+
+    e = Promiscuous::Error::Connection.new(:redis, 'Lost connection')
+    Promiscuous::Worker.stop
+    Promiscuous::Config.error_notifier.try(:call, e)
+  end
+
+  # process_when_ready() is called by the AMQP pump. This is what happens:
+  # 1. First, we subscribe to redis and wait for the confirmation.
+  # 2. Then we check if the version in redis is old enough to process the message.
+  #    If not we bail out and rely on the subscription to kick the processing.
+  # Because we subscribed in advanced, we will not miss the notification, but
+  # extra care needs to be taken to avoid processing the message twice (see
+  # perform()).
+  def process_when_ready(msg)
+    return worker.runners.process!(msg) unless msg.has_version?
+
+    on_version Promiscuous::Redis.sub_key('global'), msg.version[:global] do
+      worker.runners.process!(msg)
+    end
+  end
+
+  def on_version(key, version, &callback)
+    return unless @subscriptions
+    cb = Subscription::Callback.new(version, callback)
+    get_subscription(key).subscribe.add_callback(version, cb)
+    cb.maybe_perform(Promiscuous::Redis.get(key))
+  end
+
+  # state_lock must be taken before calling find_subscription()
+  def find_subscription(key)
+    raise "Fatal error (redis sub)" unless @subscriptions[key]
+    @subscriptions[key]
+  end
+
+  # state_lock must be taken before calling find_subscription()
+  def get_subscription(key)
+    @subscriptions[key] ||= Subscription.new(self, key)
+  end
+
+  class Subscription
+    attr_accessor :parent, :key
+
+    def initialize(parent, key)
+      self.parent = parent
+      self.key = key
+
+      @subscription_requested = false
+      @subscribed_to_redis = false
+      @callbacks = {}
+    end
+
+    # subscribe() is called with the state_lock of the parent held
+    def subscribe
+      request_subscription
+
+      loop do
+        break if @subscribed_to_redis
+        parent.wait :subscription
+      end
+      self
+    end
+
+    def request_subscription
+      # We will not send two subscription requests, since we are holding
+      # the state_lock of the parent.
+      return if @subscription_requested
+      parent.redis.client.process([[:subscribe, key]])
+      @subscription_requested = true
+    end
+
+    def finalize_subscription
+      @subscribed_to_redis = true
+      parent.signal :subscription
+    end
+
+    def destroy
+      # TODO parent.redis_client_call(:unsubscribe, key)
+    end
+
+    def add_callback(version, callback)
+      callback.subscription = self
+      @callbacks[callback.token] = callback
+    end
+
+    def remove_callback(token)
+      !!@callbacks.delete(token)
+      # TODO unsubscribe after a while?
+    end
+
+    def maybe_perform_callbacks(current_version)
+      @callbacks.values.each do |cb|
+        cb.maybe_perform(current_version)
+      end
+    end
+
+    class Callback
+      # Tokens are only used so that the callback can find and remove itself
+      # in the callback list.
+      @next_token = 0
+      def self.get_next_token
+        @next_token += 1
+      end
+
+      attr_accessor :subscription, :version, :callback, :token
+
+      def initialize(version, callback)
+        self.version = version
+        self.callback = callback
+        @token = self.class.get_next_token
+      end
+
+      def can_perform?(current_version)
+        current_version.to_i + 1 >= self.version
+      end
+
+      def perform
+        # removing the callback can happen only once, ensuring that the
+        # callback is called at most once.
+        callback.call if subscription.remove_callback(@token)
+      end
+
+      def maybe_perform(current_version)
+        perform if can_perform?(current_version)
+      end
+    end
+  end
+end

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -0,0 +1,47 @@
+class Promiscuous::Subscriber::Worker::Pump
+  attr_accessor :worker
+
+  def initialize(worker)
+    self.worker = worker
+  end
+
+  def resume
+    if @queue
+      # XXX TODO we should not access to the channel like this.
+      # The abstraction is leaking.
+      # Actually, we actually want one channel per worker.
+
+      # The following tells rabbitmq to resend the unacked messages
+      Promiscuous::AMQP::RubyAMQP.channel.recover
+    else
+      Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
+        @queue = queue
+        @queue.subscribe({:ack => true}, &method(:process_payload))
+      end
+    end
+  end
+
+  def stop
+    # we should tell amqp that we want to stop using the queue
+  end
+
+  def process_payload(metadata, payload)
+    return if worker.stopped?
+
+    msg = Promiscuous::Subscriber::Worker::Message.new(worker, metadata, payload)
+    worker.message_synchronizer.process_when_ready(msg)
+  end
+
+  def queue_bindings
+    queue_name = "#{Promiscuous::Config.app}.promiscuous"
+    exchange_name = Promiscuous::AMQP::EXCHANGE
+
+    if worker.options[:personality]
+      queue_name    += ".#{worker.options[:personality]}"
+      exchange_name += ".#{worker.options[:personality]}"
+    end
+
+    bindings = Promiscuous::Subscriber::AMQP.subscribers.keys
+    {:exchange_name => exchange_name, :queue_name => queue_name, :bindings => bindings}
+  end
+end

data/lib/promiscuous/subscriber/worker/runner.rb

@@ -0,0 +1,7 @@
+class Promiscuous::Subscriber::Worker::Runner
+  include Celluloid
+
+  def process(msg)
+    msg.process
+  end
+end

data/lib/promiscuous/subscriber/worker.rb

@@ -1,62 +1,54 @@
 class Promiscuous::Subscriber::Worker
-  include Promiscuous::Common::Worker
+  require 'celluloid'
+  require 'celluloid/io'
 
-  def resume
-    if @queue
-      # XXX TODO we should not access to the channel like this.
-      # The abstraction is leaking.
-      # Actually, we actually want one channel per worker.
+  extend Promiscuous::Autoload
+  autoload :Message, :Pump, :MessageSynchronizer, :Runner
 
-      # The following tells rabbitmq to resend the unacked messages
-      Promiscuous::AMQP::RubyAMQP.channel.recover
-    else
-      Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
-        @queue = queue
-        @queue.subscribe({:ack => true}, &method(:process_payload))
-      end
-    end
+  attr_accessor :options, :stopped, :pump, :message_synchronizer, :runners
+  alias_method :stopped?, :stopped
 
-    super
-  end
+  def initialize(options={})
+    Celluloid.exception_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
 
-  def process_payload(metadata, payload)
-    return if self.stopped?
+    options[:personality] = 'new'
 
-    # Note: This code always runs on the root Fiber,
-    # so ordering is always preserved
+    self.options = options
+    self.stopped = true
 
-    Promiscuous.debug "[receive] #{payload}"
-    parsed_payload = JSON.parse(payload)
-    queue = parsed_payload['__amqp__']
+    self.pump = Pump.new(self)
+  end
 
-    self.unit_of_work(queue) { Promiscuous::Subscriber.process(parsed_payload) }
+  def resume
+    return unless self.stopped
+    self.stopped = false
+    self.runners = Runner.pool
+    self.message_synchronizer = MessageSynchronizer.new(self)
+    self.message_synchronizer.resume
+    self.pump.resume
+  end
 
-    metadata.ack
-    made_progress
-  rescue Exception => e
-    e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
+  def stop
+    return if self.stopped
+    self.pump.stop
+    self.message_synchronizer.stop rescue Celluloid::Task::TerminatedError
+    self.message_synchronizer = nil
+    self.runners.terminate
+    self.runners = nil
+    self.stopped = true
+  end
 
-    if bareback?
-      Promiscuous.error "[receive] (bareback, don't care) #{e} #{e.backtrace.join("\n")}"
-      metadata.ack
+  def unit_of_work(type, &block)
+    # type is used by the new relic agent, by monkey patching.
+    # middleware?
+    if defined?(Mongoid)
+      Mongoid.unit_of_work { yield }
     else
-      retry_msg = stop_for_a_while(e)
-      Promiscuous.warn "[receive] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
+      yield
     end
-
-    Promiscuous::Config.error_notifier.try(:call, e)
-  end
-
-  def queue_bindings
-    queue_name = "#{Promiscuous::Config.app}.promiscuous"
-    exchange_name = Promiscuous::AMQP::EXCHANGE
-
-    if options[:personality]
-      queue_name    += ".#{options[:personality]}"
-      exchange_name += ".#{options[:personality]}"
+  ensure
+    if defined?(ActiveRecord)
+      ActiveRecord::Base.clear_active_connections!
     end
-
-    bindings = Promiscuous::Subscriber::AMQP.subscribers.keys
-    {:exchange_name => exchange_name, :queue_name => queue_name, :bindings => bindings}
   end
 end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.25'
+  VERSION = '0.31.0'
 end

data/lib/promiscuous/worker.rb

@@ -3,11 +3,7 @@ module Promiscuous::Worker
   self.workers = []
 
   def self.replicate(options={})
-    options[:action] ||= [:publish, :subscribe]
-    actions = [options[:action]].flatten
-
-    self.workers <<  Promiscuous::Publisher::Worker.new(options).tap { |w| w.resume } if :publish.in? actions
-    self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.resume } if :subscribe.in? actions
+    self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.resume }
   end
 
   def self.kill

data/lib/promiscuous.rb

@@ -6,7 +6,7 @@ module Promiscuous
 
   extend Promiscuous::Autoload
   autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,
-           :CLI, :Error, :Loader, :AMQP, :Config
+           :CLI, :Error, :Loader, :AMQP, :Redis, :Config
 
   class << self
     def configure(&block)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: '0.25'
+  version: 0.31.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-12 00:00:00.000000000 Z
+date: 2013-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,70 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: crowdtap_redis_lock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid-io
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Replicate your Mongoid/ActiveRecord models across your applications
 email:
 - nicolas@viennot.biz
@@ -117,36 +181,34 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/promiscuous/amqp/null.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/ruby_amqp.rb
+- lib/promiscuous/amqp/null.rb
 - lib/promiscuous/common/lint/base.rb
 - lib/promiscuous/common/class_helpers.rb
 - lib/promiscuous/common/options.rb
-- lib/promiscuous/common/worker.rb
 - lib/promiscuous/common/lint.rb
 - lib/promiscuous/publisher/lint/amqp.rb
 - lib/promiscuous/publisher/lint/attributes.rb
 - lib/promiscuous/publisher/lint/base.rb
 - lib/promiscuous/publisher/lint/polymorphic.rb
 - lib/promiscuous/publisher/lint/class.rb
-- lib/promiscuous/publisher/mongoid/embedded.rb
-- lib/promiscuous/publisher/mongoid/defer_embedded.rb
 - lib/promiscuous/publisher/mongoid/embedded_many.rb
-- lib/promiscuous/publisher/mongoid/defer.rb
+- lib/promiscuous/publisher/mongoid/embedded.rb
+- lib/promiscuous/publisher/model/active_record.rb
+- lib/promiscuous/publisher/model/mongoid.rb
 - lib/promiscuous/publisher/envelope.rb
 - lib/promiscuous/publisher/polymorphic.rb
 - lib/promiscuous/publisher/mock.rb
+- lib/promiscuous/publisher/class.rb
+- lib/promiscuous/publisher/ephemeral.rb
 - lib/promiscuous/publisher/attributes.rb
-- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/active_record.rb
 - lib/promiscuous/publisher/base.rb
-- lib/promiscuous/publisher/class.rb
-- lib/promiscuous/publisher/ephemeral.rb
-- lib/promiscuous/publisher/amqp.rb
-- lib/promiscuous/publisher/worker.rb
-- lib/promiscuous/publisher/mongoid.rb
 - lib/promiscuous/publisher/lint.rb
+- lib/promiscuous/publisher/mongoid.rb
+- lib/promiscuous/publisher/amqp.rb
+- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/subscriber/lint/amqp.rb
 - lib/promiscuous/subscriber/lint/base.rb
 - lib/promiscuous/subscriber/lint/class.rb
@@ -154,7 +216,10 @@ files:
 - lib/promiscuous/subscriber/lint/attributes.rb
 - lib/promiscuous/subscriber/mongoid/embedded_many.rb
 - lib/promiscuous/subscriber/mongoid/embedded.rb
-- lib/promiscuous/subscriber/mongoid/versioning.rb
+- lib/promiscuous/subscriber/worker/message.rb
+- lib/promiscuous/subscriber/worker/runner.rb
+- lib/promiscuous/subscriber/worker/pump.rb
+- lib/promiscuous/subscriber/worker/message_synchronizer.rb
 - lib/promiscuous/subscriber/active_record.rb
 - lib/promiscuous/subscriber/envelope.rb
 - lib/promiscuous/subscriber/upsert.rb
@@ -165,25 +230,26 @@ files:
 - lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/base.rb
 - lib/promiscuous/subscriber/class.rb
-- lib/promiscuous/subscriber/mongoid.rb
 - lib/promiscuous/subscriber/lint.rb
+- lib/promiscuous/subscriber/mongoid.rb
 - lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/error/subscriber.rb
 - lib/promiscuous/error/connection.rb
 - lib/promiscuous/error/publisher.rb
-- lib/promiscuous/error/subscriber.rb
 - lib/promiscuous/observer.rb
 - lib/promiscuous/ephemeral.rb
-- lib/promiscuous/worker.rb
 - lib/promiscuous/autoload.rb
 - lib/promiscuous/subscriber.rb
-- lib/promiscuous/publisher.rb
 - lib/promiscuous/error.rb
-- lib/promiscuous/common.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/loader.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/cli.rb
+- lib/promiscuous/common.rb
+- lib/promiscuous/publisher.rb
+- lib/promiscuous/worker.rb
+- lib/promiscuous/amqp.rb
 - lib/promiscuous/config.rb
+- lib/promiscuous/redis.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous

data/lib/promiscuous/common/worker.rb

@@ -1,52 +0,0 @@
-module Promiscuous::Common::Worker
-  extend ActiveSupport::Concern
-
-  def initialize(options={})
-    self.options = options
-    self.stopped = true
-    made_progress
-  end
-
-  def stop
-    self.stopped = true
-  end
-
-  def resume
-    self.stopped = false
-  end
-
-  def unit_of_work(type)
-    # type is used by the new relic agent, by monkey patching.
-    # middleware?
-    if defined?(Mongoid)
-      Mongoid.unit_of_work { yield }
-    else
-      yield
-    end
-  end
-
-  def bareback?
-    !!options[:bareback]
-  end
-
-  def stop_for_a_while(reason)
-    stop
-    self.retry_timeout = self.retry_timeout * 2
-
-    if reason.inner.is_a? Promiscuous::Error::Connection
-      "will retry when the amqp connection comes back"
-    else
-      EM::Timer.new(self.retry_timeout) { resume }
-      "retrying in #{self.retry_timeout}s"
-    end
-  end
-
-  def made_progress
-    self.retry_timeout = 1
-  end
-
-  included do
-    attr_accessor :stopped, :options, :retry_timeout
-    alias_method :stopped?, :stopped
-  end
-end