promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -1,50 +1,24 @@
-require 'eventmachine'
-
 class Promiscuous::Subscriber::Worker::Pump
-  include Celluloid
-
-  attr_accessor :subscribe_sync
-
-  def initialize
-    # We need to subscribe to everything to keep up with the version tracking
-    queue_name = "#{Promiscuous::Config.app}.promiscuous"
-    bindings = ['*']
-
-    unless Promiscuous::Config.backend == :rubyamqp
-      raise "you must use the ruby_amqp backend"
-    end
-    Promiscuous::AMQP.ensure_connected
-
-    @subscribe_sync = Promiscuous::AMQP::RubyAMQP::Synchronizer.new
-    Promiscuous::AMQP::RubyAMQP.get_channel(:pump) do |channel|
-      @channel = channel
-      # TODO channel.on_error ?
-
-      queue = channel.queue(queue_name, Promiscuous::Config.queue_options)
-      exchange = Promiscuous::AMQP::RubyAMQP.get_exchange(:pump)
-      bindings.each do |binding|
-        queue.bind(exchange, :routing_key => binding)
-        Promiscuous.debug "[bind] #{queue_name} -> #{binding}"
-      end
-      queue.subscribe(:ack => true, :confirm => proc { @subscribe_sync.signal }, &method(:process_payload))
-    end
-    @subscribe_sync.wait
+  def initialize(root)
+    @root = root
+    # late include of CelluloidSubscriber because the class is resolved
+    # at runtime since we can have different backends.
+    extend Promiscuous::AMQP::Subscriber
   end
 
-  def finalize
-    channel_sync = Promiscuous::AMQP::RubyAMQP::Synchronizer.new
-    Promiscuous::AMQP::RubyAMQP.close_channel(:pump) do
-      channel_sync.signal
-    end
-    channel_sync.wait
-  end
-
-  def recover
-    EM.next_tick { @channel.recover }
+  def connect
+    options = {}
+    options[:queue_name] = ENV['QUEUE_NAME'] || "#{Promiscuous::Config.app}.promiscuous"
+    # We need to subscribe to everything to keep up with the version tracking
+    options[:bindings] = ['*']
+    subscribe(options, &method(:on_message))
   end
 
-  def process_payload(metadata, payload)
-    msg = Promiscuous::Subscriber::Worker::Message.new(metadata, payload)
-    Celluloid::Actor[:message_synchronizer].process_when_ready(msg)
+  def on_message(metadata, payload)
+    msg = Promiscuous::Subscriber::Worker::Message.new(payload, :metadata => metadata, :root_worker => @root)
+    @root.message_synchronizer.process_when_ready(msg)
+  rescue Exception => e
+    Promiscuous.warn "[receive] cannot process message: #{e} #{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.try(:call, e)
   end
 end

data/lib/promiscuous/subscriber/worker/recorder.rb

@@ -0,0 +1,24 @@
+class Promiscuous::Subscriber::Worker::Recorder
+  def initialize(log_file)
+    @log_file = log_file
+    extend Promiscuous::AMQP::Subscriber
+  end
+
+  def start
+    @file = File.open(@log_file, 'a')
+    options = {}
+    options[:queue_name] = "#{Promiscuous::Config.app}.promiscuous"
+    # We need to subscribe to everything to keep up with the version tracking
+    options[:bindings] = ['*']
+
+    subscribe(options) do |metadata, payload|
+      @file.puts payload
+      metadata.ack
+    end
+  end
+
+  def stop
+    disconnect
+    @file.try(:close)
+  end
+end

data/lib/promiscuous/subscriber/worker/runner.rb

@@ -1,7 +1,28 @@
 class Promiscuous::Subscriber::Worker::Runner
-  include Celluloid
+  attr_accessor :messages_to_process
 
-  def process(msg)
-    msg.process
+  def initialize(root)
+    @root = root
+    @messages_to_process = Queue.new
+  end
+
+  def start
+    num_threads = Promiscuous::Config.subscriber_threads
+    @locks   ||= num_threads.times.map { Mutex.new }
+    @threads ||= num_threads.times.map { |i| Thread.new { main_loop(@locks[i]) } }
+  end
+
+  def stop
+    return unless @threads
+    @threads.zip(@locks).each { |thread, lock| lock.synchronize { thread.kill } }
+    @threads = @locks = nil
+    @messages_to_process.clear
+  end
+
+  def main_loop(lock)
+    loop do
+      msg = @messages_to_process.pop
+      lock.synchronize { msg.process }
+    end
   end
 end

data/lib/promiscuous/subscriber/worker/stats.rb

@@ -0,0 +1,62 @@
+class Promiscuous::Subscriber::Worker::Stats
+  def connect
+    @interval = Promiscuous::Config.stats_interval.to_f
+    return if @interval.zero? || @redis
+
+    url = Promiscuous::Config.redis_stats_url
+    @redis = ::Redis.new(:url => url)
+    key = Promiscuous::Key.new(:sub).join(Socket.gethostname)
+    @key_processed_message = key.join('__stats__', 'processed_messages').to_s
+    @key_total_response_time = key.join('__stats__', 'total_response_time').to_s
+
+    @redis.set(@key_processed_message, 0)
+    @redis.set(@key_total_response_time, 0)
+    @last_aggregate = Time.now
+
+    STDERR.puts ""
+
+    @timer ||= Promiscuous::Timer.new
+    @timer.run_every(@interval) { aggregate_stats }
+  end
+
+  def disconnect
+    @timer.try(:reset)
+    @redis.client.disconnect rescue nil if @redis
+  end
+
+  def aggregate_stats
+    processed_messages = nil
+    total_response_time = nil
+    @redis.multi do
+      processed_messages = @redis.getset(@key_processed_message, 0)
+      total_response_time = @redis.getset(@key_total_response_time, 0)
+    end
+
+    last_aggregate = @last_aggregate
+    @last_aggregate = Time.now
+    processed_messages = processed_messages.value.to_i
+    total_response_time = total_response_time.value.to_i
+
+    rate = sprintf("%.1f", processed_messages.to_f / (Time.now - last_aggregate))
+    latency = "N/A"
+    unless processed_messages.zero?
+      latency = total_response_time.to_f / (1000 * processed_messages).to_f
+      if latency > 2.minutes
+        latency = sprintf("%.3fmin", latency / 1.minute)
+      else
+        latency = sprintf("%.3fsec", latency)
+      end
+    end
+    STDERR.puts "\e[1A" + "\b" * 200 + "Messages: #{processed_messages}  Rate: #{rate} msg/s  Latency: #{latency}" + " " * 30
+  end
+
+  def notify_processed_message(msg, time)
+    return if msg.timestamp.zero? || !@redis
+
+    msecs = (time.to_i * 1000 + time.usec / 1000).to_i - msg.timestamp
+    @redis.pipelined do
+      @redis.incr(@key_processed_message)
+      @redis.incrby(@key_total_response_time, msecs)
+    end
+  end
+end

data/lib/promiscuous/timer.rb

@@ -0,0 +1,38 @@
+class Promiscuous::Timer
+  def initialize
+    @lock = Mutex.new
+  end
+
+  def run_every(duration, options={}, &block)
+    options = options.dup
+    duration = duration.to_f unless duration.is_a?(Integer)
+    reset
+
+    @lock.synchronize do
+      @thread ||= Thread.new do
+        loop do
+          sleep duration unless options.delete(:run_immediately)
+          @lock.synchronize do
+            if @thread == Thread.current
+              begin
+                block.call
+              rescue Exception
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def reset
+    if @thread == Thread.current
+      @thread = nil
+    else
+      @lock.synchronize do
+        @thread.try(:kill)
+        @thread = nil
+      end
+    end
+  end
+end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.53.1'
+  VERSION = '0.90.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.53.1
+  version: 0.90.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,168 +10,136 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-10 00:00:00.000000000 Z
+date: 2013-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
+    none: false
+  prerelease: false
+  name: activesupport
 - !ruby/object:Gem::Dependency
-  name: activemodel
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: bunny
-  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.8.0
+  name: activemodel
 - !ruby/object:Gem::Dependency
-  name: amqp
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.9.8
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: 0.9.0.pre8
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.9.8
+        version: 0.9.0.pre8
+    none: false
+  prerelease: false
+  name: bunny
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.2
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  name: ruby-progressbar
 - !ruby/object:Gem::Dependency
-  name: crowdtap_redis_lock
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: 3.0.2
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 3.0.2
+    none: false
+  prerelease: false
+  name: redis
 - !ruby/object:Gem::Dependency
-  name: celluloid
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.12.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: 0.6.1
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.12.4
+        version: 0.6.1
+    none: false
+  prerelease: false
+  name: algorithms
 - !ruby/object:Gem::Dependency
-  name: celluloid-io
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.12.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: 0.2.0
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.12.1
+        version: 0.2.0
+    none: false
+  prerelease: false
+  name: fnv
 - !ruby/object:Gem::Dependency
-  name: algorithms
+  type: :runtime
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: 1.7.2
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 1.7.2
+    none: false
+  prerelease: false
+  name: multi_json
 description: Replicate your Mongoid/ActiveRecord models across your applications
 email:
 - nicolas@viennot.biz
@@ -183,78 +151,65 @@ extra_rdoc_files: []
 files:
 - lib/promiscuous/amqp/null.rb
 - lib/promiscuous/amqp/bunny.rb
-- lib/promiscuous/amqp/ruby_amqp.rb
-- lib/promiscuous/common/lint/base.rb
-- lib/promiscuous/common/class_helpers.rb
-- lib/promiscuous/common/options.rb
-- lib/promiscuous/common/lint.rb
-- lib/promiscuous/publisher/lint/amqp.rb
-- lib/promiscuous/publisher/lint/attributes.rb
-- lib/promiscuous/publisher/lint/base.rb
-- lib/promiscuous/publisher/lint/polymorphic.rb
-- lib/promiscuous/publisher/lint/class.rb
-- lib/promiscuous/publisher/mongoid/embedded_many.rb
-- lib/promiscuous/publisher/mongoid/embedded.rb
+- lib/promiscuous/amqp/fake.rb
+- lib/promiscuous/amqp/hot_bunnies.rb
 - lib/promiscuous/publisher/model/active_record.rb
 - lib/promiscuous/publisher/model/mongoid.rb
-- lib/promiscuous/publisher/envelope.rb
-- lib/promiscuous/publisher/polymorphic.rb
-- lib/promiscuous/publisher/mock.rb
-- lib/promiscuous/publisher/class.rb
-- lib/promiscuous/publisher/ephemeral.rb
-- lib/promiscuous/publisher/attributes.rb
-- lib/promiscuous/publisher/active_record.rb
-- lib/promiscuous/publisher/base.rb
-- lib/promiscuous/publisher/lint.rb
-- lib/promiscuous/publisher/mongoid.rb
-- lib/promiscuous/publisher/amqp.rb
+- lib/promiscuous/publisher/model/ephemeral.rb
+- lib/promiscuous/publisher/model/base.rb
+- lib/promiscuous/publisher/model/mock.rb
+- lib/promiscuous/publisher/operation.rb
+- lib/promiscuous/publisher/operation/mongoid.rb
+- lib/promiscuous/publisher/operation/base.rb
 - lib/promiscuous/publisher/model.rb
-- lib/promiscuous/subscriber/lint/amqp.rb
-- lib/promiscuous/subscriber/lint/base.rb
-- lib/promiscuous/subscriber/lint/class.rb
-- lib/promiscuous/subscriber/lint/polymorphic.rb
-- lib/promiscuous/subscriber/lint/attributes.rb
-- lib/promiscuous/subscriber/mongoid/embedded_many.rb
-- lib/promiscuous/subscriber/mongoid/embedded.rb
-- lib/promiscuous/subscriber/worker/runner.rb
+- lib/promiscuous/publisher/mock_generator.rb
+- lib/promiscuous/publisher/context.rb
+- lib/promiscuous/publisher/worker.rb
+- lib/promiscuous/subscriber/worker/recorder.rb
 - lib/promiscuous/subscriber/worker/pump.rb
-- lib/promiscuous/subscriber/worker/message.rb
+- lib/promiscuous/subscriber/worker/stats.rb
 - lib/promiscuous/subscriber/worker/message_synchronizer.rb
-- lib/promiscuous/subscriber/active_record.rb
-- lib/promiscuous/subscriber/envelope.rb
-- lib/promiscuous/subscriber/upsert.rb
-- lib/promiscuous/subscriber/polymorphic.rb
-- lib/promiscuous/subscriber/attributes.rb
-- lib/promiscuous/subscriber/base.rb
-- lib/promiscuous/subscriber/class.rb
-- lib/promiscuous/subscriber/lint.rb
-- lib/promiscuous/subscriber/mongoid.rb
-- lib/promiscuous/subscriber/observer.rb
-- lib/promiscuous/subscriber/amqp.rb
-- lib/promiscuous/subscriber/dummy.rb
+- lib/promiscuous/subscriber/worker/message.rb
+- lib/promiscuous/subscriber/worker/runner.rb
+- lib/promiscuous/subscriber/model/active_record.rb
+- lib/promiscuous/subscriber/model/mongoid.rb
+- lib/promiscuous/subscriber/model/base.rb
+- lib/promiscuous/subscriber/model/observer.rb
 - lib/promiscuous/subscriber/model.rb
+- lib/promiscuous/subscriber/payload.rb
+- lib/promiscuous/subscriber/operation.rb
 - lib/promiscuous/subscriber/worker.rb
-- lib/promiscuous/error/subscriber.rb
-- lib/promiscuous/error/publisher.rb
+- lib/promiscuous/error/already_processed.rb
+- lib/promiscuous/error/base.rb
 - lib/promiscuous/error/connection.rb
-- lib/promiscuous/error/recover.rb
-- lib/promiscuous/ephemeral.rb
+- lib/promiscuous/error/lock_unavailable.rb
+- lib/promiscuous/error/publisher.rb
+- lib/promiscuous/error/recovery.rb
+- lib/promiscuous/error/subscriber.rb
+- lib/promiscuous/error/lost_lock.rb
+- lib/promiscuous/error/missing_context.rb
+- lib/promiscuous/error/dependency.rb
 - lib/promiscuous/autoload.rb
-- lib/promiscuous/loader.rb
-- lib/promiscuous/railtie.rb
-- lib/promiscuous/common.rb
-- lib/promiscuous/publisher.rb
 - lib/promiscuous/subscriber.rb
-- lib/promiscuous/redis.rb
-- lib/promiscuous/observer.rb
+- lib/promiscuous/dsl.rb
+- lib/promiscuous/convenience.rb
+- lib/promiscuous/key.rb
+- lib/promiscuous/publisher.rb
 - lib/promiscuous/error.rb
-- lib/promiscuous/amqp.rb
+- lib/promiscuous/sidekiq.rb
+- lib/promiscuous/resque.rb
+- lib/promiscuous/middleware.rb
+- lib/promiscuous/dependency.rb
 - lib/promiscuous/cli.rb
+- lib/promiscuous/redis.rb
+- lib/promiscuous/timer.rb
+- lib/promiscuous/amqp.rb
 - lib/promiscuous/config.rb
+- lib/promiscuous/railtie.rb
+- lib/promiscuous/loader.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous
-- README.md
 homepage: http://github.com/crowdtap/promiscuous
 licenses: []
 post_install_message: 
@@ -262,20 +217,20 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Model replication over RabbitMQ