promiscuous 0.33.1 → 0.50.0

data/lib/promiscuous/amqp/ruby_amqp.rb

@@ -27,20 +27,12 @@ module Promiscuous::AMQP::RubyAMQP
         e = Promiscuous::AMQP.lost_connection_exception
         Promiscuous.warn "[amqp] #{e}. Reconnecting..."
         Promiscuous::Config.error_notifier.try(:call, e)
-
-        worker = Promiscuous::Worker.workers.first
-        worker.message_synchronizer.disconnect if worker
-
         conn.periodically_reconnect(2.seconds)
       end
     end
 
     connection.on_recovery do |conn|
       Promiscuous.warn "[amqp] Reconnected"
-
-      worker = Promiscuous::Worker.workers.first
-      worker.message_synchronizer.reconnect if worker
-
       Promiscuous::AMQP::RubyAMQP.channel.recover
     end
 

data/lib/promiscuous/cli.rb

@@ -1,31 +1,5 @@
 class Promiscuous::CLI
-  def replicate(config_options={}, &block)
-    require 'eventmachine'
-    require 'em-synchrony'
-
-    EM.synchrony do
-      trap_signals
-      Promiscuous::Loader.load_descriptors if defined?(Rails)
-      force_backend :rubyamqp
-      block.call
-    end
-  end
-
-  def force_backend(backend)
-    Promiscuous.disconnect
-    Promiscuous::Config.backend = backend
-    Promiscuous.connect
-  end
-
-  def trap_signals
-    %w(SIGTERM SIGINT).each do |signal|
-      Signal.trap(signal) do
-        print_status "Exiting..."
-        Promiscuous::Worker.kill
-        EM.stop
-      end
-    end
-
+  def self.trap_signals
     Signal.trap 'SIGUSR2' do
       Thread.list.each do |thread|
         print_status '-' * 80
@@ -38,38 +12,25 @@ class Promiscuous::CLI
       end
     end
   end
+  trap_signals
 
-  def publish_sync(options={})
-    print_status "Replicating #{options[:criteria]}..."
+  def publish(options={})
+    print_status "Publishing #{options[:criteria]}..."
     criteria = eval(options[:criteria])
 
     bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => 'Publishing', :total => criteria.count)
     criteria.each do |doc|
-      doc.promiscuous_sync(options)
+      doc.promiscuous_sync
       bar.increment
     end
-
-    print_status "Done. You may switch your subscriber worker back to regular mode, and delete the sync queues"
   end
 
   def subscribe(options={})
-    replicate do
-      Promiscuous::Worker.replicate
-      print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
-    end
-  end
-
-  def subscribe_sync(options={})
-    replicate do
-      # Create the regular queue if needed, so we don't lose messages.
-      Promiscuous::AMQP.open_queue(Promiscuous::Subscriber::Worker.new.queue_bindings)
-
-      print_status "WARNING: --- SYNC MODE ----"
-      print_status "WARNING: Make sure you are not running the regular subscriber worker (it's racy)"
-      print_status "WARNING: --- SYNC MODE ----"
-      Promiscuous::Worker.replicate
-      print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
-    end
+    Promiscuous::Loader.load_descriptors if defined?(Rails)
+    print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
+    Promiscuous::Subscriber::Worker.run
+  rescue Interrupt
+    # SIGINT
   end
 
   def parse_args(args)
@@ -81,20 +42,12 @@ class Promiscuous::CLI
 
       opts.separator ""
       opts.separator "Actions:"
-      opts.separator "    publish (sync only)"
+      opts.separator "    publish \"Model.where(:shard => 123)\""
       opts.separator "    subscribe"
       opts.separator ""
       opts.separator "Options:"
 
-      opts.on "-s", "--sync", "Use a separate queue for sychronizing databases" do
-        options[:personality] = :sync
-      end
-
-      opts.on "-c", "--criteria CRITERIA", "Published criteria in sync mode. e.g. Member.where(:created_at.gt => 1.day.ago)" do |criteria|
-        options[:criteria] = criteria
-      end
-
-      opts.on "-b", "--bareback", "Bareback mode aka continue on error. Use with extreme caution" do
+      opts.on "-b", "--bareback", "Bareback mode aka no dependencies. Use with extreme caution" do
         options[:bareback] = true
       end
 
@@ -118,15 +71,23 @@ class Promiscuous::CLI
     parser.parse!(args)
 
     options[:action] = args.shift.try(:to_sym)
-    raise "Please specify an action (publish or subscribe)" unless options[:action].in? [:publish, :subscribe]
+    options[:criteria] = args.shift
 
-    if options[:action] == :publish && options[:personality] == :sync
+    unless options[:action].in? [:publish, :subscribe]
+      puts parser
+      exit
+    end
+
+    if options[:action] == :publish
       raise "Please specify a criteria" unless options[:criteria]
     else
       raise "Why are you specifying a criteria?" if options[:criteria]
     end
 
     options
+  rescue Exception => e
+    puts e
+    exit
   end
 
   def load_app(options={})
@@ -142,14 +103,17 @@ class Promiscuous::CLI
   def run
     options = parse_args(ARGV)
     load_app(options)
-    maybe_warn_bareback(options)
+    maybe_run_bareback(options)
 
-    # calls publish, publish_sync, subscribe, subscribe_sync
-    __send__([options[:action], options[:personality]].compact.join('_'), options)
+    case options[:action]
+    when :publish then publish(options)
+    when :subscribe then subscribe(options)
+    end
   end
 
-  def maybe_warn_bareback(options)
+  def maybe_run_bareback(options)
     if options[:bareback]
+      Promiscuous::Config.bareback = true
       print_status "WARNING: --- BAREBACK MODE ----"
       print_status "WARNING: You are replicating without protection, you can get corrupted in no time"
       print_status "WARNING: --- BAREBACK MODE ----"

data/lib/promiscuous/config.rb

@@ -1,5 +1,5 @@
 module Promiscuous::Config
-  mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url, :redis_url, :queue_options, :heartbeat
+  mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url, :redis_url, :queue_options, :heartbeat, :bareback
 
   def self.backend=(value)
     @@backend = value

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -43,6 +43,27 @@ module Promiscuous::Publisher::Model::Mongoid
     end
   end
 
+  module ModelInstanceMethods
+    extend ActiveSupport::Concern
+
+    def promiscuous_sync
+      publisher = self.class.promiscuous_publisher
+
+      fetch_proc = proc { self.class.with(:consistency => :strong).where(atomic_selector).first }
+
+      publisher.new(:operation  => :update,
+                    :instance   => self,
+                    :fetch_proc => fetch_proc).commit
+    end
+  end
+
+  module ClassMethods
+    def setup_class_binding
+      super
+      klass.__send__(:include, ModelInstanceMethods) if klass
+    end
+  end
+
   def self.hook_mongoid
     Moped::Collection.class_eval do
       alias_method :insert_orig, :insert

data/lib/promiscuous/publisher/model.rb

@@ -60,7 +60,7 @@ module Promiscuous::Publisher::Model
     with_lock do
       update_dependencies
       begin
-        ret = yield
+        ret = yield if block_given?
       rescue Exception => e
         # we must publish something so the subscriber can sync
         # with the updated dependencies
@@ -81,11 +81,10 @@ module Promiscuous::Publisher::Model
     ret
   end
 
-
   module ClassMethods
     def setup_class_binding
       super
-      Promiscuous::Publisher::Model.klasses << klass
+      Promiscuous::Publisher::Model.klasses << klass if klass
     end
   end
 end

data/lib/promiscuous/subscriber/worker/message.rb

@@ -1,8 +1,7 @@
 class Promiscuous::Subscriber::Worker::Message
-  attr_accessor :worker, :metadata, :payload, :parsed_payload
+  attr_accessor :metadata, :payload, :parsed_payload
 
-  def initialize(worker, metadata, payload)
-    self.worker = worker
+  def initialize(metadata, payload)
     self.metadata = metadata
     self.payload = payload
   end
@@ -24,6 +23,7 @@ class Promiscuous::Subscriber::Worker::Message
   end
 
   def has_dependencies?
+    return false if Promiscuous::Config.bareback
     !!global_version
   end
 
@@ -37,11 +37,25 @@ class Promiscuous::Subscriber::Worker::Message
     end
   end
 
+  def unit_of_work(type, &block)
+    # type is used by the new relic agent, by monkey patching.
+    # middleware?
+    if defined?(Mongoid)
+      Mongoid.unit_of_work { yield }
+    else
+      yield
+    end
+  ensure
+    if defined?(ActiveRecord)
+      ActiveRecord::Base.clear_active_connections!
+    end
+  end
+
   def process
-    return if worker.stopped?
+    #return if worker.stopped?
 
     Promiscuous.debug "[receive] #{payload}"
-    worker.unit_of_work(queue_name) do
+    unit_of_work(queue_name) do
       Promiscuous::Subscriber.process(parsed_payload, :message => self)
     end
 

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -1,13 +1,9 @@
 class Promiscuous::Subscriber::Worker::MessageSynchronizer
   include Celluloid::IO
 
-  attr_accessor :worker, :redis
+  attr_accessor :redis
 
-  def initialize(worker)
-    self.worker = worker
-  end
-
-  def start
+  def initialize
     connect
     main_loop!
   end
@@ -36,7 +32,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     Promiscuous.warn "[redis] #{e}. Reconnecting..."
     Promiscuous::Config.error_notifier.try(:call, e)
 
-    EM.next_tick { worker.pump.stop }
+    # TODO stop the pump to unack all messages
     reconnect_later
   end
 
@@ -56,7 +52,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       main_loop!
 
       Promiscuous.warn "[redis] Reconnected"
-      EM.next_tick { worker.pump.start }
+      # TODO restart the pump
     end
   rescue
     reconnect_later
@@ -94,7 +90,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   rescue Exception => e
     Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
 
-    Promiscuous::Worker.stop
+    #Promiscuous::Worker.stop TODO
     Promiscuous::Config.error_notifier.try(:call, e)
   end
 
@@ -110,15 +106,19 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     # when calling worker.pump.start
     return unless self.redis
 
-    return worker.runners.process!(msg) unless msg.has_dependencies?
+    return process_message!(msg) unless msg.has_dependencies?
 
     # The message synchronizer only takes care of happens before (>=) dependencies.
     # The message will handle the skip logic in case of duplicates.
     on_version Promiscuous::Redis.sub_key('global'), msg.global_version do
-      worker.runners.process!(msg)
+      process_message!(msg)
     end
   end
 
+  def process_message!(msg)
+    Celluloid::Actor[:runners].process!(msg)
+  end
+
   def on_version(key, version, &callback)
     return unless @subscriptions
     cb = Subscription::Callback.new(version, callback)

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -1,42 +1,71 @@
+require 'eventmachine'
+
 class Promiscuous::Subscriber::Worker::Pump
-  # TODO Make this celluloid happy
-  attr_accessor :worker
+  include Celluloid
+
+  def initialize
+    # signals do not work when initializing with Celluloid
+    # I wish ruby had semaphores, it would make much more sense.
+    @initialize_mutex = Mutex.new
+    @initialization_done = ConditionVariable.new
+
+    @em_thread = Thread.new { EM.run { start } }
+
+    # The event machine thread will unlock us
+    wait_for_initialization
+    raise @exception if @exception
+  end
+
+  def wait_for_initialization
+    @initialize_mutex.synchronize do
+      @initialization_done.wait(@initialize_mutex)
+    end
+  end
+
+  def finalize_initialization
+    @initialize_mutex.synchronize do
+      @initialization_done.signal
+    end
+  end
+
+  def finalize
+    @dont_reconnect = true
+    EM.next_tick do
+      Promiscuous::AMQP.disconnect
+      EM.stop
+    end
+    @em_thread.join
+  rescue
+    # Let amqp die like a pro
+  end
 
-  def initialize(worker)
-    self.worker = worker
+  def force_use_ruby_amqp
+    Promiscuous::AMQP.disconnect
+    Promiscuous::Config.backend = :rubyamqp
+    Promiscuous::AMQP.connect
   end
 
   def start
-    return if @queue
+    force_use_ruby_amqp
     Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
-      @queue = queue
-      @queue.subscribe :ack => true do |metadata, payload|
-        # we drop the payload if we switched to another queue,
-        # duplicate messages could hurt us.
-        process_payload(metadata, payload) if queue == @queue
+      queue.subscribe :ack => true do |metadata, payload|
+        process_payload(metadata, payload)
       end
     end
-  end
-
-  def stop
-    queue, @queue = @queue, nil
-    queue.unsubscribe if queue rescue nil
+  rescue Exception => @exception
+  ensure
+    finalize_initialization
   end
 
   def process_payload(metadata, payload)
-    msg = Promiscuous::Subscriber::Worker::Message.new(worker, metadata, payload)
-    worker.message_synchronizer.process_when_ready(msg)
+    msg = Promiscuous::Subscriber::Worker::Message.new(metadata, payload)
+    Celluloid::Actor[:message_synchronizer].process_when_ready(msg)
   end
 
   def queue_bindings
     queue_name = "#{Promiscuous::Config.app}.promiscuous"
     exchange_name = Promiscuous::AMQP::EXCHANGE
 
-    if worker.options[:personality]
-      queue_name    += ".#{worker.options[:personality]}"
-      exchange_name += ".#{worker.options[:personality]}"
-    end
-
     # We need to subscribe to everything to keep up with the version tracking
     bindings = ['*']
     {:exchange_name => exchange_name, :queue_name => queue_name, :bindings => bindings}

data/lib/promiscuous/subscriber/worker.rb

@@ -1,59 +1,13 @@
-class Promiscuous::Subscriber::Worker
-  require 'celluloid'
-  require 'celluloid/io'
+require 'celluloid'
+require 'celluloid/io'
 
+class Promiscuous::Subscriber::Worker < Celluloid::SupervisionGroup
   extend Promiscuous::Autoload
   autoload :Message, :Pump, :MessageSynchronizer, :Runner
 
-  attr_accessor :options, :stopped, :pump, :message_synchronizer, :runners
-  alias_method :stopped?, :stopped
-
-  def initialize(options={})
-    Celluloid.exception_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
-
-    self.options = options
-    self.stopped = true
-
-    self.pump = Pump.new(self)
-  end
-
-  def start
-    return unless self.stopped
-    self.stopped = false
-    self.runners = Runner.pool
-    self.message_synchronizer = MessageSynchronizer.new(self)
-    self.message_synchronizer.start
-    self.pump.start
-  end
-
-  def stop
-    return if self.stopped
-    self.pump.stop
-    if self.message_synchronizer
-      self.message_synchronizer.stop rescue Celluloid::Task::TerminatedError
-      self.message_synchronizer = nil
-    end
-    if self.runners
-      self.runners.terminate
-      self.runners = nil
-    end
-    self.stopped = true
-
-    # TODO wait for the runners to finish
-    sleep 1
-  end
-
-  def unit_of_work(type, &block)
-    # type is used by the new relic agent, by monkey patching.
-    # middleware?
-    if defined?(Mongoid)
-      Mongoid.unit_of_work { yield }
-    else
-      yield
-    end
-  ensure
-    if defined?(ActiveRecord)
-      ActiveRecord::Base.clear_active_connections!
-    end
-  end
+  pool      Runner,              :as => :runners, :size => 10
+  supervise MessageSynchronizer, :as => :message_synchronizer
+  supervise Pump,                :as => :pump
 end
+
+Celluloid.exception_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.33.1'
+  VERSION = '0.50.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.33.1
+  version: 0.50.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-03 00:00:00.000000000 Z
+date: 2013-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -76,22 +76,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.9.8
-- !ruby/object:Gem::Dependency
-  name: em-synchrony
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.0.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -182,8 +166,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/promiscuous/amqp/null.rb
-- lib/promiscuous/amqp/ruby_amqp.rb
 - lib/promiscuous/amqp/bunny.rb
+- lib/promiscuous/amqp/ruby_amqp.rb
 - lib/promiscuous/common/lint/base.rb
 - lib/promiscuous/common/class_helpers.rb
 - lib/promiscuous/common/options.rb
@@ -230,10 +214,10 @@ files:
 - lib/promiscuous/subscriber/lint.rb
 - lib/promiscuous/subscriber/mongoid.rb
 - lib/promiscuous/subscriber/observer.rb
-- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/amqp.rb
 - lib/promiscuous/subscriber/dummy.rb
 - lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/error/subscriber.rb
 - lib/promiscuous/error/publisher.rb
 - lib/promiscuous/error/connection.rb
@@ -245,12 +229,11 @@ files:
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/common.rb
 - lib/promiscuous/publisher.rb
-- lib/promiscuous/worker.rb
 - lib/promiscuous/amqp.rb
 - lib/promiscuous/subscriber.rb
+- lib/promiscuous/redis.rb
 - lib/promiscuous/config.rb
 - lib/promiscuous/cli.rb
-- lib/promiscuous/redis.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous

data/lib/promiscuous/worker.rb

@@ -1,22 +0,0 @@
-module Promiscuous::Worker
-  mattr_accessor :workers
-  self.workers = []
-
-  def self.replicate(options={})
-    self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.start }
-  end
-
-  def self.kill
-    stop
-    # TODO FIXME We should wait for them to be idle
-    workers.clear
-  end
-
-  def self.stop
-    workers.each(&:stop)
-  end
-
-  def self.start
-    workers.each(&:start)
-  end
-end