promiscuous 0.12.1 → 0.13.0

data/lib/promiscuous/amqp/bunny.rb

@@ -13,14 +13,18 @@ module Promiscuous
         self.connection.stop
       end
 
-      def self.publish(msg)
-        Promiscuous.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
-        exchange = connection.exchange('promiscuous', :type => :topic, :durable => true)
-        exchange.publish(msg[:payload], :key => msg[:key], :persistent => true)
+      def self.publish(options={})
+        Promiscuous.info "[publish] (#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
+        exchange(options[:exchange_name]).
+          publish(options[:payload], :key => options[:key], :persistent => true)
       end
 
-      def self.subscribe(options={}, &block)
-        raise "Cannot subscribe with bunny"
+      def self.open_queue(options={}, &block)
+        raise "Cannot open queue with bunny"
+      end
+
+      def self.exchange(name)
+        connection.exchange(name, :type => :topic, :durable => true)
       end
     end
   end

data/lib/promiscuous/amqp/null.rb

@@ -7,10 +7,10 @@ module Promiscuous
       def self.disconnect
       end
 
-      def self.publish(msg)
+      def self.publish(msg, options={})
       end
 
-      def self.subscribe(options={}, &block)
+      def self.open_queue(options={}, &block)
       end
     end
   end

data/lib/promiscuous/amqp/rubyamqp.rb

@@ -1,7 +1,7 @@
 module Promiscuous
   module AMQP
     module RubyAMQP
-      mattr_accessor :channel, :queue_options
+      mattr_accessor :channel
 
       def self.connect
         require 'amqp'
@@ -17,12 +17,11 @@ module Promiscuous
             :user   => uri.user,
             :pass   => uri.password,
             :vhost  => uri.path.empty? ? "/" : uri.path,
-         }
+          }
         end
 
         connection = ::AMQP.connect(amqp_options)
         self.channel = ::AMQP::Channel.new(connection)
-        self.queue_options = Promiscuous::Config.queue_options
       end
 
       def self.disconnect
@@ -30,23 +29,26 @@ module Promiscuous
         self.channel = nil
       end
 
-      def self.subscribe(options={}, &block)
+      def self.open_queue(options={}, &block)
         queue_name = options[:queue_name]
         bindings   = options[:bindings]
 
-        queue = self.channel.queue(queue_name, self.queue_options)
-        exchange = channel.topic('promiscuous', :durable => true)
+        queue = self.channel.queue(queue_name, Promiscuous::Config.queue_options)
         bindings.each do |binding|
-          queue.bind(exchange, :routing_key => binding)
+          queue.bind(exchange(options[:exchange_name]), :routing_key => binding)
           Promiscuous.info "[bind] #{queue_name} -> #{binding}"
         end
-        queue.subscribe(:ack => true, &block)
+        block.call(queue) if block
       end
 
-      def self.publish(msg)
-        Promiscuous.info "[publish] #{msg[:key]} -> #{msg[:payload]}"
-        exchange = channel.topic('promiscuous', :durable => true)
-        exchange.publish(msg[:payload], :routing_key => msg[:key], :persistent => true)
+      def self.publish(options={})
+        Promiscuous.info "[publish] (#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
+        exchange(options[:exchange_name]).
+          publish(options[:payload], :routing_key => options[:key], :persistent => true)
+      end
+
+      def self.exchange(name)
+        channel.topic(name, :durable => true)
       end
     end
   end

data/lib/promiscuous/amqp.rb

@@ -3,11 +3,13 @@ module Promiscuous::AMQP
   autoload :RubyAMQP, 'promiscuous/amqp/rubyamqp'
   autoload :Null,     'promiscuous/amqp/null'
 
+  EXCHANGE = 'promiscuous'.freeze
+
   class << self
     def backend
       Promiscuous::Config.backend
     end
 
-    delegate :connect, :disconnect, :publish, :subscribe, :to => :backend
+    delegate :connect, :disconnect, :publish, :open_queue, :to => :backend
   end
 end

data/lib/promiscuous/common/worker.rb

@@ -1,7 +1,8 @@
 module Promiscuous::Common::Worker
   extend ActiveSupport::Concern
 
-  def initialize
+  def initialize(options={})
+    self.options = options
     self.stop = false
   end
 
@@ -13,5 +14,5 @@ module Promiscuous::Common::Worker
     end
   end
 
-  included { attr_accessor :stop }
+  included { attr_accessor :stop, :options }
 end

data/lib/promiscuous/publisher/amqp.rb

@@ -3,7 +3,9 @@ module Promiscuous::Publisher::AMQP
   include Promiscuous::Publisher::Envelope
 
   def publish
-    Promiscuous::AMQP.publish(:key => to, :payload => payload.to_json)
+    exchange_name = Promiscuous::AMQP::EXCHANGE
+    exchange_name += ".#{options[:personality]}" if options[:personality]
+    Promiscuous::AMQP.publish(:exchange_name => exchange_name, :key => to, :payload => payload.to_json)
   end
 
   def payload

data/lib/promiscuous/publisher/model.rb

@@ -36,7 +36,12 @@ module Promiscuous::Publisher::Model
             self.class.promiscuous_publisher.new(:instance => self, :operation => operation).publish
           end
         end
-        alias :promiscuous_sync :promiscuous_publish_update
+
+        def promiscuous_sync(options={})
+          options = options.merge({ :instance => self, :operation => :update, :defer => false })
+          self.class.promiscuous_publisher.new(options).publish
+          true
+        end
       end
     end
   end

data/lib/promiscuous/publisher/worker.rb

@@ -49,7 +49,7 @@ class Promiscuous::Publisher::Worker
 
   def replicate_instance(instance)
     return if self.stop
-    instance.class.promiscuous_publisher.new(:instance => instance, :operation => :update, :defer => false).publish
+    instance.promiscuous_sync
   rescue Exception => e
     # TODO set back the psp field
     raise Promiscuous::Publisher::Error.new(e, instance)

data/lib/promiscuous/railtie/replicate.rake

@@ -1,24 +1,5 @@
 namespace :promiscuous do
-  # Note This rake task can be loaded without Rails
-  desc 'Run the workers'
-  task :replicate => :environment do |t|
-    require 'eventmachine'
-    require 'em-synchrony'
-
-    EM.synchrony do
-      trap_signals
-      force_backend :rubyamqp
-
-      Promiscuous::Loader.load_descriptors if defined?(Rails)
-
-      Promiscuous::Worker.replicate
-
-      msg = "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers" +
-            " and #{Promiscuous::Publisher::Mongoid::Defer.klasses.count} publishers"
-      Promiscuous.info msg
-      $stderr.puts msg
-    end
-  end
+  # Note These rake tasks can be loaded without Rails
 
   def trap_signals
     %w(SIGTERM SIGINT).each do |signal|
@@ -30,9 +11,77 @@ namespace :promiscuous do
     end
   end
 
+  def print_status(msg)
+    Promiscuous.info msg
+    $stderr.puts msg
+  end
+
   def force_backend(backend)
     Promiscuous::AMQP.disconnect
     Promiscuous::Config.backend = backend
     Promiscuous::AMQP.connect
   end
+
+  def replicate(config_options={}, &block)
+    require 'eventmachine'
+    require 'em-synchrony'
+
+    EM.synchrony do
+      trap_signals
+      Promiscuous::Loader.load_descriptors if defined?(Rails)
+      force_backend :rubyamqp
+      block.call
+    end
+  end
+
+  desc 'Run the publisher worker'
+  task :publish => :environment do
+    replicate do
+      Promiscuous::Worker.replicate :only => :publish
+      print_status "Replicating with #{Promiscuous::Publisher::Mongoid::Defer.klasses.count} publishers"
+    end
+  end
+
+  desc 'Run the subscriber worker'
+  task :subscribe => :environment do
+    replicate do
+      Promiscuous::Worker.replicate :only => :subscribe
+      print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
+    end
+  end
+
+  namespace :synchronized do
+    desc 'Synchronize a collection'
+    task :publish, [:criteria] => :environment do |t, args|
+      raise "Usage: rake promiscuous:synchronized:publish[Model]" unless args.criteria
+
+      criteria = eval(args.criteria)
+      count = criteria.count
+      print_status "Replicating #{args.criteria}..."
+
+      bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e',
+                               :title => 'Publishing',
+                               :total => count)
+      criteria.each do |doc|
+        doc.promiscuous_sync :personality => :sync
+        bar.increment
+      end
+
+      print_status "Done. You may switch your subscriber worker back to regular mode, and delete the sync queues"
+    end
+
+    desc 'Subscribe to a collection synchronization'
+    task :subscribe => :environment do |t|
+      replicate do
+        # Create the regular queue if needed, so we don't lose messages.
+        Promiscuous::AMQP.open_queue(Promiscuous::Subscriber::Worker.new.queue_bindings)
+
+        print_status "WARNING: --- SYNC MODE ----"
+        print_status "WARNING: Make sure you are not running the regular subscriber worker (it's racy)"
+        print_status "WARNING: --- SYNC MODE ----"
+        Promiscuous::Worker.replicate :personality => :sync, :only => :subscribe
+        print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
+      end
+    end
+  end
 end

data/lib/promiscuous/subscriber/worker.rb

@@ -2,31 +2,39 @@ class Promiscuous::Subscriber::Worker
   include Promiscuous::Common::Worker
 
   def replicate
-    Promiscuous::AMQP.subscribe(subscribe_options) do |metadata, payload|
-      # Note: This code always runs on the root Fiber,
-      # so ordering is always preserved
-      begin
-        unless self.stop
-          Promiscuous.info "[receive] #{payload}"
-          self.unit_of_work { Promiscuous::Subscriber.process(JSON.parse(payload)) }
-          metadata.ack
-        end
-      rescue Exception => e
-        e = Promiscuous::Subscriber::Error.new(e, payload)
+    Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
+      queue.subscribe :ack => true do |metadata, payload|
+        # Note: This code always runs on the root Fiber,
+        # so ordering is always preserved
+        begin
+          unless self.stop
+            Promiscuous.info "[receive] #{payload}"
+            self.unit_of_work { Promiscuous::Subscriber.process(JSON.parse(payload)) }
+            metadata.ack
+          end
+        rescue Exception => e
+          e = Promiscuous::Subscriber::Error.new(e, payload)
 
-        # TODO Discuss with Arjun about having an error queue.
-        self.stop = true
-        Promiscuous::AMQP.disconnect
-        Promiscuous.error "[receive] FATAL #{e} #{e.backtrace.join("\n")}"
-        Promiscuous.error "[receive] FATAL #{e}"
-        Promiscuous::Config.error_handler.try(:call, e)
+          self.stop = true
+          Promiscuous::AMQP.disconnect
+          Promiscuous.error "[receive] FATAL #{e} #{e.backtrace.join("\n")}"
+          Promiscuous.error "[receive] FATAL #{e}"
+          Promiscuous::Config.error_handler.try(:call, e)
+        end
       end
     end
   end
 
-  def subscribe_options
+  def queue_bindings
     queue_name = "#{Promiscuous::Config.app}.promiscuous"
+    exchange_name = Promiscuous::AMQP::EXCHANGE
+
+    if options[:personality]
+      queue_name    += ".#{options[:personality]}"
+      exchange_name += ".#{options[:personality]}"
+    end
+
     bindings = Promiscuous::Subscriber::AMQP.subscribers.keys
-    {:queue_name => queue_name, :bindings => bindings}
+    {:exchange_name => exchange_name, :queue_name => queue_name, :bindings => bindings}
   end
 end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.12.1'
+  VERSION = '0.13.0'
 end

data/lib/promiscuous/worker.rb

@@ -2,9 +2,12 @@ module Promiscuous::Worker
   mattr_accessor :workers
   self.workers = []
 
-  def self.replicate
-    self.workers << Promiscuous::Publisher::Worker.new
-    self.workers << Promiscuous::Subscriber::Worker.new
+  def self.replicate(options={})
+    publish   = options[:only].nil? || options[:only] == :publish
+    subscribe = options[:only].nil? || options[:only] == :subscribe
+
+    self.workers << Promiscuous::Publisher::Worker.new(options) if publish
+    self.workers << Promiscuous::Subscriber::Worker.new(options) if subscribe
     self.workers.each { |w| w.replicate }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-12 00:00:00.000000000 Z
+date: 2012-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -92,6 +92,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Replicate your Mongoid/ActiveRecord models across your applications
 email:
 - nicolas@viennot.biz
@@ -101,7 +117,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/promiscuous.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/null.rb
 - lib/promiscuous/amqp/rubyamqp.rb
@@ -116,7 +131,6 @@ files:
 - lib/promiscuous/observer.rb
 - lib/promiscuous/publisher.rb
 - lib/promiscuous/publisher/active_record.rb
-- lib/promiscuous/publisher/amqp.rb
 - lib/promiscuous/publisher/base.rb
 - lib/promiscuous/publisher/class.rb
 - lib/promiscuous/publisher/envelope.rb
@@ -129,15 +143,16 @@ files:
 - lib/promiscuous/publisher/lint/class.rb
 - lib/promiscuous/publisher/lint/polymorphic.rb
 - lib/promiscuous/publisher/mock.rb
-- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/mongoid/embedded.rb
 - lib/promiscuous/publisher/mongoid/defer.rb
 - lib/promiscuous/publisher/mongoid/defer_embedded.rb
 - lib/promiscuous/publisher/mongoid/embedded_many.rb
 - lib/promiscuous/publisher/polymorphic.rb
-- lib/promiscuous/publisher/worker.rb
 - lib/promiscuous/publisher/attributes.rb
 - lib/promiscuous/publisher/mongoid.rb
+- lib/promiscuous/publisher/amqp.rb
+- lib/promiscuous/publisher/model.rb
+- lib/promiscuous/publisher/worker.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/railtie/replicate.rake
 - lib/promiscuous/subscriber/active_record.rb
@@ -156,14 +171,15 @@ files:
 - lib/promiscuous/subscriber/observer.rb
 - lib/promiscuous/subscriber/polymorphic.rb
 - lib/promiscuous/subscriber/upsert.rb
-- lib/promiscuous/subscriber/worker.rb
 - lib/promiscuous/subscriber/amqp.rb
 - lib/promiscuous/subscriber/attributes.rb
 - lib/promiscuous/subscriber/base.rb
 - lib/promiscuous/subscriber/mongoid.rb
-- lib/promiscuous/worker.rb
+- lib/promiscuous/subscriber/worker.rb
 - lib/promiscuous/subscriber.rb
 - lib/promiscuous/ephemeral.rb
+- lib/promiscuous/amqp.rb
+- lib/promiscuous/worker.rb
 - lib/promiscuous/version.rb
 - README.md
 homepage: http://github.com/crowdtap/promiscuous