promiscuous 0.50.4 → 0.51.0

data/lib/promiscuous.rb

@@ -46,5 +46,29 @@ module Promiscuous
     end
   end
 
+  module ConsoleHelpers
+    # These are just for the subscriber, some helpers to debug in production
+    def global_key
+      Promiscuous::Redis.sub_key('global')
+    end
+
+    def global_version
+      Promiscuous::Redis.get(global_key).to_i
+    end
+
+    def global_version=(value)
+      Promiscuous::Redis.set(global_key, value)
+      Promiscuous::Redis.publish(global_key, value)
+      value
+    end
+
+    def global_version!
+      version = Promiscuous::Redis.incr(global_key)
+      Promiscuous::Redis.publish(global_key, version)
+      version
+    end
+  end
+  extend ConsoleHelpers
+
   at_exit { self.disconnect rescue nil }
 end

data/lib/promiscuous/amqp/ruby_amqp.rb

@@ -20,7 +20,9 @@ module Promiscuous::AMQP::RubyAMQP
     end
 
     connection = ::AMQP.connect(amqp_options)
-    self.channel = ::AMQP::Channel.new(connection, :auto_recovery => true, :prefetch => 1000)
+    self.channel = ::AMQP::Channel.new(connection,
+                                       :auto_recovery => true,
+                                       :prefetch => Promiscuous::Config.prefetch)
 
     connection.on_tcp_connection_loss do |conn|
       unless conn.reconnecting?

data/lib/promiscuous/cli.rb

@@ -51,13 +51,22 @@ class Promiscuous::CLI
       opts.separator "Options:"
 
       opts.on "-b", "--bareback", "Bareback mode aka no dependencies. Use with extreme caution" do
-        options[:bareback] = true
+        Promiscuous::Config.bareback = true
       end
 
-      opts.on "-r", "--require FILE", "File to require to load your app. Don't worry about it with rails" do |file|
+      opts.on "-l", "--require FILE", "File to require to load your app. Don't worry about it with rails" do |file|
         options[:require] = file
       end
 
+      opts.on "-r", "--recovery [TIMEOUT]", "Run in recovery mode. Defaults to 10 seconds before recovering" do |timeout|
+        Promiscuous::Config.recovery_timeout = (timeout || 10).to_i
+      end
+
+      opts.on "-p", "--prefetch [NUM]", "Number of messages to prefetch" do |prefetch|
+        exit 1 if prefetch.to_i == 0
+        Promiscuous::Config.prefetch = prefetch.to_i
+      end
+
       opts.on("-h", "--help", "Show this message") do
         puts opts
         exit
@@ -106,7 +115,7 @@ class Promiscuous::CLI
   def boot
     self.options = parse_args(ARGV)
     load_app
-    maybe_run_bareback
+    show_bareback_warnings
     run
   end
 
@@ -117,9 +126,8 @@ class Promiscuous::CLI
     end
   end
 
-  def maybe_run_bareback
-    if options[:bareback]
-      Promiscuous::Config.bareback = true
+  def show_bareback_warnings
+    if Promiscuous::Config.bareback == true
       print_status "WARNING: --- BAREBACK MODE ----"
       print_status "WARNING: You are replicating without protection, you can get out of sync in no time"
       print_status "WARNING: --- BAREBACK MODE ----"

data/lib/promiscuous/config.rb

@@ -1,20 +1,25 @@
 module Promiscuous::Config
-  mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url, :redis_url, :queue_options, :heartbeat, :bareback
+  mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url,
+                 :redis_url, :queue_options, :heartbeat, :bareback,
+                 :recovery_timeout, :prefetch
 
   def self.backend=(value)
     @@backend = value
     Promiscuous::AMQP.backend = value unless value.nil?
   end
 
-  def self.configure(&block)
+  def self.reset
     class_variables.each { |var| class_variable_set(var, nil) }
+  end
 
+  def self.configure(&block)
     block.call(self)
 
     self.backend ||= defined?(EventMachine) && EventMachine.reactor_running? ? :rubyamqp : :bunny
     self.logger ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
     self.queue_options ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
     self.heartbeat ||= 60
+    self.prefetch ||= 1000
 
     Promiscuous.connect
   end

data/lib/promiscuous/error.rb

@@ -1,4 +1,4 @@
 module Promiscuous::Error
   extend Promiscuous::Autoload
-  autoload :Connection, :Publisher, :Subscriber
+  autoload :Connection, :Publisher, :Subscriber, :Recover
 end

data/lib/promiscuous/error/recover.rb

@@ -0,0 +1 @@
+class Promiscuous::Error::Recover < RuntimeError; end

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -17,6 +17,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   end
 
   def connect
+    @queued_messages = 0
     @subscriptions = {}
     self.redis = Promiscuous::Redis.new_celluloid_connection
   end
@@ -106,6 +107,8 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     # when calling worker.pump.start
     return unless self.redis
 
+    bump_message_counter!
+
     return process_message!(msg) unless msg.has_dependencies?
 
     # The message synchronizer only takes care of happens before (>=) dependencies.
@@ -115,7 +118,76 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     end
   end
 
+  def bump_message_counter!
+    @queued_messages += 1
+    maybe_recover
+  end
+
+  def maybe_recover
+    recovery_timeout = Promiscuous::Config.recovery_timeout
+    return unless recovery_timeout
+
+    reset_recovery_timer
+    if should_recover?
+      # We've reached the amount of messages the amqp queue is willing to give us.
+      # We also know that we are not processing messages (@queued_messages is
+      # decremented before we send the message to the runners).
+
+      Promiscuous.warn "[receive] Recovering in #{recovery_timeout} seconds..."
+      @recover_timer = after(recovery_timeout) { reset_recovery_timer; recover }
+    end
+  end
+
+  def reset_recovery_timer
+    @recover_timer.try(:reset)
+    @recover_timer = nil
+  end
+
+  def should_recover?
+    @queued_messages == Promiscuous::Config.prefetch
+  end
+
+  def recover
+    return unless should_recover?
+
+    global_key = Promiscuous::Redis.sub_key('global')
+    current_version = Promiscuous::Redis.get(global_key).to_i
+
+    global_callbacks = get_subscription(global_key).callbacks
+    expected_next_msg_version = global_callbacks.values.map(&:version).min
+    version_to_allow_progress = expected_next_msg_version - 1
+
+    num_messages_to_skip = version_to_allow_progress - current_version
+
+    if num_messages_to_skip > 0
+      recovery_msg = "Recovering. Moving current version from #{current_version} " +
+                     "to #{version_to_allow_progress}. " +
+                     "Skipping #{num_messages_to_skip} messages..."
+    else
+      recovery_msg = "Not recovering. current version is #{current_version}, " +
+                     "while we just need #{version_to_allow_progress}. " +
+                     "Offset is #{num_messages_to_skip} message."
+    end
+
+    e = Promiscuous::Error::Recover.new(recovery_msg)
+    if current_version > 0
+      Promiscuous.error "[receive] #{e}"
+      Promiscuous::Config.error_notifier.try(:call, e)
+    else
+      Promiscuous.info "[receive] #{e}"
+      # Initial sync, nothing to worry about
+    end
+
+    if num_messages_to_skip > 0
+      Promiscuous::Redis.set(global_key, version_to_allow_progress)
+      Promiscuous::Redis.publish(global_key, version_to_allow_progress)
+    end
+  ensure
+    reset_recovery_timer
+  end
+
   def process_message!(msg)
+    @queued_messages -= 1
     Celluloid::Actor[:runners].process!(msg)
   end
 
@@ -136,7 +208,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   end
 
   class Subscription
-    attr_accessor :parent, :key
+    attr_accessor :parent, :key, :callbacks
 
     def initialize(parent, key)
       self.parent = parent

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.50.4'
+  VERSION = '0.51.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.50.4
+  version: 0.51.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-04 00:00:00.000000000 Z
+date: 2013-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -221,19 +221,20 @@ files:
 - lib/promiscuous/error/subscriber.rb
 - lib/promiscuous/error/publisher.rb
 - lib/promiscuous/error/connection.rb
+- lib/promiscuous/error/recover.rb
 - lib/promiscuous/ephemeral.rb
 - lib/promiscuous/autoload.rb
-- lib/promiscuous/error.rb
 - lib/promiscuous/loader.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/common.rb
 - lib/promiscuous/publisher.rb
 - lib/promiscuous/amqp.rb
 - lib/promiscuous/subscriber.rb
-- lib/promiscuous/config.rb
-- lib/promiscuous/cli.rb
 - lib/promiscuous/redis.rb
 - lib/promiscuous/observer.rb
+- lib/promiscuous/cli.rb
+- lib/promiscuous/error.rb
+- lib/promiscuous/config.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous