promiscuous 0.91.0 → 0.92.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b84ccb396c096ef9d2164d4400356606f90c3954
-  data.tar.gz: 31da27f6d1163623df3014dc15a0641d6e93c0c3
+  metadata.gz: d772f9b9786ef7ee6787030113bf9ec5934717d5
+  data.tar.gz: 163c9cb61d4b359f1e6766effef9baccdfd8c229
 SHA512:
-  metadata.gz: 2cec057b926337d6940a8d3144b88ccc3dcf200a35dc74d88f69bf4ad388d034b6e68a04dd2f270f5c581382bef3ae5985b6e4ad9e7d9cfe44babf6cdacabd02
-  data.tar.gz: d84a82ecdcb22041d72c08e8c9fb443e77c2c9a74c49589a6d209079df7055f1adc484ca12b32d6f75f9a04e8f66bcfbaefeff020eb47b38c6fecbe3e1bc0c0b
+  metadata.gz: 6cacc43f226549868f963abbad29e7ba7249a1bbc1c4eb3e0b8014275935ca37e7455a73b48e36d7c0c56906308ab334c104bbf136a7cd2c7345ee976161eead
+  data.tar.gz: 6f59e694ed2c89e2ebe9a6c73995c4b6e4987ee1f43d171f55f3783b9698c4974aff8ef684d22dd8e652956d943081b952f8bb313d492016629b9564a1447727

data/lib/promiscuous.rb

@@ -13,7 +13,7 @@ module Promiscuous
   require_for 'rails',   'promiscuous/railtie'
   require_for 'resque',  'promiscuous/resque'
   require_for 'sidekiq', 'promiscuous/sidekiq'
-
+  require_for 'mongoid', 'promiscuous/mongoid'
 
   extend Promiscuous::Autoload
   autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,

data/lib/promiscuous/amqp.rb

@@ -1,6 +1,6 @@
 module Promiscuous::AMQP
   extend Promiscuous::Autoload
-  autoload :HotBunnies, :Bunny, :Null, :Fake
+  autoload :HotBunnies, :Bunny, :Null, :Fake, :File
 
   LIVE_EXCHANGE      = 'promiscuous'
   BOOTSTRAP_EXCHANGE = 'promiscuous.bootstrap'

data/lib/promiscuous/amqp/fake.rb

@@ -38,9 +38,6 @@ class Promiscuous::AMQP::Fake
     msg && JSON.parse(msg[:payload])
   end
 
-  def open_queue(options={}, &block)
-  end
-
   module Subscriber
     def subscribe(options={}, &block)
     end

data/lib/promiscuous/amqp/file.rb

@@ -0,0 +1,81 @@
+class Promiscuous::AMQP::File
+  def connect
+  end
+
+  def disconnect
+  end
+
+  def connected?
+    true
+  end
+
+  def new_connection(options={})
+  end
+
+  def publish(options={})
+    options[:on_confirm].try(:call)
+    raise NotImplemented
+  end
+
+  module Subscriber
+    attr_accessor :lock, :prefetch_wait, :num_pending
+
+    def subscribe(options={}, &block)
+      file_name, worker_index, num_workers = Promiscuous::Config.subscriber_amqp_url.split(':')
+
+      worker_index = worker_index.to_i
+      num_workers = num_workers.to_i
+
+      file = File.open(file_name, 'r')
+
+      @prefetch = Promiscuous::Config.prefetch
+      @num_pending = 0
+      @lock = Mutex.new
+      @prefetch_wait = ConditionVariable.new
+
+      @thread = Thread.new do
+        file.each_with_index do |line, i|
+          if num_workers > 0
+            next if ((i+worker_index) % num_workers) != 0
+          end
+
+          return if @stop
+
+          @lock.synchronize do
+            @prefetch_wait.wait(@lock) until @num_pending < @prefetch
+            @num_pending += 1
+          end
+
+          block.call(Metadata.new(self), line.chomp)
+        end
+
+        @lock.synchronize do
+          @prefetch_wait.wait(@lock) until @num_pending == 0
+        end
+
+        # will shutdown the CLI gracefully
+        Process.kill("SIGTERM", Process.pid)
+      end
+    end
+
+    def recover
+    end
+
+    def disconnect
+      @stop = true
+    end
+
+    class Metadata
+      def initialize(sub)
+        @sub = sub
+      end
+
+      def ack
+        @sub.lock.synchronize do
+          @sub.num_pending -= 1
+          @sub.prefetch_wait.signal
+        end
+      end
+    end
+  end
+end

data/lib/promiscuous/amqp/null.rb

@@ -12,7 +12,4 @@ class Promiscuous::AMQP::Null
   def publish(options={})
     options[:on_confirm].try(:call)
   end
-
-  def open_queue(options={}, &block)
-  end
 end

data/lib/promiscuous/cli.rb

@@ -3,22 +3,28 @@ class Promiscuous::CLI
 
   def trap_debug_signals
     Signal.trap 'SIGUSR2' do
-      Thread.list.each do |thread|
-        print_status  '----[ Threads ]----' + '-' * (100-19)
-        if thread.backtrace
-          print_status "Thread #{thread} #{thread['label']}"
-          print_status thread.backtrace.join("\n")
-        else
-          print_status "Thread #{thread} #{thread['label']} -- no backtrace"
+      # Using a thread because we cannot acquire mutexes in a trap context in
+      # ruby 2.0
+      Thread.new do
+        Thread.list.each do |thread|
+          next if Thread.current == thread
+
+          print_status  '----[ Threads ]----' + '-' * (100-19)
+          if thread.backtrace
+            print_status "Thread #{thread} #{thread['label']}"
+            print_status thread.backtrace.join("\n")
+          else
+            print_status "Thread #{thread} #{thread['label']} -- no backtrace"
+          end
         end
-      end
 
-      if @worker && @worker.respond_to?(:message_synchronizer)
-        if blocked_messages = @worker.message_synchronizer.try(:blocked_messages)
-          print_status  '----[ Pending Dependencies ]----' + '-' * (100-32)
-          blocked_messages.reverse_each { |msg| print_status msg }
+        if @worker && @worker.respond_to?(:message_synchronizer)
+          if blocked_messages = @worker.message_synchronizer.try(:blocked_messages)
+            print_status  '----[ Pending Dependencies ]----' + '-' * (100-32)
+            blocked_messages.reverse_each { |msg| print_status msg }
+          end
+          print_status  '-' * 100
         end
-        print_status  '-' * 100
       end
     end
   end
@@ -26,14 +32,18 @@ class Promiscuous::CLI
   def trap_exit_signals
     %w(SIGTERM SIGINT).each do |signal|
       Signal.trap(signal) do
-        print_status "Exiting..."
-        if @stop
-          @worker.try(:show_stop_status)
-        else
-          @stop = true
-          @worker.try(:stop)
-          @worker = nil
-        end
+        # Using a thread because we cannot acquire mutexes in a trap context in
+        # ruby 2.0
+        Thread.new do
+          print_status "Exiting..."
+          if @stop
+            @worker.try(:show_stop_status)
+          else
+            @stop = true
+            @worker.try(:stop)
+            @worker = nil
+          end
+        end.join
       end
     end
   end
@@ -144,12 +154,12 @@ class Promiscuous::CLI
         Promiscuous::Config.no_deps = true
       end
 
-      opts.on "-l", "--require FILE", "File to require to load your app. Don't worry about it with rails" do |file|
-        options[:require] = file
+      opts.on "-x", "--ignore-exceptions", "Ignore exceptions and continue to process messages" do
+        Promiscuous::Config.ignore_exceptions = true
       end
 
-      opts.on "-r", "--recovery", "Run in recovery mode" do
-        Promiscuous::Config.recovery = true
+      opts.on "-l", "--require FILE", "File to require to load your app. Don't worry about it with rails" do |file|
+        options[:require] = file
       end
 
       opts.on "-p", "--prefetch [NUM]", "Number of messages to prefetch" do |prefetch|

data/lib/promiscuous/config.rb

@@ -3,9 +3,10 @@ module Promiscuous::Config
                  :publisher_amqp_url, :subscriber_amqp_url, :publisher_exchange,
                  :subscriber_exchanges, :queue_name, :queue_options, :redis_url,
                  :redis_urls, :redis_stats_url, :stats_interval,
-                 :socket_timeout, :heartbeat, :no_deps, :hash_size, :recovery,
+                 :socket_timeout, :heartbeat, :no_deps, :hash_size,
                  :prefetch, :recovery_timeout, :logger, :subscriber_threads,
-                 :version_field, :error_notifier, :recovery_on_boot
+                 :version_field, :error_notifier, :recovery_on_boot,
+                 :on_stats, :ignore_exceptions, :consistency, :max_retries, :generation
 
   def self.backend=(value)
     @@backend = value
@@ -53,7 +54,6 @@ module Promiscuous::Config
     self.heartbeat            ||= 60
     self.no_deps              ||= false
     self.hash_size            ||= 2**20 # one million keys ~ 200Mb.
-    self.recovery             ||= false
     self.prefetch             ||= self.bootstrap ? 10000000 : 1000
     self.recovery_timeout     ||= 10
     self.logger               ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
@@ -61,6 +61,11 @@ module Promiscuous::Config
     self.error_notifier       ||= proc {}
     self.version_field        ||= '_v'
     self.recovery_on_boot     = true if self.recovery_on_boot.nil?
+    self.on_stats             ||= proc { |rate, latency| }
+    self.ignore_exceptions    ||= false
+    self.consistency          ||= :eventual
+    self.max_retries          ||= 10
+    self.generation           ||= 1
   end
 
   def self.configure(&block)

data/lib/promiscuous/error.rb

@@ -1,6 +1,5 @@
 module Promiscuous::Error
   extend Promiscuous::Autoload
   autoload :Base, :Connection, :Publisher, :Subscriber, :Recovery,
-           :Dependency, :MissingContext, :AlreadyProcessed,
-           :LockUnavailable, :LostLock
+           :Dependency, :MissingContext, :LockUnavailable, :LostLock
 end

data/lib/promiscuous/key.rb

@@ -1,21 +1,10 @@
 class Promiscuous::Key
-  def initialize(role, nodes=[], no_join=nil)
+  def initialize(role, nodes=[])
     @role = role
     @nodes = nodes
-    @no_join = no_join
   end
 
   def join(*nodes)
-    # --- backward compatiblity code ---
-    # TODO remove code
-    if nodes == ['global', nil, nil]
-      return self.class.new(@role, @nodes + nodes, :no_join)
-    end
-    if @no_join
-      return self.class.new(@role, @nodes)
-    end
-    # --- backward compatiblity code ---
-
     self.class.new(@role, @nodes + nodes)
   end
 

data/lib/promiscuous/mongoid.rb

@@ -0,0 +1,7 @@
+class Moped::BSON::ObjectId
+  # No {"$oid": "123"}, it's horrible.
+  # TODO Document this shit.
+  def to_json(*args)
+    "\"#{to_s}\""
+  end
+end

data/lib/promiscuous/publisher/context/base.rb

@@ -24,11 +24,11 @@ class Promiscuous::Publisher::Context::Base
     end
   end
 
-  attr_accessor :name, :read_operations, :extra_dependencies, :current_user_id
+  attr_accessor :name, :read_operations, :extra_dependencies, :current_user
 
-  def initialize(*args)
-    @name = args[0].try(:to_s) || 'anonymous'
-    @current_user_id = args[1]
+  def initialize(name=nil, options={})
+    @name = name.try(:to_s) || 'anonymous'
+    @current_user = options[:current_user]
     @read_operations = []
     @extra_dependencies = []
     @transaction_managers = {}

data/lib/promiscuous/publisher/context/middleware.rb

@@ -4,12 +4,8 @@ class Promiscuous::Publisher::Context::Middleware < Promiscuous::Publisher::Cont
 
     def process_action(*args)
       full_name = "#{self.class.controller_path}/#{self.action_name}"
-      current_user_id = self.respond_to?(:current_user) ? self.current_user.try(:id) : nil
-      Promiscuous::Publisher::Context::Middleware.with_context(full_name, current_user_id) { super }
-    end
-
-    def render(*args)
-      Promiscuous::Publisher::Context::Middleware.without_context { super }
+      current_user = self.current_user if self.respond_to?(:current_user)
+      Promiscuous::Publisher::Context::Middleware.with_context(full_name, :current_user => current_user) { super }
     end
   end
 
@@ -26,23 +22,6 @@ class Promiscuous::Publisher::Context::Middleware < Promiscuous::Publisher::Cont
     Promiscuous.disabled = old_disabled
   end
 
-  def self.without_context
-    # This is different from the method without_promiscuous in convenience.rb
-    # That's used for render() and things that are *not* supposed to write.
-    # We actually force promiscuous to instrument queries, and make sure that
-    # we don't do any writes we shouldn't.
-    old_disabled, Promiscuous.disabled = Promiscuous.disabled?, false
-    old_current, self.current = self.current, nil
-    yield
-  rescue Exception => e
-    $promiscuous_last_exception = e if e.is_a? Promiscuous::Error::Base
-    pretty_print_exception(e) unless e.is_a? ActionView::MissingTemplate
-    raise e
-  ensure
-    self.current = old_current
-    Promiscuous.disabled = old_disabled
-  end
-
   def self.pretty_print_exception(e)
     return if $promiscuous_pretty_print_exception_once == :disable || ENV['RAILS_ENV'] == 'production'
     return if e.is_a?(SystemExit)

data/lib/promiscuous/publisher/model/ephemeral.rb

@@ -33,13 +33,17 @@ module Promiscuous::Publisher::Model::Ephemeral
     operation = :update  unless self.new_record
     operation = :destroy if     self.destroyed
 
-    Promiscuous::Publisher::Operation::Atomic.new(:instance => self, :operation => operation).execute {}
+    save_operation(operation)
 
     self.new_record = false
     true
   end
   alias :save! :save
 
+  def save_operation(operation)
+    Promiscuous::Publisher::Operation::Ephemeral.new(:instance => self, :operation => operation).execute
+  end
+
   def update_attributes(attrs)
     attrs.each { |attr, value| __send__("#{attr}=", value) }
     save

data/lib/promiscuous/publisher/model/mock.rb

@@ -22,15 +22,17 @@ module Promiscuous::Publisher::Model::Mock
     end
   end
 
-  class PromiscuousMethods
-    include Promiscuous::Publisher::Model::Base::PromiscuousMethodsBase
-    include Promiscuous::Publisher::Model::Ephemeral::PromiscuousMethodsEphemeral
+  def save_operation(operation)
+    payload = nil
 
-    def sync(options={}, &block)
-      payload[:operations] = [self.payload.merge(:operation => options[:operation] || :update)]
-      payload[:app] = self.class.mock_options[:from]
-      Promiscuous::Subscriber::Worker::Message.new(MultiJson.dump(payload)).process
+    Promiscuous::Publisher::Context::Middleware.with_context("mocking #{self.class}") do
+      op = Promiscuous::Publisher::Operation::Ephemeral.new(:instance => self, :operation => operation)
+      # TODO FIX the mocks to populate app name, also we need to hook before the
+      # json dump.
+      payload = op.generate_payload
     end
+
+    Promiscuous::Subscriber::Worker::Message.new(payload).process
   end
 
   module ClassMethods

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -23,7 +23,9 @@ module Promiscuous::Publisher::Model::Mongoid
 
     def sync(options={}, &block)
       raise "Use promiscuous.sync on the parent instance" if @instance.embedded?
-      super
+
+      # We can use the ephemeral because both are mongoid and ephemerals are atomic operations.
+      Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance, :operation => :update).execute
     end
 
     def attribute(attr)

data/lib/promiscuous/publisher/operation.rb

@@ -1,4 +1,4 @@
 module Promiscuous::Publisher::Operation
   extend Promiscuous::Autoload
-  autoload :Base, :Transaction, :Atomic, :NonPersistent, :ProxyForQuery
+  autoload :Base, :Transaction, :Atomic, :NonPersistent, :ProxyForQuery, :Ephemeral
 end

data/lib/promiscuous/publisher/operation/atomic.rb

@@ -35,10 +35,47 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
     end
   end
 
-  def execute_instrumented(query)
-    raise if @instance.nil? # assert()
+  def do_database_query(query)
+    case operation
+    when :create
+      # We don't stash the version in the document as we can't have races
+      # on the same document.
+    when :update
+      increment_version_in_document
+      # We are now in the possession of an instance that matches the original
+      # selector. We need to make sure the db query will operate on it,
+      # instead of the original selector.
+      use_id_selector(:use_atomic_version_selector => true)
+      # We need to use an atomic versioned selector to make sure that
+      # if we lose the lock for a long period of time, we don't mess up
+      # the record. Perhaps the operation has been recovered a while ago.
+    when :destroy
+      use_id_selector
+    end
+
+    # The driver is responsible to set instance to the appropriate value.
+    query.call_and_remember_result(:instrumented)
+
+    if query.failed?
+      # If we get an network failure, we should retry later.
+      return if recoverable_failure?(query.exception)
+      @instance = nil
+    end
+  end
+
+  def yell_about_missing_instance
+    err = "Cannot find document. Database had a dataloss?. Proceeding anyways. #{@recovery_data}"
+    e = Promiscuous::Error::Recovery.new(err)
+    Promiscuous.warn "[recovery] #{e}"
+    Promiscuous::Config.error_notifier.call(e)
+  end
 
-    unless self.recovering?
+  def execute_instrumented(query)
+    if recovering?
+      # The DB died or something. We cannot find our instance any more :(
+      # this is a problem, but we need to publish.
+      yell_about_missing_instance if @instance.nil?
+    else
       generate_read_dependencies
       acquire_op_lock
 
@@ -75,32 +112,7 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
     # documents are missing on our side to be able to resend the destroy
     # message.
 
-    case operation
-    when :create
-      # We don't stash the version in the document as we can't have races
-      # on the same document.
-    when :update
-      stash_version_in_document(@committed_write_deps.first.version)
-      # We are now in the possession of an instance that matches the original
-      # selector. We need to make sure the db query will operate on it,
-      # instead of the original selector.
-      use_id_selector(:use_atomic_version_selector => true)
-      # We need to use an atomic versioned selector to make sure that
-      # if we lose the lock for a long period of time, we don't mess up
-      # the record. Perhaps the operation has been recovered a while ago.
-    when :destroy
-      use_id_selector
-    end
-
-    # The driver is responsible to set instance to the appropriate value.
-    query.call_and_remember_result(:instrumented)
-
-    if query.failed?
-      # If we get an network failure, we should retry later.
-      return if recoverable_failure?(query.exception)
-      @instance = nil
-    end
-
+    do_database_query(query) unless @instance.nil?
     # We take a timestamp right after the write is performed because latency
     # measurements are performed on the subscriber.
     record_timestamp
@@ -141,6 +153,7 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
   def fetch_instance
     # This method is overridden to use the original query selector.
     # Should return nil if the instance is not found.
+    @instance.reload if @instance.respond_to?(:reload)
     @instance
   end
 
@@ -148,9 +161,8 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
     @instance = fetch_instance
   end
 
-  def stash_version_in_document(version)
-    # Overridden to update the query to set the version field with:
-    # instance[Promiscuous::Config.version_field] = version
+  def increment_version_in_document
+    # Overridden to increment version field in the query
   end
 
   def use_id_selector(options={})