promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous/config.rb

@@ -1,7 +1,9 @@
 module Promiscuous::Config
   mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url,
-                 :redis_url, :queue_options, :heartbeat, :bareback,
-                 :recovery, :prefetch
+                 :redis_url, :redis_urls, :redis_slave_url, :redis_stats_url,
+                 :stats_interval, :queue_options, :heartbeat, :bareback,
+                 :hash_size, :recovery, :prefetch, :recovery_timeout,
+                 :socket_timeout, :subscriber_threads
 
   def self.backend=(value)
     @@backend = value
@@ -13,17 +15,44 @@ module Promiscuous::Config
     class_variables.each { |var| class_variable_set(var, nil) }
   end
 
-  def self.configure(&block)
-    block.call(self)
+  def self.best_amqp_backend
+    if RUBY_PLATFORM == 'java'
+      begin
+        require 'hot_bunnies'
+        :hot_bunnies
+      rescue LoadError
+        :bunny
+      end
+    else
+      :bunny
+    end
+  end
+
+  def self._configure(&block)
+    block.call(self) if block
 
-    self.app           ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
-    self.amqp_url      ||= 'amqp://guest:guest@localhost:5672'
-    self.redis_url     ||= 'redis://localhost/'
-    self.backend       ||= :rubyamqp
-    self.queue_options ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
-    self.heartbeat     ||= 60
-    self.prefetch      ||= 1000
-    self.logger        ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
+    self.app              ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
+    self.amqp_url         ||= 'amqp://guest:guest@localhost:5672'
+    self.redis_url        ||= 'redis://localhost/'
+    self.redis_urls       ||= [self.redis_url]
+    #self.redis_slave_url  ||= nil
+    self.redis_stats_url  ||= self.redis_urls.first
+    self.stats_interval   ||= 0
+    self.socket_timeout   ||= 10
+    self.backend          ||= best_amqp_backend
+    self.queue_options    ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
+    self.heartbeat        ||= 60
+    self.bareback         ||= false
+    self.hash_size        ||= 2**20 # one million keys ~ 200Mb.
+    self.recovery         ||= false
+    self.prefetch         ||= 1000
+    self.recovery_timeout ||= 10
+    self.logger           ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
+    self.subscriber_threads ||= 10
+  end
+
+  def self.configure(&block)
+    self._configure(&block)
 
     unless self.app
       raise "Promiscuous.configure: please give a name to your app with \"config.app = 'your_app_name'\""
@@ -31,6 +60,38 @@ module Promiscuous::Config
 
     # amqp connection is done in when setting the backend
     Promiscuous::Redis.connect
+
+    hook_fork
+  end
+
+  def self.hook_fork
+    return if @fork_hooked
+
+    Kernel.module_eval do
+      alias_method :fork_without_promiscuous, :fork
+
+      def fork(&block)
+        Promiscuous.disconnect
+        pid = if block
+          fork_without_promiscuous do
+            Promiscuous.connect
+            block.call
+          end
+        else
+          fork_without_promiscuous
+        end
+        Promiscuous.connect
+        pid
+      rescue Exception => e
+        puts e
+        puts e.backtrace.join("\n")
+        raise e
+      end
+
+      module_function :fork
+    end
+
+    @fork_hooked = true
   end
 
   def self.configured?

data/lib/promiscuous/convenience.rb

@@ -0,0 +1,18 @@
+module Promiscuous::Convenience
+  extend self
+
+  def without_promiscuous
+    raise "No block given" unless block_given?
+    old_disabled, Promiscuous.disabled = Promiscuous.disabled, true
+    yield
+  ensure
+    Promiscuous.disabled = old_disabled
+  end
+end
+
+class ::Array
+  def without_promiscuous
+    raise "What is this block?" if block_given?
+    self
+  end
+end

data/lib/promiscuous/dependency.rb

@@ -0,0 +1,59 @@
+require 'fnv'
+
+class Promiscuous::Dependency
+  attr_accessor :internal_key, :version
+
+  def initialize(*args)
+    @internal_key = args.join('/')
+
+    if @internal_key =~ /^[0-9]+$/
+      @internal_key = @internal_key.to_i
+      @hash = @internal_key
+    else
+      @hash = FNV.new.fnv1a_32(@internal_key)
+
+      if Promiscuous::Config.hash_size.to_i > 0
+        # We hash dependencies to have a O(1) memory footprint in Redis.
+        # The hashing needs to be deterministic across instances in order to
+        # function properly.
+        @internal_key = @hash % Promiscuous::Config.hash_size.to_i
+        @hash = @internal_key
+      end
+    end
+  end
+
+  def key(role)
+    Promiscuous::Key.new(role).join(@internal_key)
+  end
+
+  def redis_node(distributed_redis=nil)
+    distributed_redis ||= Promiscuous::Redis.master
+    distributed_redis.nodes[@hash % distributed_redis.nodes.size]
+  end
+
+  def as_json(options={})
+    @version ? [@internal_key, @version].join(':') : @internal_key
+  end
+
+  def self.parse(payload)
+    case payload
+    when /^(.+):([0-9]+)$/ then new($1).tap { |d| d.version = $2.to_i }
+    when /^(.+)$/          then new($1)
+    end
+  end
+
+  def to_s
+    as_json.to_s
+  end
+
+  # We need the eql? method to function properly (we use ==, uniq, ...) in operation
+  # XXX The version is not taken in account.
+  def eql?(other)
+    self.internal_key == other.internal_key
+  end
+  alias == eql?
+
+  def hash
+    self.internal_key.hash
+  end
+end

data/lib/promiscuous/dsl.rb

@@ -0,0 +1,36 @@
+module Promiscuous::DSL
+  def define(&block)
+    instance_eval(&block)
+  end
+
+  def publish(model, options={}, &block)
+    Definition.new(:publish, model, options).instance_eval(&block)
+  end
+
+  def subscribe(model, options={}, &block)
+    Definition.new(:subscribe, model, options).instance_eval(&block)
+  end
+
+  class Definition
+    def initialize(mode, model, options)
+      @mode = mode
+      @model = model
+      @options = options
+      @model_class = @model.to_s.singularize.classify.constantize
+
+      promiscuous_include = mode == :publish ? Promiscuous::Publisher : Promiscuous::Subscriber
+      @model_class.class_eval { include promiscuous_include }
+    end
+
+    def attributes(*fields)
+      options = fields.extract_options!
+      @model_class.__send__(@mode, *fields, @options.merge(options))
+    end
+
+    def track_dependencies_of(field)
+      @model_class.track_dependencies_of(field)
+    end
+
+    alias attribute attributes
+  end
+end

data/lib/promiscuous/error.rb

@@ -1,4 +1,6 @@
 module Promiscuous::Error
   extend Promiscuous::Autoload
-  autoload :Connection, :Publisher, :Subscriber, :Recover
+  autoload :Base, :Connection, :Publisher, :Subscriber, :Recovery,
+           :Dependency, :MissingContext, :AlreadyProcessed,
+           :LockUnavailable, :LostLock
 end

data/lib/promiscuous/error/already_processed.rb

@@ -0,0 +1,5 @@
+class Promiscuous::Error::AlreadyProcessed < Promiscuous::Error::Base
+  def to_s
+    "Skipping message (already processed)"
+  end
+end

data/lib/promiscuous/error/base.rb

@@ -0,0 +1 @@
+class Promiscuous::Error::Base < RuntimeError; end

data/lib/promiscuous/error/connection.rb

@@ -1,17 +1,19 @@
-class Promiscuous::Error::Connection < RuntimeError
+class Promiscuous::Error::Connection < Promiscuous::Error::Base
   attr_accessor :service, :url
 
   def initialize(options={})
     super(nil)
     self.service = options[:service]
-    self.url = Promiscuous::Config.__send__("#{service}_url")
+    self.url = case service
+    when :zookeeper then "zookeeper://#{Promiscuous::Config.zookeeper_hosts}"
+    when :redis     then Promiscuous::Config.redis_url
+    when :amqp      then Promiscuous::Config.amqp_url
+    end
   end
 
   def message
     "Lost connection with #{url}"
   end
 
-  def to_s
-    message
-  end
+  alias to_s message
 end

data/lib/promiscuous/error/dependency.rb

@@ -0,0 +1,111 @@
+class Promiscuous::Error::Dependency < Promiscuous::Error::Base
+  attr_accessor :dependency_solutions, :operation, :context
+
+  def initialize(options={})
+    self.operation = options[:operation]
+    self.context = Promiscuous::Publisher::Context.current
+  end
+
+  # TODO Convert all that with Erb
+
+  def message
+    msg = nil
+    case operation.operation
+    when :read
+      msg =  "Promiscuous doesn't have any tracked dependencies to perform this multi read operation.\n" +
+             "This is what you can do:\n\n" +
+             "  1. Bypass Promiscuous\n\n" +
+             "     If you don't use the result of this operation in your following writes,\n" +
+             "     you can wrap your read query in a 'without_promiscuous { }' block.\n" +
+             "     This is the preferred solution when you are sure that the read doesn't\n" +
+             "     influence the value of a published attribute.\n\n" +
+             "     Rule of thumb: Predicates (methods ending with ?) are often suitable for this use case.\n\n"
+      cnt = 2
+      if operation.operation_ext != :count
+        msg += "  #{cnt}. Synchronize on individual instances\n\n" +
+               "     If the collection you are iterating through is small (<10), it becomes intersting\n" +
+               "     to track instances through their ids instead of the query selector. Example:\n\n" +
+               "          criteria.without_promiscuous.each do |doc|\n" +
+               "            next if doc.should_do_something?\n" +
+               "            doc.reload # tell promiscuous to track the instance\n" +
+               "            doc.do_something!\n" +
+               "          end\n\n"
+        cnt += 1
+      end
+      if operation.selector_keys.present?
+        msg += "  #{cnt}. Track New Dependencies\n\n" +
+               "     Add #{operation.selector_keys.count == 1 ? "the following line" : "one of the following lines"} " +
+                    "in the #{operation.instance.class} model:\n\n" +
+               "       class #{operation.instance.class}\n" +
+                    operation.selector_keys.map { |field| "         track_dependencies_of :#{field}" }.join("\n") + "\n" +
+               "       end\n\n" +
+               (operation.selector_keys.count > 1 ?
+               "     The more specific field, the better. Promiscuous works better when working with small subsets\n" +
+               "     For example, tracking something like 'member_id' is a fairly safe choice.\n\n" : "") +
+               "     Note that dependency tracking slows down your writes. It can be seen as the analogous\n" +
+               "     of an index on a regular database.\n" +
+               "     You may find more information about the implications in the Promiscuous wiki (TODO:link).\n\n"
+      end
+    when :update
+      msg = "Promiscuous cannot track dependencies of a multi update operation.\n" +
+             "This is what you can do:\n\n" +
+             "  1. Instead of doing a multi updates, update each instance separately\n\n" +
+             "  2. Do not assign has_many associations directly, but use the << operator instead.\n\n"
+    when :destroy
+      msg = "Promiscuous cannot track dependencies of a multi delete operation.\n" +
+             "This is what you can do:\n\n" +
+            "   1. Instead of doing a multi delete, delete each instance separatly.\n\n" +
+            "   2. Use destroy_all instead of destroy_all.\n\n" +
+            "   3. Declare your has_many relationships with :dependent => :destroy instead of :delete.\n\n"
+    end
+
+    msg += "#{"-" * 100}\n\n"
+
+    msg += "Promiscuous cannot allow the following "
+    case operation.operation_ext || operation.operation
+    when :count     then msg += 'count'
+    when :mapreduce then msg += 'mapreduce'
+    when :read      then msg += 'each loop'
+    when :update    then msg += 'multi update'
+    when :destroy   then msg += 'multi destroy'
+    end
+    msg += " in the '#{context.name}' context:\n\n"
+    msg += "  #{self.class.explain_operation(self.operation)}"
+    msg += "\n\nProTip: Try again with TRACE=2 in the shell or ENV['TRACE']='2' in the console.\n" unless ENV['TRACE']
+    msg
+  rescue Exception => e
+    "#{e.to_s}\n#{e.backtrace.join("\n")}"
+  end
+
+  def self.explain_operation(operation, limit=100)
+    instance = operation.instance
+    selector   = instance ? get_selector(instance.attributes, limit) : ""
+    class_name = instance ? instance.class : "Unknown"
+
+    if operation.operation == :create
+      "#{instance.class}.create(#{selector})"
+    else
+      case operation.operation_ext || operation.operation
+      when :count     then verb = 'count'
+      when :mapreduce then verb = 'mapreduce(...)'
+      when :read      then verb = operation.multi? ? 'each { ... }' : 'first'
+      when :update    then verb = operation.multi? ? 'update_all'   : 'update'
+      when :destroy   then verb = operation.multi? ? 'delete_all'   : 'delete'
+      end
+      msg = "#{class_name}#{selector.present? ? ".where(#{selector})" : ""}.#{verb}"
+      if operation.operation == :update && operation.respond_to?(:change) && operation.change
+        msg += "(#{get_selector(operation.change, limit)})"
+      end
+      msg
+    end
+  end
+
+  def self.get_selector(attributes, limit=100)
+    attributes = attributes['$set'] if attributes.count == 1 && attributes['$set']
+    selector = attributes.map { |k,v| ":#{k} => #{v}" }.join(", ")
+    selector = "#{selector[0...(limit-3)]}..." if selector.size > limit
+    selector
+  end
+
+  alias to_s message
+end

data/lib/promiscuous/error/lock_unavailable.rb

@@ -0,0 +1,12 @@
+class Promiscuous::Error::LockUnavailable < Promiscuous::Error::Base
+  def initialize(lock)
+    @lock = lock
+  end
+
+  def message
+    "The lock is not available on #{@lock}\n" +
+    "If an app instance died, the lock will expire in less than a minute."
+  end
+
+  alias to_s message
+end

data/lib/promiscuous/error/lost_lock.rb

@@ -0,0 +1,12 @@
+class Promiscuous::Error::LostLock < Promiscuous::Error::Base
+  def initialize(lock)
+    @lock = lock
+  end
+
+  def message
+    "The following lock was lost during the operation and will be recovered if not already done:\n" +
+    "  #{@lock}"
+  end
+
+  alias to_s message
+end

data/lib/promiscuous/error/missing_context.rb

@@ -0,0 +1,29 @@
+class Promiscuous::Error::MissingContext < Promiscuous::Error::Base
+  def message
+    require 'erb'
+    ERB.new(<<-ERB.gsub(/^\s+<%/, '<%').gsub(/^ {6}/, ''), nil, '-').result(binding)
+      Promiscuous needs to execute all your read/write queries in a context for publishing.
+      This is what you can do:
+       1. Wrap your operations in a Promiscuous context yourself (jobs, etc.):
+
+          Promiscuous::Middleware.with_context 'jobs/name' do
+             # Code including all your read and write queries
+           end
+
+       2. Disable Promiscuous completely (only for testing):
+
+           RSpec.configure do |config|
+             config.around do |example|
+               without_promiscuous { example.run }
+             end
+           end
+
+         Note that opening a context will reactivate promiscuous temporarily
+         even if it was disabled.
+
+       3. You are in render() in the Rails controller, and you should not write.
+    ERB
+  end
+
+  alias to_s message
+end

data/lib/promiscuous/error/publisher.rb

@@ -1,5 +1,5 @@
-class Promiscuous::Error::Publisher < RuntimeError
-  attr_accessor :inner, :instance, :payload, :out_of_sync
+class Promiscuous::Error::Publisher < Promiscuous::Error::Base
+  attr_accessor :inner, :instance, :payload
 
   def initialize(inner, options={})
     super(nil)
@@ -8,24 +8,14 @@ class Promiscuous::Error::Publisher < RuntimeError
     self.inner = inner
     self.instance = options[:instance]
     self.payload  = options[:payload]
-    self.out_of_sync = options[:out_of_sync]
   end
 
   def message
     msg = "#{inner.class}: #{inner.message}"
-    if instance
-      msg = "#{msg} while publishing #{instance.inspect}"
-      msg = "FATAL (out of sync) #{msg}" if out_of_sync
-    end
-
-    if payload
-      msg = "#{msg} payload: #{payload}"
-    end
-
+    msg = "#{msg} while publishing #{instance.inspect}" if instance
+    msg = "#{msg} payload: #{payload}" if payload
     msg
   end
 
-  def to_s
-    message
-  end
+  alias to_s message
 end