promiscuous 0.92.0 → 0.100.0

data/lib/promiscuous/publisher/context/middleware.rb

@@ -1,94 +0,0 @@
-class Promiscuous::Publisher::Context::Middleware < Promiscuous::Publisher::Context::Base
-  module Controller
-    extend ActiveSupport::Concern
-
-    def process_action(*args)
-      full_name = "#{self.class.controller_path}/#{self.action_name}"
-      current_user = self.current_user if self.respond_to?(:current_user)
-      Promiscuous::Publisher::Context::Middleware.with_context(full_name, :current_user => current_user) { super }
-    end
-  end
-
-  def self.with_context(*args, &block)
-    # XXX We turn off the disabled flag when entering a middleware.
-    # It has priority because it's much simpler to use for testing.
-    old_disabled, Promiscuous.disabled = Promiscuous.disabled?, false
-    super
-  rescue Exception => e
-    $promiscuous_last_exception = e if e.is_a? Promiscuous::Error::Base
-    pretty_print_exception(e) unless e.is_a? ActionView::MissingTemplate
-    raise e
-  ensure
-    Promiscuous.disabled = old_disabled
-  end
-
-  def self.pretty_print_exception(e)
-    return if $promiscuous_pretty_print_exception_once == :disable || ENV['RAILS_ENV'] == 'production'
-    return if e.is_a?(SystemExit)
-
-    e = e.original_exception if defined?(ActionView::Template::Error) && e.is_a?(ActionView::Template::Error)
-
-    STDERR.puts
-    STDERR.puts "\e[0;#{36}m/---[ Exception: #{e.class} ]#{'-'*[0, 84 - e.class.name.size].max}\e[0m"
-    STDERR.puts "\e[0;#{36}m|"
-
-    highlight_indent = false
-    msg = e.to_s.split("\n").map do |line|
-      highlight_indent = true if line =~ /The problem comes from the following/ ||
-                                 line =~ /Promiscuous is tracking this read/
-      line = "\e[1;#{31}m#{line}\e[0;#{31}m" if highlight_indent && line =~ /^  /
-      "\e[0;#{36}m|  \e[0;#{31}m#{line}\e[0m"
-    end
-
-    STDERR.puts msg.join("\n")
-    STDERR.puts "\e[0;#{36}m|"
-    STDERR.puts "\e[0;#{36}m+---[ Backtrace ]--------------------------------------------------------------------------------------\e[0m"
-    STDERR.puts "\e[0;#{36}m|"
-
-    expand = ENV['TRACE'].to_i > 1
-    bt = e.backtrace.map do |line|
-       line = case line
-              when /(rspec-core|instrumentation)/
-                "\e[1;30m#{line}\e[0m" if expand
-              when /#{Rails.root}\/app\/controllers/
-                "\e[1;35m#{line}\e[0m"
-              when /#{Rails.root}\/app\/models/
-                "\e[1;33m#{line}\e[0m"
-              when /#{Rails.root}\/lib/
-                "\e[1;34m#{line}\e[0m"
-              when /(mongoid|active_record).*`(count|distinct|each|first|last)'$/
-                "\e[1;32m#{line}\e[0m"
-              when /(mongoid|active_record).*`(create|insert|save|update|modify|remove|remove_all)'$/
-                "\e[1;31m#{line}\e[0m"
-              when /#{Rails.root}/
-                if line =~ /\/support\//
-                  "\e[1;30m#{line}\e[0m" if expand
-                else
-                  "\e[1;36m#{line}\e[0m"
-                end
-              else
-                "\e[1;30m#{line}\e[0m" if expand
-              end
-       "\e[0;#{36}m|  #{line}" if line
-      end
-      .compact
-      .join("\n")
-    STDERR.puts bt
-    STDERR.puts "\e[0;#{36}m|"
-
-    if $cucumber_extra
-      STDERR.puts "\e[0;#{36}m+---[ Cucumber ]--------------------------------------------------------------------------------------\e[0m"
-      STDERR.puts "\e[0;#{36}m|"
-      $cucumber_extra.each_with_index do |line, i|
-        line = line.gsub(/([^:]*: )(.*)$/, "\\1\e[1;36m\\2")
-        STDERR.puts "\e[0;#{36}m| \e[0;36m#{line}\e[0m"
-        STDERR.puts "\e[0;#{36}m|" if i.zero?
-      end
-      STDERR.puts "\e[0;#{36}m|"
-    end
-
-    STDERR.puts "\e[0;#{36}m\\------------------------------------------------------------------------------------------------------\e[0m"
-    STDERR.puts
-    $promiscuous_pretty_print_exception_once = :disable if $promiscuous_pretty_print_exception_once
-  end
-end

data/lib/promiscuous/resque.rb

@@ -1,12 +0,0 @@
-require 'resque/job'
-
-class Resque::Job
-  alias_method :perform_without_promiscuous, :perform
-
-  def perform
-    name = "resque/#{payload_class.name.underscore}"
-    Promiscuous::Middleware.with_context(name) do
-      perform_without_promiscuous
-    end
-  end
-end

data/lib/promiscuous/sidekiq.rb

@@ -1,15 +0,0 @@
-module Sidekiq
-  class Promiscuous
-    def call(worker_class, item, queue)
-      ::Promiscuous::Middleware.with_context "sidekiq/#{item['queue']}/#{worker_class.class.to_s.underscore}" do
-        yield
-      end
-    end
-  end
-end
-
-Sidekiq.configure_server do |config|
-  config.server_middleware do |chain|
-    chain.add Sidekiq::Promiscuous
-  end
-end

data/lib/promiscuous/subscriber/message_processor.rb

@@ -1,4 +0,0 @@
-module Promiscuous::Subscriber::MessageProcessor
-  extend Promiscuous::Autoload
-  autoload :Base, :Regular, :Bootstrap
-end

data/lib/promiscuous/subscriber/message_processor/base.rb

@@ -1,54 +0,0 @@
-class Promiscuous::Subscriber::MessageProcessor::Base
-  attr_accessor :message
-
-  def initialize(message)
-    self.message = message
-  end
-
-  def operations
-    message.parsed_payload['operations'].map { |op| operation_class.new(op) }
-  end
-
-  def self.process(*args)
-    raise "Same thread is processing a message?" if self.current
-
-    begin
-      self.current = new(*args)
-      self.current.process_message
-    ensure
-      self.current = nil
-    end
-  end
-
-  def self.current
-    Thread.current[:promiscuous_message_processor]
-  end
-
-  def self.current=(value)
-    Thread.current[:promiscuous_message_processor] = value
-  end
-
-  def process_message
-    begin
-      on_message
-    rescue Exception => e
-      @fail_count ||= 0;  @fail_count += 1
-
-      if @fail_count <= Promiscuous::Config.max_retries
-        Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
-        sleep @fail_count ** 2
-        process_message
-      else
-        raise e
-      end
-    end
-  end
-
-  def on_message
-    raise "Must be implemented"
-  end
-
-  def operation_class
-    raise "Must be implemented"
-  end
-end

data/lib/promiscuous/subscriber/message_processor/bootstrap.rb

@@ -1,17 +0,0 @@
-class Promiscuous::Subscriber::MessageProcessor::Bootstrap < Promiscuous::Subscriber::MessageProcessor::Base
-  def on_message
-    if bootstrap_operation?
-      operations.each(&:execute)
-    else
-      # Postpone message by doing nothing
-    end
-  end
-
-  def bootstrap_operation?
-    operations.first.try(:operation) =~ /bootstrap/
-  end
-
-  def operation_class
-    Promiscuous::Subscriber::Operation::Bootstrap
-  end
-end

data/lib/promiscuous/subscriber/message_processor/regular.rb

@@ -1,238 +0,0 @@
-class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscriber::MessageProcessor::Base
-  delegate :write_dependencies, :read_dependencies, :dependencies, :to => :message
-
-  def nodes_with_deps
-    @nodes_with_deps ||= dependencies.group_by(&:redis_node)
-  end
-
-  def instance_dep
-    @instance_dep ||= write_dependencies.first
-  end
-
-  def master_node
-    @master_node ||= instance_dep.redis_node
-  end
-
-  def master_node_with_deps
-    @master_node_with_deps ||= nodes_with_deps.select { |node| node == master_node }.first
-  end
-
-  def secondary_nodes_with_deps
-    @secondary_nodes_with_deps ||= nodes_with_deps.reject { |node| node == master_node }.to_a
-  end
-
-  def recovery_key
-    # We use a recovery_key unique to the operation to avoid any trouble of
-    # touching another operation.
-    @recovery_key ||= instance_dep.key(:sub).join(instance_dep.version).to_s
-  end
-
-  def get_current_instance_version
-    master_node.get(instance_dep.key(:sub).join('rw').to_s).to_i
-  end
-
-  # XXX TODO Code is not tolerant to losing a lock.
-
-  def update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options={})
-    argv = []
-    argv << MultiJson.dump([r_deps.map { |dep| dep.key(:sub) },
-                            w_deps.map { |dep| dep.key(:sub) },
-                            w_deps.map { |dep| dep.version }])
-    argv << recovery_key if options[:with_recovery]
-
-    @@update_script_bootstrap ||= Promiscuous::Redis::Script.new <<-SCRIPT
-      local _args = cjson.decode(ARGV[1])
-      local read_deps = _args[1]
-      local write_deps = _args[2]
-      local write_versions = _args[3]
-      local recovery_key = ARGV[2]
-
-      if recovery_key and redis.call('exists', recovery_key) == 1 then
-        return
-      end
-
-      for i, _key in ipairs(read_deps) do
-        local key = _key .. ':rw'
-        local v = redis.call('incr', key)
-        redis.call('publish', key, v)
-      end
-
-      for i, _key in ipairs(write_deps) do
-        local key = _key .. ':rw'
-        local v = write_versions[i]
-        local current_version = tonumber(redis.call('get', key)) or 0
-        if current_version < v then
-          redis.call('set', key, v)
-          redis.call('publish', key, v)
-        end
-      end
-
-      if recovery_key then
-        redis.call('set', recovery_key, 'done')
-      end
-    SCRIPT
-
-    @@update_script_bootstrap.eval(node, :argv => argv)
-  end
-
-  def update_dependencies_fast(node, r_deps, w_deps, options={})
-    keys = (r_deps + w_deps).map { |dep| dep.key(:sub) }
-    argv = options[:with_recovery] ? [recovery_key] : []
-
-    @@update_script_fast ||= Promiscuous::Redis::Script.new <<-SCRIPT
-      local deps = KEYS
-      local recovery_key = ARGV[1]
-
-      if recovery_key and redis.call('exists', recovery_key) == 1 then
-        return
-      end
-
-      for i, _key in ipairs(deps) do
-        local key = _key .. ':rw'
-        local v = redis.call('incr', key)
-        redis.call('publish', key, v)
-      end
-
-      if recovery_key then
-        redis.call('set', recovery_key, 'done')
-      end
-    SCRIPT
-
-    @@update_script_fast.eval(node, :keys => keys, :argv => argv)
-  end
-
-  def update_dependencies_on_node(node_with_deps, options={})
-    # Read and write dependencies are not handled the same way:
-    # * Read dependencies are just incremented (which allow parallelization).
-    # * Write dependencies are set to be max(current_version, received_version).
-    #   This allow the version bootstrapping process to be non-atomic.
-    #   Publishers upgrade their reads dependencies to write dependencies
-    #   during bootstrapping to permit the mechanism to function properly.
-
-    # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
-    # worth optimizing it for the non-bootstrap case.
-
-    node = node_with_deps[0]
-    r_deps = node_with_deps[1].select(&:read?)
-    w_deps = node_with_deps[1].select(&:write?)
-
-    if message.was_during_bootstrap?
-      raise "Message should not have any read deps" unless r_deps.empty?
-      update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options)
-    else
-      update_dependencies_fast(node, r_deps, w_deps, options)
-    end
-  end
-
-  def update_dependencies_master(options={})
-    update_dependencies_on_node(master_node_with_deps, options)
-  end
-
-  def update_dependencies_secondaries(options={})
-    secondary_nodes_with_deps.each do |node_with_deps|
-      update_dependencies_on_node(node_with_deps, options.merge(:with_recovery => true))
-      after_secondary_update_hook
-    end
-  end
-
-  def after_secondary_update_hook
-    # Hook only used for testing
-  end
-
-  def cleanup_dependency_secondaries
-    secondary_nodes_with_deps.each do |node, deps|
-      node.del(recovery_key)
-    end
-  end
-
-  def update_dependencies(options={})
-    # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
-    # 1) We do the secondaries first, with a recovery token.
-    # 2) Then we do the master.
-    # 3) Then we cleanup the recovery token on secondaries.
-    update_dependencies_secondaries(options)
-    update_dependencies_master(options)
-    cleanup_dependency_secondaries
-  end
-
-  def duplicate_message?
-    unless instance_dep.version >= get_current_instance_version + 1
-      # We happen to get a duplicate message, or we are recovering a dead
-      # worker. During regular operations, we just need to cleanup the 2pc (from
-      # the dead worker), and ack the message to rabbit.
-      # TODO Test cleanup
-      cleanup_dependency_secondaries
-
-      # But, if the message was generated during bootstrap, we don't really know
-      # if the other dependencies are up to date (because of the non-atomic
-      # bootstrapping process), so we do the max() trick (see in update_dependencies_on_node).
-      # Since such messages can come arbitrary late, we never really know if we
-      # can assume regular operations, thus we always assume that such message
-      # can originate from the bootstrapping period.
-      # Note that we are not in the happy path. Such duplicates messages are
-      # seldom: either (1) the publisher recovered a payload that didn't need
-      # recovery, or (2) a subscriber worker died after # update_dependencies_master,
-      # but before the message acking).
-      update_dependencies if message.was_during_bootstrap?
-
-      true
-    else
-      false
-    end
-  end
-
-  LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute, we give up
-                   :sleep   => 0.1,        # polling every 100ms.
-                   :expire  => 1.minute }  # after one minute, we are considered dead
-
-  def check_duplicate_and_update_dependencies
-    if duplicate_message?
-      Promiscuous.debug "[receive] Skipping message (already processed) #{message}"
-      return
-    end
-
-    yield
-
-    update_dependencies
-  end
-
-  def with_instance_locked(&block)
-    return yield unless message.has_dependencies?
-
-    lock_options = LOCK_OPTIONS.merge(:node => master_node)
-    mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
-
-    unless mutex.lock
-      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
-    end
-
-    begin
-      yield
-    ensure
-      unless mutex.unlock
-        # TODO Be safe in case we have a duplicate message and lost the lock on it
-        raise "The subscriber lost the lock during its operation. It means that someone else\n"+
-          "received a duplicate message, and we got screwed.\n"
-      end
-    end
-  end
-
-  def execute_operations
-    self.operations.each(&:execute)
-  end
-
-  def on_message
-    with_instance_locked do
-      if Promiscuous::Config.consistency == :causal && message.has_dependencies?
-        self.check_duplicate_and_update_dependencies { execute_operations }
-      else
-        execute_operations
-      end
-    end
-    message.ack
-  end
-
-  def operation_class
-    Promiscuous::Subscriber::Operation::Regular
-  end
-end

data/lib/promiscuous/subscriber/operation/base.rb

@@ -1,66 +0,0 @@
-class Promiscuous::Subscriber::Operation::Base
-  attr_accessor :model, :id, :operation, :attributes
-  delegate :message, :to => :message_processor
-
-  def initialize(payload)
-    if payload.is_a?(Hash)
-      self.id         = payload['id']
-      self.operation  = payload['operation'].try(:to_sym)
-      self.attributes = payload['attributes']
-      self.model      = self.get_subscribed_model(payload) if payload['types']
-    end
-  end
-
-  def get_subscribed_model(payload)
-    [message.app, '*'].each do |app|
-      app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
-      payload['types'].to_a.each do |ancestor|
-        model = app_mapping[ancestor]
-        return model if model
-      end
-    end
-    nil
-  end
-
-  def warn(msg)
-    Promiscuous.warn "[receive] #{msg} #{message.payload}"
-  end
-
-  def create(options={})
-    model.__promiscuous_fetch_new(id).tap do |instance|
-      instance.__promiscuous_update(self)
-      instance.save!
-    end
-  rescue Exception => e
-    if model.__promiscuous_duplicate_key_exception?(e)
-      options[:on_already_created] ||= proc { warn "ignoring already created record" }
-      options[:on_already_created].call
-    else
-      raise e
-    end
-  end
-
-  def update(should_create_on_failure=true)
-    model.__promiscuous_fetch_existing(id).tap do |instance|
-      if instance.__promiscuous_eventual_consistency_update(self)
-        instance.__promiscuous_update(self)
-        instance.save!
-      end
-    end
-  rescue model.__promiscuous_missing_record_exception
-    warn "upserting"
-    create :on_already_created => proc { update(false) if should_create_on_failure }
-  end
-
-  def destroy
-    if Promiscuous::Config.consistency == :eventual
-      Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
-    end
-
-    model.__promiscuous_fetch_existing(id).tap do |instance|
-      instance.destroy
-    end
-  rescue model.__promiscuous_missing_record_exception
-    warn "ignoring missing record"
-  end
-end