promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous.rb

@@ -1,12 +1,28 @@
 require 'active_support/core_ext'
+require 'active_model/callbacks'
+require 'multi_json'
 
 module Promiscuous
+  def self.require_for(gem, file)
+    require gem
+    require file
+  rescue LoadError
+  end
+
   require 'promiscuous/autoload'
-  require 'promiscuous/railtie' if defined?(Rails)
+  require_for 'rails',   'promiscuous/railtie'
+  require_for 'resque',  'promiscuous/resque'
+  require_for 'sidekiq', 'promiscuous/sidekiq'
+
 
   extend Promiscuous::Autoload
   autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,
-           :CLI, :Error, :Loader, :AMQP, :Redis, :Config
+           :CLI, :Error, :Loader, :AMQP, :Redis, :ZK, :Config, :DSL, :Key,
+           :Convenience, :Dependency, :Middleware, :Timer
+
+  extend Promiscuous::DSL
+
+  Object.__send__(:include, Promiscuous::Convenience)
 
   class << self
     def configure(&block)
@@ -19,13 +35,6 @@ module Promiscuous
       end
     end
 
-    def reload
-      desc  = Promiscuous::Publisher::Base.descendants
-      desc += Promiscuous::Subscriber::Base.descendants
-      desc.reject! { |klass| klass.name =~ /^Promiscuous::/ }
-      desc.each { |klass| klass.setup_class_binding }
-    end
-
     def connect
       AMQP.connect
       Redis.connect
@@ -39,36 +48,24 @@ module Promiscuous
     def healthy?
       AMQP.ensure_connected
       Redis.ensure_connected
-    rescue
+    rescue Exception
       false
     else
       true
     end
-  end
-
-  module ConsoleHelpers
-    # These are just for the subscriber, some helpers to debug in production
-    def global_key
-      Promiscuous::Redis.sub_key('global')
-    end
 
-    def global_version
-      Promiscuous::Redis.get(global_key).to_i
+    def disabled
+      Thread.current[:promiscuous_disabled] || $promiscuous_disabled
     end
 
-    def global_version=(value)
-      Promiscuous::Redis.set(global_key, value)
-      Promiscuous::Redis.publish(global_key, value)
-      value
+    def disabled=(value)
+      Thread.current[:promiscuous_disabled] = value
     end
 
-    def global_version!
-      version = Promiscuous::Redis.incr(global_key)
-      Promiscuous::Redis.publish(global_key, version)
-      version
+    def context(*args, &block)
+      Publisher::Context.open(*args, &block)
     end
   end
-  extend ConsoleHelpers
 
   at_exit { self.disconnect rescue nil }
 end

data/lib/promiscuous/amqp.rb

@@ -1,16 +1,18 @@
 module Promiscuous::AMQP
   extend Promiscuous::Autoload
-  autoload :Bunny, :RubyAMQP, :Null
+  autoload :HotBunnies, :Bunny, :Null, :Fake
 
-  EXCHANGE = 'promiscuous'.freeze
+  PUB_EXCHANGE = ENV['PUB_EXCHANGE'] || 'promiscuous'
+  SUB_EXCHANGE = ENV['SUB_EXCHANGE'] || 'promiscuous'
 
   class << self
     attr_accessor :backend
+    attr_accessor :backend_class
 
     def backend=(value)
-      disconnect if @backend
-      @backend = value.nil? ? nil : "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize
-      connect if @backend
+      disconnect
+      @backend_class = value.nil? ? nil : "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize
+      connect if @backend_class
     end
 
     def lost_connection_exception
@@ -23,10 +25,27 @@ module Promiscuous::AMQP
 
     def publish(options={})
       ensure_connected
-      Promiscuous.debug "[publish] (#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
-      backend.publish(options)
+      Promiscuous.debug "[publish] #{options[:key]} -> #{options[:payload]}"
+      backend.respond_to?(:async) ? backend.async.publish(options) : backend.publish(options)
     end
 
-    delegate :connect, :disconnect, :connected?, :open_queue, :to => :backend
+    def connect
+      return if @backend
+      @backend = backend_class.new
+      @backend.connect
+    end
+
+    def disconnect
+      return unless @backend
+      @backend.disconnect
+      @backend.terminate if @backend.respond_to?(:terminate)
+      @backend = nil
+    end
+
+    delegate :connected?, :to => :backend
+
+    def const_missing(sym)
+      backend_class.const_get(sym)
+    end
   end
 end

data/lib/promiscuous/amqp/bunny.rb

@@ -1,30 +1,145 @@
-module Promiscuous::AMQP:: Bunny
-  mattr_accessor :connection
+class Promiscuous::AMQP::Bunny
+  def self.hijack_bunny
+    return if @bunny_hijacked
+    ::Bunny::Session.class_eval do
+      alias_method :handle_network_failure_without_promiscuous, :handle_network_failure
 
-  def self.connect
+      def handle_network_failure(e)
+        Promiscuous.warn "[amqp] #{e}. Reconnecting..."
+        Promiscuous::Config.error_notifier.try(:call, e)
+        handle_network_failure_without_promiscuous(e)
+      end
+    end
+    @bunny_hijacked = true
+  end
+
+  attr_accessor :connection, :connection_lock, :callback_mapping
+
+  def initialize_driver
     require 'bunny'
-    self.connection = ::Bunny.new(Promiscuous::Config.amqp_url, { :heartbeat => Promiscuous::Config.heartbeat })
-    self.connection.start
+    self.class.hijack_bunny
+  end
+
+  def initialize
+    initialize_driver
+    # The bunny socket doesn't like when multiple threads access to it apparently
+    @connection_lock = Mutex.new
+    @callback_mapping = {}
+  end
+
+  def connect
+    @connection, @channel = new_connection
+    @exchange = exchange(@channel, :pub)
+    confirm_select(@channel, &method(:on_confirm))
+  end
+
+  def new_connection
+    connection = ::Bunny.new(Promiscuous::Config.amqp_url,
+                             :heartbeat_interval => Promiscuous::Config.heartbeat,
+                             :socket_timeout     => Promiscuous::Config.socket_timeout,
+                             :connect_timeout    => Promiscuous::Config.socket_timeout)
+    connection.start
+
+    channel = connection.create_channel
+    [connection, channel]
+  end
+
+  def disconnect
+    @connection_lock.synchronize do
+      return unless connected?
+      @connection.stop
+      @connection = @channel = nil
+    end
   end
 
-  def self.disconnect
-    self.connection.stop
+  def connected?
+    !!@connection.try(:connected?)
   end
 
-  def self.connected?
-    !!self.connection.try(:connected?)
+  def raw_publish(options)
+    @exchange.publish(options[:payload], :key => options[:key], :persistent => true)
   end
 
-  def self.publish(options={})
-    exchange(options[:exchange_name]).
-      publish(options[:payload], :key => options[:key], :persistent => true)
+  def exchange(channel, which)
+    exchange_name = which == :pub ? Promiscuous::AMQP::PUB_EXCHANGE :
+                                    Promiscuous::AMQP::SUB_EXCHANGE
+    channel.exchange(exchange_name, :type => :topic, :durable => true)
   end
 
-  def self.open_queue(options={}, &block)
-    raise "Cannot open queue with bunny"
+  def publish(options={})
+    @connection_lock.synchronize do
+      tag = @channel.next_publish_seq_no if options[:on_confirm]
+      raw_publish(options)
+      @callback_mapping[tag] = options[:on_confirm] if options[:on_confirm]
+    end
+  rescue Exception => e
+    e = Promiscuous::Error::Publisher.new(e, :payload => options[:payload])
+    Promiscuous.warn "[publish] #{e} #{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.try(:call, e)
   end
 
-  def self.exchange(name)
-    connection.exchange(name, :type => :topic, :durable => true)
+  def confirm_select(channel, &callback)
+    channel.confirm_select(callback)
+  end
+
+  def on_confirm(tag, multiple, nack=false)
+    if multiple
+      cbs = @callback_mapping.keys
+              .select { |k| k <= tag }
+              .map    { |k| @callback_mapping.delete(k) }
+      cbs.each(&:call) unless nack
+    else
+      cb = @callback_mapping.delete(tag)
+      cb.try(:call) unless nack
+    end
+  end
+
+  module Subscriber
+    def subscribe(options={}, &block)
+      queue_name    = options[:queue_name]
+      bindings      = options[:bindings]
+      Promiscuous::AMQP.ensure_connected
+
+      @lock = Mutex.new
+      @connection, @channel = Promiscuous::AMQP.backend.new_connection
+      @channel.basic_qos(Promiscuous::Config.prefetch)
+      exchange = Promiscuous::AMQP.backend.exchange(@channel, :sub)
+      @queue = @channel.queue(queue_name, Promiscuous::Config.queue_options)
+      bindings.each do |binding|
+        @queue.bind(exchange, :routing_key => binding)
+        Promiscuous.debug "[bind] #{queue_name} -> #{binding}"
+      end
+
+      @subscription = subscribe_queue(@queue, &block)
+    end
+
+    def subscribe_queue(queue, &block)
+      queue.subscribe(:ack => true) do |delivery_info, metadata, payload|
+        block.call(MetaData.new(self, delivery_info), payload)
+      end
+    end
+
+    class MetaData
+      def initialize(subscriber, delivery_info)
+        @subscriber = subscriber
+        @delivery_info = delivery_info
+      end
+
+      def ack
+        @subscriber.ack_message(@delivery_info.delivery_tag)
+      end
+    end
+
+    def ack_message(tag)
+      @lock.synchronize { @channel.ack(tag) } if @channel
+    end
+
+    def recover
+      @lock.synchronize { @channel.basic_recover(true) } if @channel
+    end
+
+    def disconnect
+      @lock.synchronize { @connection.stop; @channel = nil }
+    end
   end
 end

data/lib/promiscuous/amqp/fake.rb

@@ -0,0 +1,52 @@
+class Promiscuous::AMQP::Fake
+  attr_accessor :messages
+
+  class << self
+    def backend
+      Promiscuous::AMQP.backend
+    end
+    delegate :num_messages, :get_next_message, :get_next_payload, :to => :backend
+  end
+
+  def connect
+    @messages = []
+  end
+
+  def disconnect
+  end
+
+  def connected?
+    true
+  end
+
+  def publish(options={})
+    @messages << options
+    options[:on_confirm].try(:call)
+  end
+
+  def num_messages
+    @messages.count
+  end
+
+  def get_next_message
+    @messages.shift
+  end
+
+  def get_next_payload
+    JSON.parse(get_next_message[:payload])
+  end
+
+  def open_queue(options={}, &block)
+  end
+
+  module Subscriber
+    def subscribe(options={}, &block)
+    end
+
+    def recover
+    end
+
+    def disconnect
+    end
+  end
+end

data/lib/promiscuous/amqp/hot_bunnies.rb

@@ -0,0 +1,56 @@
+class Promiscuous::AMQP::HotBunnies < Promiscuous::AMQP::Bunny
+  attr_accessor :connection
+
+  def initialize_driver
+    require 'hot_bunnies'
+  end
+
+  # TODO auto reconnect
+
+  def new_connection
+    connection = ::HotBunnies.connect(:uri => Promiscuous::Config.amqp_url,
+                                      :heartbeat_interval => Promiscuous::Config.heartbeat,
+                                      :connection_timeout => Promiscuous::Config.socket_timeout)
+
+    channel = connection.create_channel
+    [connection, channel]
+  end
+
+  def disconnect
+    @connection_lock.synchronize do
+      return unless connected?
+      @channel.close rescue nil
+      @connection.close rescue nil
+      @connection = @channel = nil
+    end
+  end
+
+  def connected?
+    !!@connection.try(:is_open)
+  end
+
+  def raw_publish(options={})
+    @exchange.publish(options[:payload], :routing_key => options[:key], :persistent => true)
+  end
+
+  def confirm_select(channel, &callback)
+    channel.add_confirm_listener(&callback)
+    channel.confirm_select
+  end
+
+  module Subscriber
+    include Promiscuous::AMQP::Bunny::Subscriber
+
+    def subscribe_queue(queue, &block)
+      queue.subscribe(:ack => true, :blocking => false, &block)
+    end
+
+    def disconnect
+      @lock.synchronize do
+        @channel = nil
+        @subscription.shutdown! rescue nil
+        @connection.close rescue nil
+      end
+    end
+  end
+end

data/lib/promiscuous/amqp/null.rb

@@ -1,17 +1,17 @@
-module Promiscuous::AMQP::Null
-  def self.connect
+class Promiscuous::AMQP::Null
+  def connect
   end
 
-  def self.disconnect
+  def disconnect
   end
 
-  def self.connected?
+  def connected?
     true
   end
 
-  def self.publish(options={})
+  def publish(options={})
   end
 
-  def self.open_queue(options={}, &block)
+  def open_queue(options={}, &block)
   end
 end

data/lib/promiscuous/cli.rb

@@ -1,10 +1,10 @@
 class Promiscuous::CLI
   attr_accessor :options
 
-  def trap_signals
+  def trap_debug_signals
     Signal.trap 'SIGUSR2' do
       Thread.list.each do |thread|
-        print_status '-' * 80
+        print_status  '----[ Threads ]----' + '-' * (100-19)
         if thread.backtrace
           print_status "Thread #{thread} #{thread['label']}"
           print_status thread.backtrace.join("\n")
@@ -12,37 +12,98 @@ class Promiscuous::CLI
           print_status "Thread #{thread} #{thread['label']} -- no backtrace"
         end
       end
+
+      if @worker && @worker.respond_to?(:message_synchronizer)
+        if blocked_messages = @worker.message_synchronizer.try(:blocked_messages)
+          print_status  '----[ Pending Dependencies ]----' + '-' * (100-32)
+          blocked_messages.reverse_each { |msg| print_status msg }
+        end
+        print_status  '-' * 80
+      end
     end
+  end
 
+  def trap_exit_signals
     %w(SIGTERM SIGINT).each do |signal|
       Signal.trap(signal) do
+        exit 1 if @stop
         print_status "Exiting..."
-        @worker.terminate if @worker.try(:alive?)
+        @worker.try(:stop)
+        @worker = nil
         @stop = true
       end
     end
   end
 
+  def trap_signals
+    trap_debug_signals
+    trap_exit_signals
+  end
+
   def publish
     options[:criterias].map { |criteria| eval(criteria) }.each do |criteria|
       break if @stop
       title = criteria.name
       title = "#{title}#{' ' * [0, 20 - title.size].max}"
       bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => title, :total => criteria.count)
-      criteria.each do |doc|
+      criteria.unscoped.each do |doc|
         break if @stop
-        doc.promiscuous_sync
+        Promiscuous.context("cli/sync") { doc.promiscuous.sync }
         bar.increment
       end
     end
   end
 
+  def record
+    @worker = Promiscuous::Subscriber::Worker::Recorder.new(options[:log_file])
+    @worker.start
+    print_status "Recording..."
+    sleep 0.2 until !@worker
+  end
+
+  def replay_payload(payload)
+    endpoint = MultiJson.load(payload)['__amqp__']
+    if endpoint
+      # TODO confirm
+      Promiscuous::AMQP.publish(:key => endpoint, :payload => payload)
+      @num_msg += 1
+    else
+      puts "[warn] missing destination in #{payload}"
+    end
+  end
+
+  def replay
+    require 'json'
+    @num_msg = 0
+    File.open(options[:log_file], 'r').each do |line|
+      break if @stop
+      case line
+      when /^\[promiscuous\] \[receive\] ({.*})$/ then replay_payload($1)
+      when /^\[promiscuous\] \[publish\] .* -> ({.*})$/ then replay_payload($1)
+      when /^({.*})$/ then replay_payload($1)
+      end
+    end
+
+    puts "Replayed #{@num_msg} messages"
+  end
+
   def subscribe
-    Promiscuous::Loader.load_descriptors if defined?(Rails)
-    @worker = Promiscuous::Subscriber::Worker.run!
-    Celluloid::Actor[:pump].subscribe_sync.wait
-    print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
-    sleep 1 until !@worker.alive?
+    @worker = Promiscuous::Subscriber::Worker.new
+    @worker.start
+    print_status "Replicating..."
+    sleep 0.2 until !@worker
+  end
+
+  def publisher_recovery
+    @worker = Promiscuous::Publisher::Worker.new
+    @worker.start
+    print_status "Waiting for messages to recover..."
+    sleep 0.2 until !@worker
+  end
+
+  def generate_mocks
+    f = options[:output] ? File.open(options[:output], 'w') : STDOUT
+    f.write Promiscuous::Publisher::MockGenerator.generate
   end
 
   def parse_args(args)
@@ -54,8 +115,12 @@ class Promiscuous::CLI
 
       opts.separator ""
       opts.separator "Actions:"
-      opts.separator "    promiscuous publish \"Member.where(:updated_at.gt => 1.day.ago)\" BrandAction"
+      opts.separator "    promiscuous publish \"Model1.where(:updated_at.gt => 1.day.ago)\" Model2 Model3..."
       opts.separator "    promiscuous subscribe"
+      opts.separator "    promiscuous mocks"
+      opts.separator "    promiscuous record logfile"
+      opts.separator "    promiscuous replay logfile"
+      opts.separator "    promiscuous publisher_recovery"
       opts.separator ""
       opts.separator "Options:"
 
@@ -76,6 +141,14 @@ class Promiscuous::CLI
         Promiscuous::Config.prefetch = prefetch.to_i
       end
 
+      opts.on "-o", "--output FILE", "Output file for mocks. Defaults to stdout" do |file|
+        options[:output] = file
+      end
+
+      opts.on "-s", "--stat-interval [DURATION]", "Stats refresh rate (0 to disable)" do |duration|
+        Promiscuous::Config.stats_interval = duration.to_f
+      end
+
       opts.on("-h", "--help", "Show this message") do
         puts opts
         exit
@@ -93,19 +166,21 @@ class Promiscuous::CLI
 
     options[:action] = args.shift.try(:to_sym)
     options[:criterias] = args
+    options[:log_file] = args.first
 
-    unless options[:action].in? [:publish, :subscribe]
-      puts parser
-      exit
-    end
-
-    if options[:action] == :publish
-      raise "Please specify one or more criterias" unless options[:criterias].present?
-    else
-      raise "Why are you specifying a criteria?" if options[:criterias].present?
+    case options[:action]
+    when :publish   then raise "Please specify one or more criterias" unless options[:criterias].present?
+    when :subscribe then raise "Why are you specifying a criteria?"   if     options[:criterias].present?
+    when :record    then raise "Please specify a log file to record"  unless options[:log_file].present?
+    when :replay    then raise "Please specify a log file to replay"  unless options[:log_file].present?
+    when :publisher_recovery
+    when :mocks
+    else puts parser; exit 1
     end
 
     options
+  rescue SystemExit
+    exit
   rescue Exception => e
     puts e
     exit
@@ -113,10 +188,15 @@ class Promiscuous::CLI
 
   def load_app
     if options[:require]
-      require options[:require]
+      begin
+        require options[:require]
+      rescue LoadError
+        require "./#{options[:require]}"
+      end
     else
       require 'rails'
-      require File.expand_path("./config/environment.rb")
+      require 'promiscuous/railtie'
+      require File.expand_path("./config/environment")
       ::Rails.application.eager_load!
     end
   end
@@ -131,8 +211,12 @@ class Promiscuous::CLI
   def run
     trap_signals
     case options[:action]
-    when :publish then publish
+    when :publish   then publish
     when :subscribe then subscribe
+    when :record    then record
+    when :replay    then replay
+    when :mocks     then generate_mocks
+    when :publisher_recovery then publisher_recovery
     end
   end
 
@@ -146,6 +230,6 @@ class Promiscuous::CLI
 
   def print_status(msg)
     Promiscuous.info msg
-    $stderr.puts msg
+    STDERR.puts msg
   end
 end