promiscuous 0.92.0 → 0.100.0

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -14,21 +14,12 @@ class Promiscuous::Subscriber::Worker::Pump
       options[:bindings][exchange] = ['*']
     end
 
-    if Promiscuous::Config.bootstrap
-      options[:bindings][Promiscuous::AMQP::BOOTSTRAP_EXCHANGE] = ['*']
-    end
-
     subscribe(options, &method(:on_message))
   end
 
   def on_message(metadata, payload)
-    msg = Promiscuous::Subscriber::Worker::Message.new(payload, :metadata => metadata, :root_worker => @root)
-    if Promiscuous::Config.bootstrap
-      # Bootstrapping doesn't require synchronzation
-      @root.runner.messages_to_process << msg
-    else
-      @root.message_synchronizer.process_when_ready(msg)
-    end
+    msg = Promiscuous::Subscriber::Message.new(payload, :metadata => metadata, :root_worker => @root)
+    @root.runner.messages_to_process << msg
   rescue Exception => e
     Promiscuous.warn "[receive] cannot process message: #{e}\n#{e.backtrace.join("\n")}"
     Promiscuous::Config.error_notifier.call(e)

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.92.0'
+  VERSION = '0.100.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.92.0
+  version: 0.100.0
 platform: ruby
 authors:
 - Nicolas Viennot
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-11 00:00:00.000000000 Z
+date: 2014-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -137,7 +137,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.0
-description: Replicate your Mongoid/ActiveRecord models across your applications
+description: Replicate models across applications
 email:
 - nicolas@viennot.biz
 - kareem@doubleonemedia.com
@@ -146,52 +146,40 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- bin/promiscuous
-- lib/promiscuous.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/fake.rb
 - lib/promiscuous/amqp/file.rb
 - lib/promiscuous/amqp/hot_bunnies.rb
 - lib/promiscuous/amqp/null.rb
+- lib/promiscuous/amqp.rb
 - lib/promiscuous/autoload.rb
 - lib/promiscuous/cli.rb
 - lib/promiscuous/config.rb
 - lib/promiscuous/convenience.rb
 - lib/promiscuous/dependency.rb
 - lib/promiscuous/dsl.rb
-- lib/promiscuous/error.rb
 - lib/promiscuous/error/base.rb
 - lib/promiscuous/error/connection.rb
 - lib/promiscuous/error/dependency.rb
 - lib/promiscuous/error/lock_unavailable.rb
 - lib/promiscuous/error/lost_lock.rb
-- lib/promiscuous/error/missing_context.rb
 - lib/promiscuous/error/publisher.rb
 - lib/promiscuous/error/recovery.rb
 - lib/promiscuous/error/subscriber.rb
+- lib/promiscuous/error.rb
 - lib/promiscuous/key.rb
 - lib/promiscuous/loader.rb
 - lib/promiscuous/mongoid.rb
-- lib/promiscuous/publisher.rb
-- lib/promiscuous/publisher/bootstrap.rb
-- lib/promiscuous/publisher/bootstrap/connection.rb
-- lib/promiscuous/publisher/bootstrap/data.rb
-- lib/promiscuous/publisher/bootstrap/mode.rb
-- lib/promiscuous/publisher/bootstrap/status.rb
-- lib/promiscuous/publisher/bootstrap/version.rb
-- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/context/base.rb
-- lib/promiscuous/publisher/context/middleware.rb
 - lib/promiscuous/publisher/context/transaction.rb
+- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/mock_generator.rb
-- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/model/active_record.rb
 - lib/promiscuous/publisher/model/base.rb
 - lib/promiscuous/publisher/model/ephemeral.rb
 - lib/promiscuous/publisher/model/mock.rb
 - lib/promiscuous/publisher/model/mongoid.rb
-- lib/promiscuous/publisher/operation.rb
+- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/operation/active_record.rb
 - lib/promiscuous/publisher/operation/atomic.rb
 - lib/promiscuous/publisher/operation/base.rb
@@ -200,35 +188,30 @@ files:
 - lib/promiscuous/publisher/operation/non_persistent.rb
 - lib/promiscuous/publisher/operation/proxy_for_query.rb
 - lib/promiscuous/publisher/operation/transaction.rb
+- lib/promiscuous/publisher/operation.rb
 - lib/promiscuous/publisher/worker.rb
+- lib/promiscuous/publisher.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/redis.rb
-- lib/promiscuous/resque.rb
-- lib/promiscuous/sidekiq.rb
-- lib/promiscuous/subscriber.rb
-- lib/promiscuous/subscriber/message_processor.rb
-- lib/promiscuous/subscriber/message_processor/base.rb
-- lib/promiscuous/subscriber/message_processor/bootstrap.rb
-- lib/promiscuous/subscriber/message_processor/regular.rb
-- lib/promiscuous/subscriber/model.rb
+- lib/promiscuous/subscriber/message.rb
 - lib/promiscuous/subscriber/model/active_record.rb
 - lib/promiscuous/subscriber/model/base.rb
 - lib/promiscuous/subscriber/model/mongoid.rb
 - lib/promiscuous/subscriber/model/observer.rb
+- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/operation.rb
-- lib/promiscuous/subscriber/operation/base.rb
-- lib/promiscuous/subscriber/operation/bootstrap.rb
-- lib/promiscuous/subscriber/operation/regular.rb
-- lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/subscriber/unit_of_work.rb
 - lib/promiscuous/subscriber/worker/eventual_destroyer.rb
-- lib/promiscuous/subscriber/worker/message.rb
-- lib/promiscuous/subscriber/worker/message_synchronizer.rb
 - lib/promiscuous/subscriber/worker/pump.rb
 - lib/promiscuous/subscriber/worker/recorder.rb
 - lib/promiscuous/subscriber/worker/runner.rb
 - lib/promiscuous/subscriber/worker/stats.rb
+- lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/subscriber.rb
 - lib/promiscuous/timer.rb
 - lib/promiscuous/version.rb
+- lib/promiscuous.rb
+- bin/promiscuous
 homepage: http://github.com/crowdtap/promiscuous
 licenses: []
 metadata: {}
@@ -248,9 +231,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: Model replication over RabbitMQ
+summary: Replicate models across applications
 test_files: []
-has_rdoc: false

data/lib/promiscuous/error/missing_context.rb

@@ -1,29 +0,0 @@
-class Promiscuous::Error::MissingContext < Promiscuous::Error::Base
-  def message
-    require 'erb'
-    ERB.new(<<-ERB.gsub(/^\s+<%/, '<%').gsub(/^ {6}/, ''), nil, '-').result(binding)
-      Promiscuous needs to execute all your read/write queries in a context for publishing.
-      This is what you can do:
-       1. Wrap your operations in a Promiscuous context yourself (jobs, etc.):
-
-          Promiscuous::Middleware.with_context 'jobs/name' do
-             # Code including all your read and write queries
-           end
-
-       2. Disable Promiscuous completely (only for testing):
-
-           RSpec.configure do |config|
-             config.around do |example|
-               without_promiscuous { example.run }
-             end
-           end
-
-         Note that opening a context will reactivate promiscuous temporarily
-         even if it was disabled.
-
-       3. You are in render() in the Rails controller, and you should not write.
-    ERB
-  end
-
-  alias to_s message
-end

data/lib/promiscuous/publisher/bootstrap.rb

@@ -1,27 +0,0 @@
-module Promiscuous::Publisher::Bootstrap
-  extend Promiscuous::Autoload
-  autoload :Connection, :Version, :Data, :Mode, :Status
-
-  def self.setup(options={})
-    puts "Enabling bootstrapping mode"
-    Mode.enable
-    puts "Bootstrapping versions..."
-    Version.bootstrap
-    puts "Setting up data bootstrap..."
-    Data.setup(options)
-  end
-
-  def self.run
-    raise "Setup must be run before starting to bootstrap" unless Mode.enabled?
-    Data.run
-  end
-
-  def self.finalize
-    raise "Setup must be run before disabling" unless Mode.enabled?
-    Mode.disable
-  end
-
-  def self.status
-    Status.monitor
-  end
-end

data/lib/promiscuous/publisher/bootstrap/connection.rb

@@ -1,25 +0,0 @@
-class Promiscuous::Publisher::Bootstrap::Connection
-  def initialize
-    # We don't put the connection in the initializer because it gets funny when
-    # it comes to the disconnection.
-    connection_options = { :url      => Promiscuous::Config.publisher_amqp_url,
-                           :exchange => Promiscuous::AMQP::BOOTSTRAP_EXCHANGE }
-    @connection, @channel, @exchange = Promiscuous::AMQP.new_connection(connection_options)
-    self
-  end
-
-  def publish(options={})
-    options[:key]      ||= "#{Promiscuous::Config.app}/__bootstrap__"
-    options[:exchange] ||= @exchange
-    Promiscuous::AMQP.publish(options)
-  end
-
-  def close
-    if @connection
-      # TODO not very pretty... We should abstract that
-      @channel.respond_to?(:stop)    ? @channel.stop    : @channel.close
-      @connection.respond_to?(:stop) ? @connection.stop : @connection.close
-      @exchange = nil
-    end
-  end
-end

data/lib/promiscuous/publisher/bootstrap/data.rb

@@ -1,127 +0,0 @@
-class Promiscuous::Publisher::Bootstrap::Data
-  class << self
-    def setup(options={})
-      Promiscuous::Publisher::Bootstrap::Status.reset
-      range_redis_keys.each { |key| redis.del(key) }
-
-      models    = options[:models]
-      models   ||= Promiscuous::Publisher::Model.publishers.values
-      .reject { |publisher| publisher.include? Promiscuous::Publisher::Model::Ephemeral }
-
-      models.each_with_index do |model, i|
-        create_range(i, model, options)
-        Promiscuous::Publisher::Bootstrap::Status.total(model.count)
-      end
-    end
-
-    def run(options={})
-      connection = Promiscuous::Publisher::Bootstrap::Connection.new
-
-      range_redis_keys.each do |key|
-        lock = Promiscuous::Redis::Mutex.new(key, lock_options(options[:lock]))
-        if lock.try_lock
-          if range = redis.get(key)
-            range = MultiJson.load(range)
-            selector = range['selector']
-            options  = range['options']
-            klass    = range['class'].constantize
-            start    = range['start'].to_i
-            finish   = range['finish'].to_i
-
-            lock_extended_at = Time.now
-            without_promiscuous do
-              range_selector(klass, selector, options, start, finish).each_with_index do |instance, i|
-                publish_data(connection, instance)
-
-                if (Time.now - lock_extended_at) > lock_options[:expire]/5
-                  raise "Another worker stole your work!" unless lock.extend
-                  lock_extended_at = Time.now
-                end
-                Promiscuous::Publisher::Bootstrap::Status.inc
-              end
-              redis.del(key)
-              lock.unlock
-            end
-          end
-        end
-      end
-    end
-
-    private
-
-    def create_range(namespace, model, options)
-      range_size = options[:range_size] || 1000
-
-      without_promiscuous do
-        count = model.all.count
-        return if count == 0
-
-        num_ranges = (count/range_size.to_f).ceil
-        first, last = min_max(model).map { |id| id.to_s.to_i(16) }
-        last += 10 # Ensure that we capture the last ID based on BSON encoding
-        increment  = ((last - first)/num_ranges).ceil
-
-        num_ranges.times do |i|
-          range_start  = first + (increment * i)
-          range_finish = range_start + increment
-
-          key = range_redis_key.join(namespace).join(i)
-          value = MultiJson.dump(:selector => model.all.selector,
-                                 :options  => model.all.options,
-                                 :class    => model.all.klass.to_s,
-                                 :start    => range_start.to_s,
-                                 :finish   => range_finish.to_s)
-          redis.set(key, value)
-        end
-      end
-    end
-
-    def range_selector(klass, selector, options, start, finish)
-      option ||= {}
-      criteria = Mongoid::Criteria.new(klass)
-      criteria.selector = selector
-      criteria.options = options.merge(:timeout => false)
-
-      criteria.order_by("$natural" =>  1).where(:_id => { '$gte' =>  BSON::ObjectId.from_string(start.to_s(16)),
-                                                          '$lt'  =>  BSON::ObjectId.from_string(finish.to_s(16)) })
-    end
-
-    def range_redis_key
-      Promiscuous::Key.new(:pub).join('bootstrap:range')
-    end
-
-    def range_redis_keys
-      redis.keys("#{range_redis_key}*").reject { |k| k =~ /:lock$/ }.sort
-    end
-
-    def lock_options(options=nil)
-      options ||= {}
-      @@lock_options ||= {
-        :timeout  => 10.seconds,
-        :sleep    => 0.01.seconds,
-        :expire   => 5.minutes,
-        :node     => redis
-      }.merge(options)
-    end
-
-    def publish_data(connection, instance)
-      operation = instance.promiscuous.payload
-      operation[:operation] = :bootstrap_data
-
-      payload = {}
-      payload[:app] = Promiscuous::Config.app
-      payload[:operations] = [operation]
-
-      connection.publish(:key => Promiscuous::Config.app, :payload => MultiJson.dump(payload))
-    end
-
-    def min_max(model)
-      query = proc { |sort_order| model.order_by("_id" =>  sort_order).only(:id).limit(1).first.id }
-      [query.call(1), query.call(-1)]
-    end
-
-    def redis
-      Promiscuous::Redis.master.nodes.first
-    end
-  end
-end

data/lib/promiscuous/publisher/bootstrap/mode.rb

@@ -1,19 +0,0 @@
-module Promiscuous::Publisher::Bootstrap::Mode
-  def self.enable
-    Promiscuous::Redis.master.nodes.each { |node| node.set(key, 1) }
-  end
-
-  def self.disable
-    Promiscuous::Redis.master.nodes.each { |node| node.del(key) }
-  end
-
-  def self.enabled?
-    !!Promiscuous::Redis.master.nodes.first.get(key)
-  end
-
-  def self.key
-    # XXX You must change the LUA script in promiscuous/publisher/operation/base.rb
-    # if you change this value
-    Promiscuous::Key.new(:pub).join('bootstrap')
-  end
-end

data/lib/promiscuous/publisher/bootstrap/status.rb

@@ -1,40 +0,0 @@
-require 'ruby-progressbar'
-
-module Promiscuous::Publisher::Bootstrap::Status
-  def self.reset
-    redis.del(key)
-  end
-
-  def self.total(count)
-    redis.hincrby(key, 'total', count)
-  end
-
-  def self.inc
-    redis.hincrby(key, 'processed', 1)
-  end
-
-  def self.monitor
-    total ||= redis.hget(key, 'total').to_i
-    processed = 0
-    exit_now = false
-
-    %w(SIGTERM SIGINT).each { |signal| Signal.trap(signal) { exit_now = true  } }
-    bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => "Bootstrapping", :total => total)
-    while processed < total
-      processed = redis.hget(key, 'processed').to_i
-      bar.progress = processed
-      sleep 1
-      break if exit_now
-    end
-  end
-
-  private
-
-  def self.key
-    Promiscuous::Key.new(:pub).join('bootstrap:counter')
-  end
-
-  def self.redis
-    Promiscuous::Redis.master.nodes.first
-  end
-end

data/lib/promiscuous/publisher/bootstrap/version.rb

@@ -1,46 +0,0 @@
-class Promiscuous::Publisher::Bootstrap::Version
-  def self.bootstrap
-    connection = Promiscuous::Publisher::Bootstrap::Connection.new
-    Promiscuous::Redis.master.nodes.each_with_index do |node, node_index|
-      begin_at = 0
-      chunk_size = Promiscuous::Config.bootstrap_chunk_size
-
-      while begin_at < Promiscuous::Config.hash_size do
-        end_at = [begin_at + chunk_size, Promiscuous::Config.hash_size].min
-        Chunk.new(connection, node, node_index, (begin_at...end_at)).fetch_and_send
-        begin_at += chunk_size
-      end
-    end
-  end
-
-  class Chunk
-    def initialize(connection, node, node_index, range)
-      @connection = connection
-      @node = node
-      @range = range
-      @node_index = node_index
-    end
-
-    def fetch_and_send
-      num_nodes = Promiscuous::Redis.master.nodes.size
-
-      futures = {}
-      @node.pipelined do
-        @range.each do |i|
-          next unless i % num_nodes == @node_index
-          futures[i] = @node.get(Promiscuous::Key.new(:pub).join(i, 'rw'))
-        end
-      end
-
-      operation = {}
-      operation[:operation] = :bootstrap_versions
-      operation[:keys] = futures.map { |i, f| "#{i}:#{f.value}" if f.value }.compact
-
-      payload = {}
-      payload[:app] = Promiscuous::Config.app
-      payload[:operations] = [operation]
-
-      @connection.publish(:payload => MultiJson.dump(payload)) if operation[:keys].present?
-    end
-  end
-end