promiscuous 0.92.0 → 0.100.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d772f9b9786ef7ee6787030113bf9ec5934717d5
-  data.tar.gz: 163c9cb61d4b359f1e6766effef9baccdfd8c229
+  metadata.gz: b49e20ef32de7aa2d0ff24941962adf673ab5332
+  data.tar.gz: 036b09d5ea13bc45751221693a7f8bb4edea4d1c
 SHA512:
-  metadata.gz: 6cacc43f226549868f963abbad29e7ba7249a1bbc1c4eb3e0b8014275935ca37e7455a73b48e36d7c0c56906308ab334c104bbf136a7cd2c7345ee976161eead
-  data.tar.gz: 6f59e694ed2c89e2ebe9a6c73995c4b6e4987ee1f43d171f55f3783b9698c4974aff8ef684d22dd8e652956d943081b952f8bb313d492016629b9564a1447727
+  metadata.gz: ceaab91a632d2b7eaa71f0ae59f14d66a47e7c5f531582425f24b567b79530dc3f2602ca01b139001dd047001c1b5f54ed41f1beb798d9dc8976de93c26a29ae
+  data.tar.gz: 5d3313bc9aabb5f1da0a6e71216637dd326bc4d8c4a10cbfc3cbeb83b45733479c348f20a91b1b56160670d54e9653c70fd08e169c9a68dfaba8109aefddcad9

data/lib/promiscuous.rb

@@ -20,9 +20,6 @@ module Promiscuous
            :CLI, :Error, :Loader, :AMQP, :Redis, :ZK, :Config, :DSL, :Key,
            :Convenience, :Dependency, :Timer
 
-  # Shortcut for the middleware, TODO make load on demand
-  Middleware = Publisher::Context::Middleware
-
   extend Promiscuous::DSL
 
   Object.__send__(:include, Promiscuous::Convenience)
@@ -82,8 +79,8 @@ module Promiscuous
       !!Thread.current[:promiscuous_disabled]
     end
 
-    def context(*args, &block)
-      Publisher::Context::Base.with_context(*args, &block)
+    def context
+      Publisher::Context::Base.current
     end
   end
 

data/lib/promiscuous/amqp.rb

@@ -2,8 +2,7 @@ module Promiscuous::AMQP
   extend Promiscuous::Autoload
   autoload :HotBunnies, :Bunny, :Null, :Fake, :File
 
-  LIVE_EXCHANGE      = 'promiscuous'
-  BOOTSTRAP_EXCHANGE = 'promiscuous.bootstrap'
+  LIVE_EXCHANGE = 'promiscuous'
 
   class << self
     attr_accessor :backend

data/lib/promiscuous/cli.rb

@@ -1,3 +1,5 @@
+require 'ruby-progressbar'
+
 class Promiscuous::CLI
   attr_accessor :options
 
@@ -17,14 +19,6 @@ class Promiscuous::CLI
             print_status "Thread #{thread} #{thread['label']} -- no backtrace"
           end
         end
-
-        if @worker && @worker.respond_to?(:message_synchronizer)
-          if blocked_messages = @worker.message_synchronizer.try(:blocked_messages)
-            print_status  '----[ Pending Dependencies ]----' + '-' * (100-32)
-            blocked_messages.reverse_each { |msg| print_status msg }
-          end
-          print_status  '-' * 100
-        end
       end
     end
   end
@@ -61,7 +55,7 @@ class Promiscuous::CLI
       bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => title, :total => criteria.count)
       criteria.each do |doc|
         break if @stop
-        Promiscuous.context("cli/sync") { doc.promiscuous.sync }
+        doc.promiscuous.sync
         bar.increment
       end
     end
@@ -100,16 +94,6 @@ class Promiscuous::CLI
     print_status "Replayed #{@num_msg} messages"
   end
 
-  def bootstrap
-    phase = options[:criterias][0].to_sym
-    raise "Subscriber bootstrap must be one of [setup|run|finalize|status]" unless [:setup, :run, :finalize, :status].include?(phase)
-    if phase == :setup && criteria = options[:bootstrap_criteria]
-      Promiscuous::Publisher::Bootstrap.setup(:models => eval(criteria).to_a)
-    else
-      Promiscuous::Publisher::Bootstrap.__send__(phase)
-    end
-  end
-
   def subscribe
     @worker = Promiscuous::Subscriber::Worker.new
     @worker.start
@@ -143,7 +127,6 @@ class Promiscuous::CLI
       opts.separator "    promiscuous publish \"Model1.where(:updated_at.gt => 1.day.ago)\" [Model2 Model3...]"
       opts.separator "    promiscuous publisher_recovery"
       opts.separator "    promiscuous subscribe"
-      opts.separator "    promiscuous bootstrap phase"
       opts.separator "    promiscuous mocks"
       opts.separator "    promiscuous record logfile"
       opts.separator "    promiscuous replay logfile"
@@ -154,10 +137,6 @@ class Promiscuous::CLI
         Promiscuous::Config.no_deps = true
       end
 
-      opts.on "-x", "--ignore-exceptions", "Ignore exceptions and continue to process messages" do
-        Promiscuous::Config.ignore_exceptions = true
-      end
-
       opts.on "-l", "--require FILE", "File to require to load your app. Don't worry about it with rails" do |file|
         options[:require] = file
       end
@@ -175,20 +154,6 @@ class Promiscuous::CLI
         Promiscuous::Config.stats_interval = duration.to_f
       end
 
-      opts.on "-b", "--bootstrap [pass1|pass2]", "Run subscriber in bootstrap mode" do |mode|
-        mode = mode.to_sym
-        raise "Subscriber bootstrap must be run in pass1 or pass2 mode" unless [:pass1, :pass2].include?(mode)
-        Promiscuous::Config.bootstrap = mode.to_sym
-      end
-
-      opts.on "-t", "--threads [NUM]", "Number of subscriber worker threads to run. Defaults to 10." do |threads|
-        Promiscuous::Config.subscriber_threads = threads.to_i
-      end
-
-      opts.on "-c", "--criteria FILE", "Criteria to bootstrap a subset of data" do |criteria|
-        options[:bootstrap_criteria] = criteria
-      end
-
       opts.on "-D", "--daemonize", "Daemonize process" do
         options[:daemonize] = true
       end
@@ -210,15 +175,12 @@ class Promiscuous::CLI
     options[:action] = args.shift.try(:to_sym)
     options[:criterias] = args
     options[:log_file] = args.first
-    options[:publisher_bootstrap] = args.first
 
     case options[:action]
     when :publish             then raise "Please specify one or more criterias" unless options[:criterias].present?
     when :subscribe           then raise "Why are you specifying a criteria?"   if     options[:criterias].present?
-    when :bootstrap           then raise "You must specify one of [setup|start|finalize]" unless options[:criterias].present?
     when :record              then raise "Please specify a log file to record"  unless options[:log_file].present?
     when :replay              then raise "Please specify a log file to replay"  unless options[:log_file].present?
-    when :publisher_bootstrap then raise "Please specify 'on' or 'off'" unless options[:publisher_bootstrap].present?
     when :publisher_recovery
     when :mocks
     else puts parser; exit 1
@@ -270,12 +232,10 @@ class Promiscuous::CLI
     case options[:action]
     when :publish   then publish
     when :subscribe then subscribe
-    when :bootstrap then bootstrap
     when :record    then record
     when :replay    then replay
     when :mocks     then generate_mocks
     when :publisher_recovery  then publisher_recovery
-    when :publisher_bootstrap then set_publisher_bootstrap
     end
   end
 

data/lib/promiscuous/config.rb

@@ -1,12 +1,12 @@
 module Promiscuous::Config
-  mattr_accessor :app, :bootstrap, :bootstrap_chunk_size, :backend, :amqp_url,
+  mattr_accessor :app, :backend, :amqp_url,
                  :publisher_amqp_url, :subscriber_amqp_url, :publisher_exchange,
                  :subscriber_exchanges, :queue_name, :queue_options, :redis_url,
                  :redis_urls, :redis_stats_url, :stats_interval,
                  :socket_timeout, :heartbeat, :no_deps, :hash_size,
                  :prefetch, :recovery_timeout, :logger, :subscriber_threads,
                  :version_field, :error_notifier, :recovery_on_boot,
-                 :on_stats, :ignore_exceptions, :consistency, :max_retries, :generation
+                 :on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval
 
   def self.backend=(value)
     @@backend = value
@@ -35,8 +35,6 @@ module Promiscuous::Config
     block.call(self) if block
 
     self.app                  ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
-    self.bootstrap            ||= false
-    self.bootstrap_chunk_size ||= 10000
     self.backend              ||= best_amqp_backend
     self.amqp_url             ||= 'amqp://guest:guest@localhost:5672'
     self.publisher_amqp_url   ||= self.amqp_url
@@ -54,7 +52,7 @@ module Promiscuous::Config
     self.heartbeat            ||= 60
     self.no_deps              ||= false
     self.hash_size            ||= 2**20 # one million keys ~ 200Mb.
-    self.prefetch             ||= self.bootstrap ? 10000000 : 1000
+    self.prefetch             ||= 1000
     self.recovery_timeout     ||= 10
     self.logger               ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
     self.subscriber_threads   ||= 10
@@ -62,10 +60,10 @@ module Promiscuous::Config
     self.version_field        ||= '_v'
     self.recovery_on_boot     = true if self.recovery_on_boot.nil?
     self.on_stats             ||= proc { |rate, latency| }
-    self.ignore_exceptions    ||= false
-    self.consistency          ||= :eventual
     self.max_retries          ||= 10
     self.generation           ||= 1
+    self.destroy_timeout      ||= 1.hour
+    self.destroy_check_interval ||= 10.minutes
   end
 
   def self.configure(&block)

data/lib/promiscuous/error/dependency.rb

@@ -1,9 +1,8 @@
 class Promiscuous::Error::Dependency < Promiscuous::Error::Base
-  attr_accessor :dependency_solutions, :operation, :context
+  attr_accessor :dependency_solutions, :operation
 
   def initialize(options={})
     self.operation = options[:operation]
-    self.context = Promiscuous::Publisher::Context.current
   end
 
   # TODO Convert all that with Erb
@@ -69,7 +68,6 @@ class Promiscuous::Error::Dependency < Promiscuous::Error::Base
     when :update    then msg += 'multi update'
     when :destroy   then msg += 'multi destroy'
     end
-    msg += " in the '#{context.name}' context:\n\n"
     msg += "  #{self.class.explain_operation(self.operation)}"
     msg += "\n\nProTip: Try again with TRACE=2 in the shell or ENV['TRACE']='2' in the console.\n" unless ENV['TRACE']
     msg

data/lib/promiscuous/publisher/context.rb

@@ -1,6 +1,6 @@
 module Promiscuous::Publisher::Context
   extend Promiscuous::Autoload
-  autoload :Base, :Transaction, :Middleware
+  autoload :Base, :Transaction
 
   def self.current
     Base.current

data/lib/promiscuous/publisher/context/base.rb

@@ -2,44 +2,13 @@ class Promiscuous::Publisher::Context::Base
   # XXX Context are not sharable among threads
 
   def self.current
-    Thread.current[:promiscuous_context]
+    Thread.current[:promiscuous_context] ||= self.new
   end
 
-  def self.current=(value)
-    Thread.current[:promiscuous_context] = value
-  end
-
-  def self.with_context(*args, &block)
-    raise "You cannot nest contexts" if self.current
-
-    self.current = new(*args)
-    begin
-      self.current.trace "<<< open <<<", :level => 1
-      yield
-    ensure
-      self.current.trace "<<< close <<<", :level => 1
-      self.current = nil
-
-      ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
-    end
-  end
-
-  attr_accessor :name, :read_operations, :extra_dependencies, :current_user
-
-  def initialize(name=nil, options={})
-    @name = name.try(:to_s) || 'anonymous'
-    @current_user = options[:current_user]
-    @read_operations = []
-    @extra_dependencies = []
-    @transaction_managers = {}
-
-    Promiscuous::AMQP.ensure_connected
-
-    Mongoid::IdentityMap.clear if defined?(Mongoid::IdentityMap)
-    ActiveRecord::IdentityMap.clear if defined?(ActiveRecord::IdentityMap)
-  end
+  attr_accessor :current_user
 
   def transaction_context_of(driver)
+    @transaction_managers ||= {}
     @transaction_managers[driver] ||= Promiscuous::Publisher::Context::Transaction.new(driver)
   end
 

data/lib/promiscuous/publisher/model/base.rb

@@ -2,12 +2,10 @@ module Promiscuous::Publisher::Model::Base
   extend ActiveSupport::Concern
 
   included do
-    class_attribute :published_attrs, :tracked_attrs
+    class_attribute :published_attrs
     cattr_accessor  :published_db_fields # There is one on each root class, none on the subclasses
     self.published_attrs = []
-    self.tracked_attrs = []
     self.published_db_fields = []
-    track_dependencies_of :id
     Promiscuous::Publisher::Model.publishers[self.promiscuous_collection_name] = self
   end
 
@@ -38,26 +36,13 @@ module Promiscuous::Publisher::Model::Base
       value
     end
 
-    def get_dependency(attr, value)
-      return nil unless value
+    def get_dependency
       @collection ||= @instance.class.promiscuous_collection_name
-      Promiscuous::Dependency.new(@collection, attr, value)
+      Promiscuous::Dependency.new(@collection, :id, id)
     end
 
-    def tracked_dependencies(options={})
-      # FIXME This is not sufficient, we need to consider the previous and next
-      # values in case of an update.
-      # Note that the caller expect the id dependency to come first
-      @instance.class.tracked_attrs.map do |attr|
-        begin
-          [attr, @instance.__send__(attr)]
-        rescue Exception => e
-          # Don't care about missing attributes for read dependencies.
-          raise e unless options[:allow_missing_attributes] && e.is_a?(ActiveModel::MissingAttributeError)
-        end
-      end
-      .map { |attr, value| get_dependency(attr, value) }
-      .compact
+    def id
+      @instance.id
     end
   end
 
@@ -110,10 +95,6 @@ module Promiscuous::Publisher::Model::Base
       @in_publish_block.to_i > 0
     end
 
-    def track_dependencies_of(*attributes)
-      ([self] + descendants).each { |klass| klass.tracked_attrs |= attributes.map(&:to_sym) }
-    end
-
     def promiscuous_collection_name
       self.name.pluralize.underscore
     end
@@ -129,7 +110,6 @@ module Promiscuous::Publisher::Model::Base
     def inherited(subclass)
       super
       subclass.published_attrs = self.published_attrs.dup
-      subclass.tracked_attrs   = self.tracked_attrs.dup
       # no copy for published_db_fields
     end
   end

data/lib/promiscuous/publisher/model/mock.rb

@@ -25,14 +25,12 @@ module Promiscuous::Publisher::Model::Mock
   def save_operation(operation)
     payload = nil
 
-    Promiscuous::Publisher::Context::Middleware.with_context("mocking #{self.class}") do
-      op = Promiscuous::Publisher::Operation::Ephemeral.new(:instance => self, :operation => operation)
-      # TODO FIX the mocks to populate app name, also we need to hook before the
-      # json dump.
-      payload = op.generate_payload
-    end
+    op = Promiscuous::Publisher::Operation::Ephemeral.new(:instance => self, :operation => operation)
+    # TODO FIX the mocks to populate app name, also we need to hook before the
+    # json dump.
+    payload = op.generate_payload
 
-    Promiscuous::Subscriber::Worker::Message.new(payload).process
+    Promiscuous::Subscriber::Message.new(payload).process
   end
 
   module ClassMethods

data/lib/promiscuous/publisher/operation/active_record.rb

@@ -64,8 +64,7 @@ class ActiveRecord::Base
             alias_method :release_savepoint_without_promiscuous,       :release_savepoint
 
             def with_promiscuous_transaction_context(&block)
-              ctx = Promiscuous::Publisher::Context.current
-              block.call(ctx.transaction_context_of(:active_record)) if ctx
+              block.call(Promiscuous::Publisher::Context.current.transaction_context_of(:active_record))
             end
 
             def begin_db_transaction
@@ -106,24 +105,10 @@ class ActiveRecord::Base
               with_promiscuous_transaction_context { |tx| tx.commit }
             end
 
-            alias_method :select_all_without_promiscuous, :select_all
-            alias_method :select_values_without_promiscuous, :select_values
             alias_method :insert_without_promiscuous, :insert
             alias_method :update_without_promiscuous, :update
             alias_method :delete_without_promiscuous, :delete
 
-            def select_all(arel, name = nil, binds = [])
-              PromiscuousSelectOperation.new(arel, name, binds, :connection => self).execute do
-                select_all_without_promiscuous(arel, name, binds)
-              end
-            end
-
-            def select_values(arel, name = nil)
-              PromiscuousSelectOperation.new(arel, name, [], :connection => self).execute do
-                select_values_without_promiscuous(arel, name)
-              end
-            end
-
             def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [])
               PromiscuousInsertOperation.new(arel, name, pk, id_value, sequence_name, binds, :connection => self).execute do
                 insert_without_promiscuous(arel, name, pk, id_value, sequence_name, binds)
@@ -157,11 +142,11 @@ class ActiveRecord::Base
     end
 
     def transaction_context
-      current_context.transaction_context_of(:active_record)
+      Promiscuous::Publisher::Context.current.transaction_context_of(:active_record)
     end
 
     def ensure_transaction!
-      if current_context && write? && !transaction_context.in_transaction?
+      if !transaction_context.in_transaction?
         raise "You need to write to the database within an ActiveRecord transaction"
       end
     end
@@ -180,7 +165,7 @@ class ActiveRecord::Base
         query.non_instrumented { db_operation.call }
         query.instrumented do
           db_operation_and_select.tap do
-            transaction_context.add_write_operation(self) if write? && !@instances.empty?
+            transaction_context.add_write_operation(self) if !@instances.empty?
           end
         end
       end
@@ -277,56 +262,6 @@ class ActiveRecord::Base
     end
   end
 
-  class PromiscuousSelectOperation < PromiscousOperation
-    def initialize(arel, name, binds, options={})
-      super
-      @operation = :read
-      @result = []
-    end
-
-    def model
-      @model ||= begin
-        case @arel
-        when Arel::SelectManager
-          raise "SQL statement too complicated (joins?)" if @arel.ast.cores.size != 1
-          model = @arel.ast.cores.first.source.left.engine
-        when ActiveRecord::Relation
-          return nil # TODO
-        else
-          raise "What is this query?" unless @arel.is_a?(Arel::SelectManager)
-        end
-
-        model = nil unless model < Promiscuous::Publisher::Model::ActiveRecord
-        model
-      end
-    rescue
-      # TODO Track dependencies of complex queries properly...
-      nil
-    end
-
-    def get_selector_instance
-      attrs = @arel.ast.cores.first.wheres.map { |w| [w.children.first.left.name, w.children.first.right] }
-      model.instantiate(Hash[attrs])
-    end
-
-    def query_dependencies
-      deps = dependencies_for(get_selector_instance)
-      deps.empty? ? super : deps
-    end
-
-    def execute(&db_operation)
-      # We dup because ActiveRecord modifies our return value
-      super.tap { @result = @result.dup }
-    end
-
-    def db_operation_and_select
-      # XXX This is only supported by Postgres.
-      @connection.exec_query("#{@connection.to_sql(@arel, @binds)}", @operation_name, @binds).to_a.tap do |result|
-        @instances = result.map { |row| model.instantiate(row) }
-      end
-    end
-  end
-
   class PromiscuousTransaction < Promiscuous::Publisher::Operation::Transaction
     attr_accessor :connection