promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous/error/recovery.rb

@@ -0,0 +1,7 @@
+class Promiscuous::Error::Recovery < Promiscuous::Error::Base
+  def initialize(message, inner=nil)
+    message = "#{inner.class}: #{inner} -- #{message}" if inner
+    super(message)
+    set_backtrace(inner.backtrace) if inner
+  end
+end

data/lib/promiscuous/error/subscriber.rb

@@ -1,4 +1,4 @@
-class Promiscuous::Error::Subscriber < RuntimeError
+class Promiscuous::Error::Subscriber < Promiscuous::Error::Base
   attr_accessor :inner, :payload
 
   def initialize(inner, options={})
@@ -12,7 +12,5 @@ class Promiscuous::Error::Subscriber < RuntimeError
     "#{inner.class}: #{inner.message} while processing #{payload}"
   end
 
-  def to_s
-    message
-  end
+  alias to_s message
 end

data/lib/promiscuous/key.rb

@@ -0,0 +1,36 @@
+class Promiscuous::Key
+  def initialize(role, nodes=[], no_join=nil)
+    @role = role
+    @nodes = nodes
+    @no_join = no_join
+  end
+
+  def join(*nodes)
+    # --- backward compatiblity code ---
+    # TODO remove code
+    if nodes == ['global', nil, nil]
+      return self.class.new(@role, @nodes + nodes, :no_join)
+    end
+    if @no_join
+      return self.class.new(@role, @nodes)
+    end
+    # --- backward compatiblity code ---
+
+    self.class.new(@role, @nodes + nodes)
+  end
+
+  def to_s
+    path = []
+    case @role
+    when :pub then path << 'publishers'
+    when :sub then path << 'subscribers'
+    end
+    path << Promiscuous::Config.app
+    path += @nodes.compact
+    path.join(':')
+  end
+
+  def as_json(options={})
+    to_s
+  end
+end

data/lib/promiscuous/loader.rb

@@ -1,23 +1,19 @@
 module Promiscuous::Loader
-  def self.load_descriptors(descriptors=[:publishers, :subscribers])
-    [descriptors].flatten.each do |descriptor|
-      dir, file_matcher = case descriptor
-        when :publishers then %w(publishers **_publisher.rb)
-        when :subscribers then %w(subscribers **_subscriber.rb)
-        end
+  CONFIG_FILES = %w(config/publishers.rb config/subscribers.rb config/promiscuous.rb)
 
-      Dir[Rails.root.join('app', dir, file_matcher)].map do |file|
-        File.basename(file, ".rb").camelize.constantize
-      end
+  def self.prepare
+    CONFIG_FILES.each do |file_name|
+      file = defined?(Rails) ?  Rails.root.join(file_name) : File.join('.', file_name)
+      load file if File.exists?(file)
     end
+
+    # A one shot recovery on boot
+    Promiscuous::Publisher::Worker.new.try_recover
   end
 
-  def self.unload_descriptors(descriptors=[:publishers, :subscribers])
-    [descriptors].flatten.each do |descriptor|
-      dir, file_matcher = case descriptor
-        when :publishers then # TODO Cleanup publishers
-        when :subscribers then Promiscuous::Subscriber::AMQP.subscribers.clear
-        end
-    end
+  def self.cleanup
+    Promiscuous::Publisher::Model.publishers.clear
+    Promiscuous::Publisher::Model::Mongoid.collection_mapping.clear if defined?(Mongoid)
+    Promiscuous::Subscriber::Model.mapping.select! { |k| k.to_s =~ /__promiscuous__/ }
   end
 end

data/lib/promiscuous/middleware.rb

@@ -0,0 +1,112 @@
+class Promiscuous::Middleware
+  Context = Promiscuous::Publisher::Context
+
+  module Controller
+    extend ActiveSupport::Concern
+
+    def process_action(*args)
+      full_name = "#{self.class.controller_path}/#{self.action_name}"
+      Promiscuous::Middleware.with_context(full_name) { super }
+    end
+
+    def render(*args)
+      Promiscuous::Middleware.without_context { super }
+    end
+  end
+
+  def self.with_context(*args, &block)
+    Promiscuous.context(*args, &block)
+  rescue Exception => e
+    $promiscuous_last_exception = e if e.is_a? Promiscuous::Error::Base
+    pretty_print_exception(e)
+    raise e
+  end
+
+  def self.without_context
+    # This is different from the method without_promiscuous in convenience.rb
+    # That's used for render() and things that are *not* supposed to write.
+    # We actually force promiscuous to instrument queries, and make sure that
+    # we don't do any write we shouldn't.
+    old_context, Context.current = Context.current, nil
+    old_disabled, Promiscuous.disabled = Promiscuous.disabled, true
+    yield
+  rescue Exception => e
+    $promiscuous_last_exception = e if e.is_a? Promiscuous::Error::Base
+    pretty_print_exception(e)
+    raise e
+  ensure
+    Context.current = old_context
+    Promiscuous.disabled = old_disabled
+  end
+
+  def self.pretty_print_exception(e)
+    return if $promiscuous_pretty_print_exception_once == :disable || ENV['RAILS_ENV'] == 'production'
+    return if e.is_a?(SystemExit)
+
+    e = e.original_exception if defined?(ActionView::Template::Error) && e.is_a?(ActionView::Template::Error)
+
+    STDERR.puts
+    STDERR.puts "\e[0;#{36}m/---[ Exception: #{e.class} ]#{'-'*[0, 84 - e.class.name.size].max}\e[0m"
+    STDERR.puts "\e[0;#{36}m|"
+
+    highlight_indent = false
+    msg = e.to_s.split("\n").map do |line|
+      highlight_indent = true if line =~ /The problem comes from the following/ ||
+                                 line =~ /Promiscuous is tracking this read/
+      line = "\e[1;#{31}m#{line}\e[0;#{31}m" if highlight_indent && line =~ /^  /
+      "\e[0;#{36}m|  \e[0;#{31}m#{line}\e[0m"
+    end
+
+    STDERR.puts msg.join("\n")
+    STDERR.puts "\e[0;#{36}m|"
+    STDERR.puts "\e[0;#{36}m+---[ Backtrace ]--------------------------------------------------------------------------------------\e[0m"
+    STDERR.puts "\e[0;#{36}m|"
+
+    expand = ENV['TRACE'].to_i > 1
+    bt = e.backtrace.map do |line|
+       line = case line
+              when /(rspec-core|instrumentation)/
+                "\e[1;30m#{line}\e[0m" if expand
+              when /#{Rails.root}\/app\/controllers/
+                "\e[1;35m#{line}\e[0m"
+              when /#{Rails.root}\/app\/models/
+                "\e[1;33m#{line}\e[0m"
+              when /#{Rails.root}\/lib/
+                "\e[1;34m#{line}\e[0m"
+              when /(mongoid|active_record).*`(count|distinct|each|first|last)'$/
+                "\e[1;32m#{line}\e[0m"
+              when /(mongoid|active_record).*`(create|insert|save|update|modify|remove|remove_all)'$/
+                "\e[1;31m#{line}\e[0m"
+              when /#{Rails.root}/
+                if line =~ /\/support\//
+                  "\e[1;30m#{line}\e[0m" if expand
+                else
+                  "\e[1;36m#{line}\e[0m"
+                end
+              else
+                "\e[1;30m#{line}\e[0m" if expand
+              end
+       "\e[0;#{36}m|  #{line}" if line
+      end
+      .compact
+      .join("\n")
+    STDERR.puts bt
+    STDERR.puts "\e[0;#{36}m|"
+
+    if $cucumber_extra
+      STDERR.puts "\e[0;#{36}m+---[ Cucumber ]--------------------------------------------------------------------------------------\e[0m"
+      STDERR.puts "\e[0;#{36}m|"
+      $cucumber_extra.each_with_index do |line, i|
+        line = line.gsub(/([^:]*: )(.*)$/, "\\1\e[1;36m\\2")
+        STDERR.puts "\e[0;#{36}m| \e[0;36m#{line}\e[0m"
+        STDERR.puts "\e[0;#{36}m|" if i.zero?
+      end
+      STDERR.puts "\e[0;#{36}m|"
+    end
+
+    STDERR.puts "\e[0;#{36}m\\------------------------------------------------------------------------------------------------------\e[0m"
+    STDERR.puts
+    $promiscuous_pretty_print_exception_once = :disable if $promiscuous_pretty_print_exception_once
+  end
+
+end

data/lib/promiscuous/publisher.rb

@@ -1,9 +1,12 @@
 module Promiscuous::Publisher
   extend Promiscuous::Autoload
-  autoload :ActiveRecord, :AMQP, :Attributes, :Base, :Class, :Envelope, :Lint,
-           :Mock, :Model, :Mongoid, :Polymorphic, :Error, :Ephemeral
+  autoload :Model, :Operation, :MockGenerator, :Context, :Worker
 
-  def self.lint(*args)
-    Lint.lint(*args)
+  extend ActiveSupport::Concern
+
+  included do
+    include Model::Mongoid      if defined?(Mongoid::Document)  && self < Mongoid::Document
+    include Model::ActiveRecord if defined?(ActiveRecord::Base) && self < ActiveRecord::Base
+    raise "What kind of model is this? try including Promiscuous::Publisher after all your includes" unless self < Model::Base
   end
 end

data/lib/promiscuous/publisher/context.rb

@@ -0,0 +1,92 @@
+class Promiscuous::Publisher::Context
+  # XXX Context are not sharable among threads
+
+  def self.open(*args, &block)
+    old_disabled, Promiscuous.disabled = Promiscuous.disabled, false
+    old_current, self.current = self.current, new(*args)
+
+    begin
+      self.current.alt_trace ">>> open >>>", :backtrace => :none if ENV['TRACE']
+      yield
+    ensure
+      self.current.alt_trace "<<< close <<<", :backtrace => :none if ENV['TRACE']
+
+      self.current.close
+      self.current = old_current
+      Promiscuous.disabled = old_disabled
+    end
+  end
+
+  def self.current
+    Thread.current[:promiscuous_context]
+  end
+
+  def self.current=(value)
+    Thread.current[:promiscuous_context] = value
+  end
+
+  attr_accessor :name, :operations, :nesting_level, :last_write_dependency
+
+  def initialize(*args)
+    options = args.extract_options!
+    @parent = self.class.current unless !!options[:detached_from_parent]
+    @last_write_dependency = @parent.try(:last_write_dependency)
+    @nesting_level = @parent.try(:nesting_level).to_i + 1
+    @name = args.first.try(:to_s)
+    @name ||= "#{@parent.next_child_name}" if @parent
+    @name ||= 'anonymous'
+    @operations = []
+    @next_child = 0
+
+    Promiscuous::AMQP.ensure_connected
+    Mongoid::IdentityMap.clear if defined?(Mongoid::IdentityMap)
+  end
+
+  def close
+    @parent.try(:close_child, self)
+  end
+
+  def close_child(child)
+    @last_write_dependency = child.last_write_dependency if child.last_write_dependency
+  end
+
+  def next_child_name
+    @next_child += 1
+    "#{name}/#{@next_child}"
+  end
+
+  def add_operation(operation)
+    @operations << operation
+    trace_operation(operation) if ENV['TRACE']
+  end
+
+  def trace_operation(operation, options={})
+    msg = Promiscuous::Error::Dependency.explain_operation(operation, 70)
+    trace(msg, options.merge(:color => operation.read? ? '0;32' : '1;31'))
+  end
+
+  def alt_trace(msg, options={})
+    trace(msg, options.merge(:alt_fmt => true))
+  end
+
+  def trace(msg, options={})
+    backtrace = options[:backtrace]
+    alt_fmt = options[:alt_fmt]
+    color = alt_fmt ? "1;36" : options[:color]
+
+    name = "(#{self.name})#{' ' * ([0, 25 - self.name.size].max)}"
+    name = '  ' * @nesting_level + name
+    STDERR.puts "\e[#{color}m#{name}#{alt_fmt ? '':'  '} #{msg}\e[0m"
+
+    level = ENV['TRACE'].to_i
+    if level > 1 && defined?(Rails) && backtrace != :none
+      bt = (backtrace || caller)
+        .grep(/#{Rails.root}/)
+        .map { |line| line.gsub(/#{Rails.root}\/?/, '') }
+        .take(level-1)
+        .map { |line| "\e[1;#{30}m#{name}     #{line}\e[0m" }
+        .join("\n")
+      STDERR.puts bt
+    end
+  end
+end

data/lib/promiscuous/publisher/mock_generator.rb

@@ -0,0 +1,72 @@
+require 'erb'
+
+module Promiscuous::Publisher::MockGenerator
+  def self.generate
+    ERB.new(<<-ERB.gsub(/^\s+<%/, '<%').gsub(/^ {6}/, ''), nil, '-').result(binding)
+      # ---------------------------------
+      # Auto-generated file. Do not edit.
+      # ---------------------------------
+
+      module <%= Promiscuous::Config.app.camelize %>::Publishers
+      <% modules.each do |mod| -%>
+        <%- if mod.constantize.is_a?(Class) -%>
+        class <%= mod %>; end
+        <%- else -%>
+        module <%= mod %>; end
+        <% end -%>
+      <% end -%>
+
+      <% publishers.each do |publisher| -%>
+        <% next unless publisher.publish_to -%>
+        <% %>
+        # ------------------------------------------------------------------
+
+        class <%= publisher.publish_as %>
+          include Promiscuous::Publisher::Model::Mock
+          publish :to => '<%= publisher.publish_to %>'
+          <% if defined?(Mongoid::Document) && publisher.include?(Mongoid::Document) -%>
+          mock    :id => :bson
+          <% end -%>
+          <% %>
+          <% attributes_for(publisher).each do |attr| -%>
+          publish :<%= attr %>
+          <% end -%>
+        end
+
+        <% publisher.descendants.each do |subclass| -%>
+        class <%= subclass.publish_as %> < <%= publisher.publish_as %>
+          <% attributes_for(subclass, publisher).each do |attr| -%>
+          publish :<%= attr %>
+          <% end -%>
+        end
+        <% end -%>
+      <% end -%>
+      end
+    ERB
+  end
+
+  def self.attributes_for(klass, parent=nil)
+    attrs = klass.published_attrs
+    attrs -= parent.published_attrs if parent
+    attrs
+  end
+
+  def self.modules
+    publishers
+      .map    { |publisher| [publisher, *publisher.descendants.map(&:name)] }
+      .flatten
+      .select { |name| name =~ /::/ }
+      .map    { |name| name.gsub(/::[^:]+$/, '') }
+      .uniq
+      .sort
+  end
+
+  def self.publishers
+    Promiscuous::Publisher::Model.publishers.values
+      .reject { |publisher| publisher.publish_to =~ /^__promiscuous__\// }
+      .map    { |publisher| [publisher, publisher.descendants] }
+      .flatten
+      .reject { |publisher| publisher.superclass.respond_to?(:publish_to) &&
+                            publisher.superclass.publish_to }
+  end
+end

data/lib/promiscuous/publisher/model.rb

@@ -1,90 +1,7 @@
-require 'crowdtap_redis_lock'
-
 module Promiscuous::Publisher::Model
   extend Promiscuous::Autoload
-  autoload :ActiveRecord, :Mongoid
-
-  extend ActiveSupport::Concern
-  include Promiscuous::Publisher::Envelope
-
-  mattr_accessor :klasses
-  self.klasses = []
-
-  def operation
-    options[:operation]
-  end
-
-  def fetch
-    case operation
-    when :create  then instance
-    when :update  then options[:fetch_proc].call
-    when :destroy then nil
-    end
-  end
-
-  def payload
-    super.merge(:id => instance.id, :operation => operation, :version => version)
-  end
-
-  def include_attributes?
-    !operation.in? [:destroy, :dummy]
-  end
-
-  def instance
-    @new_instance || super
-  end
-
-  def with_lock(&block)
-    return yield if Promiscuous::Config.backend == :null
-    return yield if operation == :create
-
-    key = Promiscuous::Redis.pub_key(instance.id)
-    # We'll block for 60 seconds before raising an exception
-    ::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
-  end
-
-  def version
-    {:global => @global_version}
-  end
-
-  def update_dependencies
-    @global_version = Promiscuous::Redis.incr(Promiscuous::Redis.pub_key('global'))
-  end
-
-  def commit
-    ret = nil
-    exception = nil
-
-    Promiscuous::AMQP.ensure_connected
-
-    with_lock do
-      update_dependencies
-      begin
-        ret = yield if block_given?
-      rescue Exception => e
-        # we must publish something so the subscriber can sync
-        # with the updated dependencies
-        options[:operation] = :dummy
-        exception = e
-      end
-
-      begin
-        @new_instance = fetch
-      rescue Exception => e
-        raise_out_of_sync(e, payload.to_json)
-      end
-    end
-
-    publish
-
-    raise exception if exception
-    ret
-  end
+  autoload :Base, :ActiveRecord, :Ephemeral, :Mock, :Mongoid
 
-  module ClassMethods
-    def setup_class_binding
-      super
-      Promiscuous::Publisher::Model.klasses << klass if klass
-    end
-  end
+  mattr_accessor :publishers
+  self.publishers = {}
 end