ruote-amqp 2.2.0 → 2.3.0

data/lib/ruote/amqp/participant.rb

@@ -0,0 +1,350 @@
+#--
+# Copyright (c) 2010-2012, Kenneth Kalmer, John Mettraux.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#++
+
+
+module Ruote::Amqp
+
+  #
+  # This participant publishes messages on AMQP exchanges.
+  #
+  # == options
+  #
+  # A few options are supported. They can be declared at 3 levels:
+  #
+  # * options (when the participant is registered)
+  #
+  #   dashboard.register(
+  #     'amqp_participant',
+  #     Ruote::Amqp::Participant,
+  #     :routing_key => 'nada.x')
+  #
+  # * params (from the process definition)
+  #
+  #   sequence do
+  #     amqp_participant :routing_key => 'nada.x'
+  #   end
+  #
+  # * fields (from the passing workitem)
+  #
+  #   sequence do
+  #     set 'f:routing_key' => 'nada.x'
+  #     amqp_participant
+  #   end
+  #
+  # The 'conf' option (only available at participant registration) decides
+  # which levels are enabled or not.
+  #
+  # By default 'conf' is set to 'params, fields, options'.
+  #
+  # === 'conf'
+  #
+  # As said above, this option decides who can tweak this participant's
+  # options. It accepts a comma-separated list of levels.
+  #
+  # The levels are "params", "fields", "options".
+  #
+  # The order in which the levels are given is the order in which they
+  # are investigated for values.
+  #
+  # === 'connection'
+  #
+  # A hash of connection options. This is direcly fed to the amqp gem, the
+  # options of that gem apply thus ('host', 'port', 'vhost', 'username' and
+  # 'password').
+  #
+  # If no 'connection' (or :connection) hash is passed, the participant
+  # will attempt to use the connection (AMQP::Session) found in
+  # Ruote::Amqp.session. If there is nothing in there, it will [attempt] to
+  # create a new connection with AMQP's default settings.
+  #
+  # === 'exchange'
+  #
+  # Accepts a two or three sized Array.
+  #
+  # The first element is a string or symbol detailing the exchange type,
+  # like :direct, :fanout, :topic, ...
+  #
+  # The second element is an exchange name.
+  #
+  # The third, optional, element is a hash of exchange options.
+  #
+  # There is more information at http://rubyamqp.info/
+  #
+  # By default, 'exchange' is set to [ 'direct', '' ] (the default exchange).
+  #
+  # Note: you cannot pass an instantiated Ruby-AMQP exchange here. Ruote
+  # cannot serialize it for remote workers, so the settings are passed
+  # in a flat form, easily JSONifiable.
+  #
+  # === 'field_prefix'
+  #
+  # Sometimes one wants to separate his AMQP participant settings from other
+  # workitem fields.
+  #
+  #   dashboard.register(
+  #     'amqp_participant',
+  #     Ruote::Amqp::Participant,
+  #     :conf => 'fields', :field_prefix => 'amqp_')
+  #
+  # registers a participant that draws is configuration from workitem fields
+  # prefixed with 'amqp_'.
+  #
+  # Note that setting this option doesn't implicitely add 'fields' to the
+  # 'conf' option.
+  #
+  # === 'forget'
+  #
+  # When set to true forces the participant to reply to the engine immediately
+  # after the message got published, in a "fire and forget" fashion.
+  #
+  # === 'routing_key'
+  #
+  # Depending on the exchange used, this option lets you influence how the
+  # exchange routes the message towards queues.
+  #
+  # Consult your AMQP documentation for more information.
+  #
+  # === 'message'
+  #
+  # By default, the workitem is turned into a JSON string and transmitted in
+  # the AMQP message payload. If this 'message' option is set, its value is
+  # used as the payload.
+  #
+  # === 'persistent'
+  #
+  # If this option is set to something else than false or nil, messages
+  # messages published by this participant will be persistent (hopefully
+  # the queues they'll end up in will be persistent as well).
+  #
+  #
+  # == #encode_workitem
+  #
+  # The default way to encode a workitem before pushing it to the exchange
+  # is by turning it entirely into a JSON string.
+  #
+  # To alter this, one can subclass this participant and provide its own
+  # #encode_workitem(wi) method:
+  #
+  #   require 'yaml'
+  #
+  #   class MyAmqpParticipant < Ruote::Amqp::Participant
+  #
+  #     def encode_workitem(workitem)
+  #       YAML.dump(workitem)
+  #     end
+  #   end
+  #
+  # or when one needs to filter some fields:
+  #
+  #   class MyAmqpParticipant < Ruote::Amqp::Participant
+  #
+  #     def encode_workitem(workitem)
+  #       workitem.fields.delete_if { |k, v| k.match(/^private_/) }
+  #       super(workitem)
+  #     end
+  #   end
+  #
+  class Participant
+    include Ruote::LocalParticipant
+
+    # Initializing the participant, right before calling #on_workitem or
+    # another on_ method.
+    #
+    def initialize(options)
+
+      @options = options
+
+      @conf = (@options['conf'] || 'params, fields, options').split(/\s*,\s*/)
+      @conf = %w[ params fields options ] if @conf.include?('all')
+
+      @field_prefix = @options['field_prefix'] || ''
+    end
+
+    # Workitem consumption code.
+    #
+    def on_workitem
+
+      instantiate_exchange.publish(
+        message,
+        :routing_key => routing_key,
+        :persistent => persistent,
+        :correlation_id => correlation_id)
+
+      reply if forget
+    end
+
+    def on_cancel
+
+      return if opt('discard_cancel')
+
+      instantiate_exchange.publish(
+        encode_cancelitem,
+        :routing_key => routing_key,
+        :persistent => persistent,
+        :correlation_id => correlation_id)
+    end
+
+    # No need for a dedicated thread when dispatching messages. Respond
+    # true.
+    #
+    def do_not_thread; true; end
+
+    # Returns the exchange coordinates (a triple [ type, name, options ]).
+    # Defaults to the direct exchange.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def exchange
+
+      opt('exchange') || [ 'direct', '', {} ]
+        #
+        # defaults to the "default exchange"...
+    end
+
+    # Returns the message to publish.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def message; opt('message') || encode_workitem; end
+
+    # Returns the routing key for the message to publish.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def routing_key; opt('routing_key'); end
+
+    # Returns whether the publish should be persistent or not.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def persistent; opt('persistent'); end
+
+    # Returns the correlation_id for the message publication. Returns ''
+    # by default.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def correlation_id
+
+      opt('correlation_id') || ''
+    end
+
+    # Returns something true-ish if the participant should not reply to the
+    # engine once the publish operation is done.
+    #
+    # Available as a method so it can be overriden (the return value could
+    # depend on the @workitem or other factors).
+    #
+    def forget; opt('forget'); end
+
+    protected
+
+    # How a workitem is turned into an AMQP message payload (string).
+    #
+    # Feel free to override this method to accommodate your needs.
+    #
+    def encode_workitem
+
+      workitem.as_json
+    end
+
+    # How a "cancelitem" is turned into an AMQP message payload (string).
+    #
+    # Feel free to override this method to accommodate your needs.
+    #
+    def encode_cancelitem
+
+      Rufus::Json.encode(
+        'cancel' => true, 'fei' => @fei.h, 'flavour' => @flavour)
+    end
+
+    # Default AMQP.connect method.
+    #
+    # Feel free to override this method to accommodate your needs (See
+    # the README for an example).
+    #
+    def amqp_connect
+
+      ocon = opt('connection')
+
+      if Ruote::Amqp.session && ( ! ocon)
+        Ruote::Amqp.session
+      else
+        AMQP.connect(Ruote.keys_to_sym(ocon || {}))
+      end
+    end
+
+    # Given connection options passed at registration time (when the
+    # participant is registered in ruote) or from the process definition,
+    # returns an AMQP::Channel instance.
+    #
+    def channel
+
+      Thread.current['_ruote_amqp_channel'] ||= AMQP::Channel.new(amqp_connect)
+    end
+
+    # Given exchange options passed at registrations time or from the process
+    # definition, returns an AMQP::Exchange instance.
+    #
+    def instantiate_exchange
+
+      Thread.current['_ruote_amqp_exchange'] ||= begin
+
+        exc = exchange
+        type, name, options = exc
+
+        raise ArgumentError.new(
+          "couldn't determine exchange from #{exc.inspect}"
+        ) unless name
+
+        exchange_opts = (options || {}).each_with_object({}) { |(k, v), h|
+          h[k.to_sym] = v
+        }
+
+        AMQP::Exchange.new(channel, type.to_sym, name, exchange_opts)
+      end
+    end
+
+    # The mechanism for looking up options like 'connection', 'exchange',
+    # 'routing_key' in either the participant options, the process
+    # definition or the workitem fields...
+    #
+    def opt(key)
+
+      @conf.each do |type|
+
+        container = (type == 'options' ? @options : workitem.send(type))
+        k = type == 'fields' ? "#{@field_prefix}#{key}" : key
+
+        return container[k] if container.has_key?(k)
+      end
+
+      nil
+    end
+  end
+end
+

data/lib/ruote/amqp/receiver.rb

@@ -0,0 +1,181 @@
+#--
+# Copyright (c) 2010-2012, Kenneth Kalmer, John Mettraux.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#++
+
+
+module Ruote::Amqp
+
+  #
+  # A receiver is plugged between a ruote engine/storage and an AMQP queue.
+  # It will listen on the queue for messages, try to turn them into
+  # workitems and feed those workitem back to the engine/storage (in the usual
+  # use case, those workitems were initially emitted by the engine).
+  #
+  # == #decode_message(headers, payload)
+  #
+  # By default, the receiver expects the incoming workitem to be serialized
+  # entirely in the payload of the AMQP message. One can change this
+  # behaviour by overriding the #decode_workitem method (usually when
+  # subclassing)
+  #
+  #   class MyYamlReceiver < Ruote::Amqp::Receiver
+  #     def decode_message(headers, payload)
+  #       YAML.load(payload)
+  #     end
+  #   end
+  #
+  # #decode_message is supposed to return a Ruby hash (describing either a
+  # workitem or a launchitem), the difference is explained below.
+  #
+  # === workitems and launchitems
+  #
+  # The standard use case is to accept workitems coming back (they probably
+  # left the engine via Ruote::Amqp::Participant). But it's also OK
+  # to accept "launchitems", hashes with at least one 'process_definition'
+  # (or 'definition') entry.
+  #
+  # Upon receiving a launchitem, the receiver will launch a new process
+  # instances.
+  #
+  # Launchitems may have two more optional entries, 'workitems_fields' (or
+  # 'fields') and 'process_variables' (or 'variables').
+  #
+  # 'workitem_fields' must contain a hash of initial workitem fields (they will
+  # populate the initial workitem.
+  #
+  # 'process_variables' are a very advanced option. It's possible to set the
+  # initial variables in a workflow. Read the general ruote documentation to
+  # learn about the difference between fields and variables.
+  #
+  # The #decode_message is supposed to return a hash representing either a
+  # workitem, either a launchitem.
+  #
+  # == #handle_error(err)
+  #
+  # Out of the box, the receiver will print out to $stderr the details of
+  # errors it encounters when receiving, decoding and handing back the
+  # workitems (messages?) to the engine. This can be changed by overriding
+  # the #handle_error method:
+  #
+  #   class MyReceiver < Ruote::Amqp::Receiver
+  #     def handle_error(err)
+  #       ThatLoggerService.log(err)
+  #     end
+  #   end
+  #
+  #
+  # == initialization options
+  #
+  # One can set the "launch_only" option to true when initializing the receiver
+  # to prevent it from handling anything but launchitems.
+  #
+  #   receiver = Ruote::Amqp::Receiver.new(
+  #     @dashboard, @amqp_queue, :launch_only => true)
+  #
+  # 'launch_only' (string) is valid too.
+  #
+  class Receiver < Ruote::Receiver
+
+    attr_reader :queue
+
+    def initialize(engine_or_storage, queue, options={})
+
+      super(engine_or_storage, Ruote.keys_to_s(options))
+
+      @queue = queue
+      @queue.subscribe(&method(:handle))
+    end
+
+    def shutdown
+
+      @queue.unsubscribe
+    end
+
+    protected
+
+    def handle(header, payload)
+
+      item = decode_message(header, payload)
+
+      if item['error'] && item['fei']
+        flunk(item)
+      elsif item['fields'] && item['fei']
+        receive(item)
+      elsif item['process_definition'] || item['definition']
+        launch(item)
+      else
+        raise ArgumentError.new("cannot receive or launch #{item.inspect}")
+      end
+
+    rescue => e
+      handle_error(e)
+    end
+
+    def decode_message(header, payload)
+
+      Rufus::Json.decode(payload)
+    end
+
+    def handle_error(err)
+
+      $stderr.puts '**err**'
+      $stderr.puts err.inspect
+      $stderr.puts err.backtrace
+    end
+
+    def flunk(h)
+
+      return if @options['launch_only']
+
+      err = h.delete('error')
+
+      args = case err
+        when String then [ RemoteError, err ]
+        when Hash then [ Ruote.constantize(err['class']), err['message'] ]
+        else [ RemoteError, err.inspect ]
+      end
+
+      super(h, *args)
+    end
+
+    def receive(h)
+
+      return if @options['launch_only']
+
+      super(h)
+    end
+
+    def launch(h)
+
+      super(
+        h['process_definition'] || h['definition'],
+        h['workitem_fields'] || h['fields'] || {},
+        h['process_variables'] || h['variables'] || {})
+    end
+  end
+
+  #
+  # Used to wrap errors that come as string (well, errors that don't come
+  # with a class name). Could be thought of as "anonymous remote error".
+  #
+  class RemoteError < StandardError; end
+end
+

data/lib/ruote/amqp/version.rb

@@ -0,0 +1,28 @@
+#--
+# Copyright (c) 2010-2012, Kenneth Kalmer, John Mettraux.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#++
+
+
+module Ruote::Amqp
+
+  VERSION = '2.3.0'
+end
+

data/lib/ruote/amqp.rb

@@ -0,0 +1,68 @@
+#--
+# Copyright (c) 2010-2012, Kenneth Kalmer, John Mettraux.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#++
+
+require 'amqp'
+require 'ruote'
+
+require 'ruote/amqp/participant'
+require 'ruote/amqp/alert_participant'
+require 'ruote/amqp/receiver'
+
+
+module Ruote::Amqp
+
+  # Returns the AMQP::Session shared by the ruote participants (and potentially
+  # the receivers too).
+  #
+  # Returns nil if none is set (the participant will mostly create a connection
+  # on their own).
+  #
+  # See http://rubydoc.info/github/ruby-amqp/amqp/master/AMQP/Session
+  #
+  def self.session
+
+    @session
+  end
+
+  # Sets the AMQP::Session shared by the ruote participants (and potentially
+  # the receivers too).
+  #
+  #   Ruote::Amqp.session = AMQP.connect(:auto_recovery => true) do |con|
+  #     con.on_recovery do |con|
+  #       puts "Recovered..."
+  #     end
+  #     connection.on_tcp_connection_loss do |con, settings|
+  #       puts "Reconnecting... please wait"
+  #       conn.reconnect(false, 20)
+  #     end
+  #   end
+  #
+  # (Thanks Jim Li - https://github.com/marsbomber/ruote-amqp/commit/0f36a41f)
+  #
+  # See http://rubydoc.info/github/ruby-amqp/amqp/master/AMQP/Session
+  #
+  def self.session=(s)
+
+    @session = s
+  end
+end
+

data/lib/ruote-amqp.rb

@@ -1,80 +1,3 @@
 
-require 'mq'
-
-require 'ruote-amqp/version'
-
-
-#
-# AMQP participant and listener pair for ruote.
-#
-# == Documentation
-#
-# See #RuoteAMQP::Listener and #RuoteAMQP::Participant for detailed
-# documentation on using each of them.
-#
-# == AMQP Notes
-#
-# RuoteAMQP uses durable queues and persistent messages by default, to ensure
-# no messages get lost along the way and that running expressions doesn't have
-# to be restarted in order for messages to be resent.
-#
-module RuoteAMQP
-
-  autoload 'ParticipantProxy',   'ruote-amqp/participant'
-
-  autoload 'Receiver',           'ruote-amqp/receiver'
-  autoload 'WorkitemListener',   'ruote-amqp/workitem_listener'
-  autoload 'LaunchitemListener', 'ruote-amqp/launchitem_listener'
-
-  class << self
-
-    attr_writer :use_persistent_messages
-
-    # Whether or not to use persistent messages (true by default)
-    def use_persistent_messages?
-      @use_persistent_messages = true if @use_persistent_messages.nil?
-      @use_persistent_messages
-    end
-
-    # Ensure the AMQP connection is started
-    def start!
-      return if started?
-
-      mutex = Mutex.new
-      cv = ConditionVariable.new
-
-      Thread.main[:ruote_amqp_connection] = Thread.new do
-        Thread.abort_on_exception = true
-        AMQP.start {
-          started!
-          cv.signal
-        }
-      end
-
-      mutex.synchronize { cv.wait(mutex) }
-
-      MQ.prefetch(1)
-
-      yield if block_given?
-    end
-
-    # Check whether the AMQP connection is started
-    def started?
-      Thread.main[:ruote_amqp_started] == true
-    end
-
-    def started! #:nodoc:
-      Thread.main[:ruote_amqp_started] = true
-    end
-
-    # Close down the AMQP connections
-    def stop!
-      return unless started?
-
-      AMQP.stop
-      Thread.main[:ruote_amqp_connection].join
-      Thread.main[:ruote_amqp_started] = false
-    end
-  end
-end
+require 'ruote/amqp'