message 0.0.1

data/README.md

@@ -0,0 +1,121 @@
+# Message
+
+Message provides flexible & reliable background/asynchronous job processing mechanism on top of simple queue interface.
+
+Any developer can create queue adapter for Message to hook up different messaging/queue system.
+
+One in-memory queue is included with Message for you to start development and test,
+and you can easily swap in other queues later.
+
+
+## Installation
+
+
+
+## How to use
+
+### Queuing jobs
+
+
+Inspired by delayed_job API, call .enq.method(params) on any object and it will be processed in the background.
+
+    # without message
+    @img.resize(36)
+
+    # with message
+    @img.enq.resize(36)
+
+### Start worker to process jobs
+
+    Message.worker.start
+
+## Job interface specification
+
+Job = a queue + message processor
+
+### initialize a job
+
+    job = Message.job('name') { |msg| ... process msg ... }
+
+### enq(msg), alias: <<
+
+Queue up a message for processing:
+
+    job << msg
+
+### process(size=1)
+
+Process a message in queue by processor defined when initializing the job
+
+    job.process
+
+Process multiple messages
+
+    job.process(5)
+
+## Job filters
+
+You can add job filter to add additional functions to enqueue and process job message
+
+    Message.job.filter(filter_name) do |next_filter, job|
+      lambda do |*args, &block|
+        next_filter.call(*args, &block)
+      end
+    end
+
+Filters will be applied as the order initialized.
+Checkout all filters:
+
+    Message.job.filters
+
+### enq filter
+
+    Message.job.filter(:enq) do |filter, job|
+      lambda do |work|
+        filter.call(work)
+      end
+    end
+
+### process filter
+
+    Message.job.filter(:process) do |filter, job|
+      lambda do |size, &processor|
+        filter.call(size) do |msg|
+          processor.call(msg)
+        end
+      end
+    end
+
+## Queue adapters
+
+Change queue adapter to change different queue implementation. Default is a in memory queue for testing and development environments.
+
+### Change adapter
+
+    Message.queue.adapter = :sqs
+
+### Add a new adapter
+
+    Message.queue.adapters[:sqs] = Message::SqsQueue
+
+## Queue interface specification
+
+To hook up a new queue into Message.
+
+### name
+
+Queue name.
+
+### enq(msg), alias: <<
+
+Enqueue message, non-blocking.
+
+### deq(size, &block)
+
+Dequeue message, non-blocking.
+It is up to Queue implementation when to delete the message in queue when deq got called with a block.
+
+### size
+
+(Approximate) queue size.
+

data/lib/message/filters.rb

@@ -0,0 +1,66 @@
+require 'benchmark'
+
+module Message
+  class Filters
+    attr_reader :config
+
+    def initialize
+      @data = Hash.new{|h,k|h[k]=[]}
+      @config = {
+        :error_handling_callback => lambda {|type, job, msg, e| }
+      }
+      defaults.each do |t, m|
+        self[t] << [m, method(m)]
+      end
+    end
+
+    def [](type)
+      @data[type]
+    end
+
+    def defaults
+      [
+        [:process, :error_handling],
+        [:enq, :error_handling],
+        [:process, :benchmarking]
+      ]
+    end
+
+    def error_handling(filter, job)
+      lambda do |arg, &processor|
+        type = processor ? :process : :enq
+        log_error(type, job, (type == :enq ? arg : nil)) do
+          if processor
+            filter.call(arg) do |msg|
+              log_error(type, job, msg) { processor.call(msg) }
+            end
+          else
+            filter.call(arg)
+          end
+        end
+      end
+    end
+
+    def benchmarking(filter, job)
+      lambda do |size, &processor|
+        filter.call(size) do |msg|
+          ret = nil
+          Message.logger.info { "#{job.name}: processing one message"}
+          s = Benchmark.realtime do
+            ret = processor.call(msg)
+          end
+          Message.logger.info { "#{job.name}: processed in #{(1000 * s).to_i}ms" }
+          ret
+        end
+      end
+    end
+
+    private
+    def log_error(type, job, msg, &block)
+      block.call
+    rescue => e
+      Message.logger.error {"#{type} #{job.name} message failed, #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"}
+      config[:error_handling_callback].call(type, job, msg, e)
+    end
+  end
+end

data/lib/message/in_memory_queue.rb

@@ -0,0 +1,39 @@
+require 'thread'
+
+module Message
+  class InMemoryQueue
+    attr :name
+    def initialize(name)
+      @name = name
+      @queue = ::Queue.new
+    end
+
+    def enq(msg)
+      @queue << msg
+    end
+    alias :<< :enq
+
+    def deq(size=1, &block)
+      if size == 1
+        __deq__(&block)
+      else
+        size.times { __deq__(&block) }
+      end
+    rescue ThreadError
+      #no message in queue
+    end
+
+    def size
+      @queue.size
+    end
+
+    private
+    def __deq__(&block)
+      if block_given?
+        yield(@queue.deq(true))
+      else
+        @queue.deq(true)
+      end
+    end
+  end
+end

data/lib/message/job.rb

@@ -0,0 +1,48 @@
+require 'message/filters'
+
+module Message
+  class Job
+    class << self
+      def filters
+        @filters ||= Filters.new
+      end
+
+      def filter(type, name, &block)
+        filters[type] << [name, block]
+      end
+
+      def reset
+        @filters = nil
+      end
+    end
+
+    def initialize(queue, &processor)
+      @queue = queue
+      @processor = processor || lambda {|msg| msg}
+    end
+
+    def name
+      @queue.name
+    end
+
+    def size
+      @queue.size
+    end
+
+    def enq(msg)
+      chain(:enq, @queue.method(:enq)).call(msg)
+    end
+    alias :<< :enq
+
+    def process(size=1)
+      chain(:process, @queue.method(:deq)).call(size, &@processor)
+    end
+
+    private
+    def chain(type, base)
+      Job.filters[type].reverse.inject(base) do |m, f|
+        f[1].call(m, self)
+      end
+    end
+  end
+end

data/lib/message/q.rb

@@ -0,0 +1,26 @@
+require 'message/in_memory_queue'
+module Message
+  module Q
+    module_function
+    def init(name)
+      adapters[adapter].new(name)
+    end
+
+    def adapters
+      @adapters ||= { :in_memory => InMemoryQueue }
+    end
+
+    def adapter
+      @adapter ||= :in_memory
+    end
+
+    def adapter=(name)
+      @adapter = name
+    end
+
+    def reset
+      @adapter = nil
+      @adapters = nil
+    end
+  end
+end

data/lib/message/worker.rb

@@ -0,0 +1,79 @@
+require 'yaml'
+
+module Message
+  class Worker
+    DEFAULT_JOB_NAME = 'message-worker-default'
+
+    module Enqueue
+      class Enq
+        def initialize(obj, job)
+          @obj = obj
+          @job = job
+        end
+
+        def method_missing(m, *args, &block)
+          if block_given?
+            raise ArgumentError, "Can't enqueue with block call."
+          end
+          unless @obj.respond_to?(m)
+            raise NoMethodError, "undefined method `#{m}' for #{@obj.inspect}"
+          end
+          Worker.enq(@job, [@obj, m, args])
+        end
+      end
+
+      def enq(job=DEFAULT_JOB_NAME)
+        Enq.new(self, job)
+      end
+    end
+
+    class << self
+      def jobs
+        @jobs ||= RUBY_PLATFORM =~ /java/ ? java.util.concurrent.ConcurrentHashMap.new : {}
+      end
+
+      def job(name)
+        jobs[name] ||= Message.job(name, &job_processor)
+      end
+
+      def enq(name, work)
+        job(name).enq(YAML.dump(work))
+      end
+
+      def job_processor
+        lambda do |msg|
+          obj, m, args = YAML.load(msg)
+          obj.send(m, *args)
+        end
+      end
+    end
+
+    def initialize(job_name)
+      @job_name = job_name || DEFAULT_JOB_NAME
+    end
+
+    def start(size=10, interval=1)
+      Thread.start do
+        begin
+          log(:info) { "start" }
+          loop do
+            process(size)
+            sleep interval
+          end
+          log(:info) { "stopped" }
+        rescue => e
+          log(:error) { "crashed: #{e.message}\n#{e.backtrace.join("\n")}"}
+        end
+      end
+    end
+
+    def process(size=1)
+      Worker.job(@job_name).process(size)
+    end
+
+    private
+    def log(level, &block)
+      Message.logger.send(level) { "[Worker(#{Thread.current.object_id})] #{block.call}" }
+    end
+  end
+end

data/lib/message.rb

@@ -0,0 +1,44 @@
+require 'message/q'
+require 'message/job'
+require 'message/worker'
+require 'logger'
+
+module Message
+  module_function
+  def queue(name=nil)
+    if name
+      Q.init(name)
+    else
+      Q
+    end
+  end
+
+  def job(name=nil, &block)
+    if name
+      Job.new(queue(name), &block)
+    else
+      Job
+    end
+  end
+
+  def worker(job=nil)
+    Worker.new(job)
+  end
+
+  def logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def reset
+    Message.queue.reset
+    Message.job.reset
+  end
+
+  reset
+end
+
+Object.send(:include, Message::Worker::Enqueue)

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: message
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Xiao Li
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-09 00:00:00.000000000 Z
+dependencies: []
+description: ! 'Message provides flexible & reliable background/asynchronous job processing
+  mechanism on top of simple queue interface.
+
+
+  Any developer can create queue adapter for Message to hook up different messaging/queue
+  system.
+
+
+  One in-memory queue is included with Message for you to start development and test,
+
+  and you can easily swap in other queues later.
+
+'
+email:
+- swing1979@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/message/filters.rb
+- lib/message/in_memory_queue.rb
+- lib/message/job.rb
+- lib/message/q.rb
+- lib/message/worker.rb
+- lib/message.rb
+homepage: https://github.com/xli/message
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Simplify Ruby messaging/queue/async/job processing.
+test_files: []