arsenicum 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 51962f53bdcb4a46365d2e73dc6730fb663b1436
+  data.tar.gz: 75cee77e64c8c6620006b6d6cb6452544319cbbe
+SHA512:
+  metadata.gz: 2307adda63e934e0eab37770afd623cbd5d81d9fe08ed2c9277ec4b5021f039b42230109cb8fdbb1364526bfa58c5ec9da1fa9b0a71f7c7ffd0f9bdd8c05d31e
+  data.tar.gz: 0cabdc6010016709c9b0ca3dde02838812a7eda4af21837ff202af6b9a4b26d6084468486f51a61c952f33612349b368c25b5a39d1e12f9d5ab26a875d3d6433

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/config.yml
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in arsenicum.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 condor
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# Arsenicum
+
+Asyncronous processing engine that can use any queue storage including Amazon SQS and so on.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'arsenicum'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install arsenicum
+
+## Usage
+
+You can boot arsenicum.
+
+	$ bundle exec arsenicum
+
+If you use arsenicum with rails, you should boot arsenicum_rails as below:
+
+	$ bundle exec arsenicum_rails
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/arsenicum.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'arsenicum/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "arsenicum"
+  spec.version       = Arsenicum::VERSION
+  spec.authors       = ["condor"]
+  spec.email         = ["condor1226@gmail.com"]
+  spec.description   = %q{Arsenicum: multi-backend asyncronous processor.}
+  spec.summary       = %q{Arsenicum: multi-backend asyncronous processor.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/).reject{|s|s == 'build.rake'}
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'aws-sdk'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/bin/arsenicum

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'arsenicum'
+require 'optparse'
+require 'yaml'
+
+config = {}
+opt = OptionParser.new
+
+opt.on('-c', '--config-file=CONFIG_FILE'){|v|config.merge! YAML.load(File.read(v))}
+# TODO implement other options
+
+Arsenicum::Server.start config

data/bin/arsenicum_rails

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'arsenicum'
+
+Arsenicum::CLI::Rails.new(ARGV).boot

data/lib/arsenicum-rails.rb

@@ -0,0 +1,10 @@
+require 'arsenicum'
+require 'rails'
+
+module Arsenicum
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load 'arsenicum/rake_tasks.rake'
+    end
+  end
+end

data/lib/arsenicum.rb

@@ -0,0 +1,14 @@
+module Arsenicum
+  autoload :Version,        'arsenicum/version'
+  autoload :Queue,          'arsenicum/queue'
+  autoload :Task,           'arsenicum/task'
+  autoload :Configuration,  'arsenicum/configuration'
+  autoload :Serialization,  'arsenicum/serialization'
+  autoload :WatchDog,       'arsenicum/watchdog'
+  autoload :QueueProxy,     'arsenicum/queue_proxy'
+  autoload :Syntax,         'arsenicum/syntax'
+  autoload :Sqs,            'arsenicum/sqs'
+  autoload :CLI,            'arsenicum/cli'
+  autoload :Server,         'arsenicum/server'
+  autoload :Actor,          'arsenicum/actor'
+end

data/lib/arsenicum/actor.rb

@@ -0,0 +1,14 @@
+module Arsenicum
+  class Actor
+    attr_reader :queue
+
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def process(task)
+      task.execute
+      queue.update_message_status(task.message_id, task.successful?)
+    end
+  end
+end

data/lib/arsenicum/cli.rb

@@ -0,0 +1,54 @@
+require 'optparse'
+require 'yaml'
+
+module Arsenicum
+  class CLI
+    autoload :Rails, 'arsenicum/cli/rails'
+
+    attr_reader :configuration
+
+    def initialize(argv)
+      @configuration = option_parser.parse!(argv)
+    end
+
+    def boot
+      Arsenicum::Server.start(Arsenicum::Configuration.new(configuration))
+    end
+
+    class OptionParser
+      def initialize
+        @values = {}
+        @parser = ::OptionParser.new
+      end
+
+      def register(*args)
+        block = args.pop
+        block = block.to_proc unless block.is_a? Proc
+
+        if block.arity == 2
+          @parser.on(*args) {|v|block.call(v, @values)}
+        else
+          @parser.on(*args) {|v|@values.merge! block.call(v)}
+        end
+        self
+      end
+
+      def parse!(argv)
+        @parser.parse! argv
+        @values
+      end
+    end
+
+    private
+    def option_parser
+      OptionParser.new.
+        register("-c", "--config-file=YAML", -> v {YAML.load(File.read(v, encoding: "UTF-8"))}).
+        register("-t", "--default-concurrency=VALUE", -> v {{default_concurrency: v.to_i}}).
+        register("-q", "--queue-type=QUEUE_TYPE", -> v{{queue_type: v.to_s}}).
+        register("--queue-engine-config=CONFIGKEY_VALUE", -> v, config {
+          config[:engine_config] ||= {};(key, value) = v.split(':');config[:engine_config][key.to_sym] = value.to_s
+        })
+    end
+
+  end
+end

data/lib/arsenicum/cli/rails.rb

@@ -0,0 +1,26 @@
+module Arsenicum
+  class CLI::Rails < CLI
+    def boot
+      ENV['RACK_ENV'] = (ENV['RAILS_ENV'] ||= (configuration[:rails_env] || 'development'))
+      rootdir = ENV['RAILS_ROOT'] || configuration[:dir] || Dir.pwd
+      Dir.chdir rootdir
+
+      if configuration[:background] && !configuration[:pidfile]
+        configuration[:pidfile] = "#{rootdir}/tmp/pids/arsenicum.pid"
+      end
+
+      load File.join(rootdir, 'config/environment.rb')
+      Arsenicum::Configuration.reconfigure configuration
+      Arsenicum::Server.start
+    end
+
+    def option_parser
+      OptionParser.new.register("-e", "--environment=ENVIRONMENT", -> v { {rails_env: v} }).
+        register("-d", "--dir=DIRECTORY", -> v { {dir: v} }).
+        register("-p", "--pidfile=PID_FILE", -> v { { pidfile: v } }).
+        register("-l", "--log-file=LOG_FILE", -> v { { log_file: v } }).
+        register("-D", "--daemon", -> v { { background: true } }).
+        register("-L", "--log-level=LOG_LEVEL", -> v { {log_level: v} })
+    end
+  end
+end

data/lib/arsenicum/configuration.rb

@@ -0,0 +1,92 @@
+require 'logger'
+
+module Arsenicum
+  class Configuration
+    attr_accessor :queue_namespace, :queue_type, :queue_configurations, :engine_configuration, :pidfile, :background, :log_level
+    attr_reader :logger
+
+    DEFAULT_QUEUES = {
+      default: {
+        concurrency: 2,
+      },
+    }.freeze
+
+    class << self
+      def configure(configuration)
+        @instance = new(configuration)
+      end
+      attr_reader :instance
+
+      def reconfigure(settings)
+        instance.reconfigure additional_settings: settings
+      end
+    end
+
+    def initialize(settings)
+      @log_level = Logger::INFO
+      @logger = Logger.new(STDOUT)
+
+      @original_settings = {queues: DEFAULT_QUEUES}.merge(normalize_hash_key(settings))
+      raise MisconfigurationError, "queue_type is required" unless @original_settings[:queue_type]
+
+      reconfigure
+    end
+
+    def reconfigure(additional_settings: nil)
+      @original_settings.merge! additional_settings if additional_settings
+      settings = @original_settings.dup
+
+      @pidfile = settings.delete(:pidfile)
+      @background = (settings.delete(:background).to_s.downcase == "true")
+      @queue_type = settings.delete(:queue_type).to_s
+      @engine_configuration = settings[queue_type.to_sym]
+      @queue_namespace = queue_type.gsub(/_([a-z])/){|_|$1.upcase}.gsub(/^([a-z])/){|_|$1.upcase}.to_sym
+
+      queue_settings = settings.delete(:queues)
+      @queue_configurations = queue_settings.inject({}) do |h, kv|
+        (queue_name, queue_setting) = kv
+        h[queue_name] = queue_setting
+        h
+      end
+
+      if log_level = settings.delete(:log_level)
+        @log_level = Logger.const_get(log_level.upcase.to_sym)
+      end
+
+      if log_file = settings.delete(:log_file)
+        self.logger = Logger.new(log_file)
+      end
+    end
+
+    def logger=(new_logger)
+      @logger = new_logger
+      if @logger
+        @logger.level = log_level
+      end
+    end
+
+    def create_queues
+      queue_configurations.map do |queue_name_config|
+        (queue_name, queue_config) = queue_name_config
+        queue_class.new(queue_name, engine_configuration.merge(queue_config), logger)
+      end
+    end
+
+    def queue_class
+      Arsenicum.const_get(queue_namespace).const_get(:Queue)
+    end
+
+    private
+    def normalize_hash_key(hash, to_sym: true)
+      hash.inject({}) do |h, kv|
+        (key, value) = kv
+        value = normalize_hash_key(value, to_sym: to_sym) if value.is_a? Hash
+        key = to_sym ? key.to_sym : key.to_s
+        h[key] = value
+        h
+      end
+    end
+  end
+
+  class MisconfigurationError < StandardError;end
+end

data/lib/arsenicum/queue.rb

@@ -0,0 +1,58 @@
+require 'json'
+
+module Arsenicum
+  class Queue
+    DEFAULT_CONCURRENCY = 5
+
+    attr_reader :name, :concurrency, :queue_methods, :queue_classes, :logger
+
+    def initialize(name, config = {}, logger = nil)
+      @name = name
+      @concurrency = (config.delete(:concurrency) || DEFAULT_CONCURRENCY).to_i
+      @queue_methods = config.delete(:methods)
+      @queue_classes = config.delete(:classes)
+      @logger = logger || Logger.new(STDOUT)
+      configure(config)
+    end
+
+    def put(hash)
+      json = JSON(hash.merge(timestamp: (Time.now.to_f * 1000000).to_i.to_s))
+      logger.debug { "Queue Put[#{name}] values #{json}" }
+      put_to_queue(json)
+    end
+
+    ######################################################
+    #
+    # Queue must implement the methods as below:
+    #   1. put_to_queue(json): putting the actual message
+    #     into the queue backend. The argument of this
+    #     method will be the JSON string.
+    #   2. poll: polling the queue and retrieve the
+    #     message information. This method is expected to
+    #     return the message Hash. Its keys and values
+    #     are expected as below:
+    #       :message_body: the raw string that was pushed
+    #         via the :put_to_queue method.
+    #       :message_id: the identifier of this message.
+    #         This is usually used to update the status of
+    #         message on the queue backend.
+    #   3. update_message_status(message_id, successful, json):
+    #     Update the status of the message. Arguments are
+    #     as following:
+    #       message_id: The identifier of the message to
+    #         be updated. This value will be set as the
+    #         :message_id of the return value of :poll.
+    #       successful: The result of the process done.
+    #         If the process complete successfully,
+    #         this argument will be set true. Otherwise,
+    #         this will be false.
+    #       json: the message received.
+    #   4. create_queue_backend - optional
+    #     Register the queue itself on its backend.
+    #     This will be invoked from the rake task
+    #     'arsenicum:create_queues'.
+    #     Note: this method should be implemented idempotently.
+    #
+    #####################################################
+  end
+end

data/lib/arsenicum/queue_proxy.rb

@@ -0,0 +1,73 @@
+module Arsenicum
+  class QueueProxy
+    include Serialization
+
+    def self.instance
+      @instance ||= new(Arsenicum::Configuration.instance)
+    end
+
+    attr_reader :configuration
+    attr_reader :queues
+    attr_reader :default_queue
+    attr_reader :method_queue_tables
+    attr_reader :class_queue_tables
+
+    def initialize(configuration)
+      @configuration = configuration
+      queue_class = configuration.queue_class
+      @method_queue_tables = {}
+      @class_queue_tables = {}
+
+      @queues = configuration.queue_configurations.inject({}) do |h, kv|
+        (queue_name, queue_configuration) = kv
+        queue = queue_class.new(queue_name, queue_configuration.merge(configuration.engine_configuration))
+        Array(queue.queue_methods).tap(&:compact!).each do |m|
+          method_queue_tables[m] ||= queue
+        end
+        Array(queue.queue_classes).tap(&:compact!).each do |m|
+          class_queue_tables[m] ||= queue
+        end
+        h[queue_name] = queue
+
+        h
+      end
+      @default_queue = queues[:default]
+    end
+
+    def async(target, method, *arguments)
+      values = {
+        target: prepare_serialization(target),
+        method_name: method,
+        arguments: arguments.map{|arg|prepare_serialization(arg)},
+      }
+      specify_queue(target, method).
+        tap{|q|logger.debug { "Queue #{q.name}: Param #{values.inspect}" }}.
+        put(values)
+    end
+
+    def logger
+      configuration.logger
+    end
+
+    private
+    def specify_queue(target, method)
+      if target.is_a?(Module)
+        conjunction = '.'
+        klass = target
+      else
+        conjunction = '#'
+        klass = target.class
+      end
+      method_signature = [target.class.name, method].join conjunction
+      if queue = method_queue_tables[method_signature]
+        return queue
+      end
+      klass_signature = target.class.name
+      if queue = class_queue_tables[klass_signature]
+        return queue
+      end
+
+      return default_queue
+    end
+  end
+end

data/lib/arsenicum/rake_tasks.rake

@@ -0,0 +1,24 @@
+require 'arsenicum'
+require 'yaml'
+
+namespace :arsenicum do
+  desc 'Create queues defined in the configuration file. Specify configuration with CONFIG=config_file_path.'
+  task :create_queues do
+    config_file = ENV['CONFIG'] || 'config/arsenicum.yml'
+    yaml = YAML.load(Erubis::Eruby.new(File.read(config_file, encoding: 'UTF-8')).result)
+    config_values =
+      if ENV['CONFIG_KEY']
+        ENV['CONFIG_KEY'].split('.').inject(yaml) do |values, key|
+          values[key]
+        end
+      else
+        yaml
+      end
+
+    config = Arsenicum::Configuration.new(config_values)
+    queue_class = config.queue_class
+    raise Arsenicum::MisconfigurationError, "class #{queue_class.name} doesn't support create_queue" unless queue_class.instance_methods.include?(:create_queue_backend)
+
+    config.create_queues.each(&:create_queue_backend)
+  end
+end

data/lib/arsenicum/serialization.rb

@@ -0,0 +1,110 @@
+require 'date'
+require 'time'
+
+module Arsenicum
+  module Serialization
+    DATE_FORMAT = "%Y-%m-%d".freeze
+    DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S %Z %z".freeze
+
+    def prepare_serialization(value)
+      hash_value = prepare_serialization_specific(value) || prepare_serialization_default(value)
+    end
+
+    def prepare_serialization_specific(value)
+      case value
+      when Integer, Float, String, TrueClass, FalseClass, NilClass
+        {
+          type: :raw,
+          value: value.inspect,
+        }
+      when Date
+        {
+          type: 'date',
+          value: value.strftime(DATE_FORMAT),
+        }
+      when DateTime, Time
+        {
+          type: 'time',
+          value: value.strftime(DATE_TIME_FORMAT),
+        }
+      when Class
+        {
+          type: :class,
+          value: value.name,
+        }
+      when Array
+        {
+          type: :array,
+          values: value.map{|v|serialize(v)},
+        }
+      when Hash
+        {
+          type: :hash,
+          values: value.inject({}){|h, kv|(k,v)=kv;h[k.to_s]=serialize(v);h},
+        }
+      end
+    end
+
+    def prepare_serialization_default(value)
+      {
+        type: 'marshal',
+        value: Marshal.dump(value).unpack('H*').first,
+      }
+    end
+
+    module_function :prepare_serialization_specific, :prepare_serialization_default, :prepare_serialization
+
+    module WithActiveRecord
+      def self.included(base)
+        base.module_eval do
+          alias_method :prepare_serialization_specific_original, :prepare_serialization_specific
+
+          def prepare_serialization_specific(value)
+            prepare_serialization_specific_original(value) || prepare_serialization_active_record(value)
+          end
+
+          private
+          def prepare_serialization_active_record(value)
+            return {
+              type: :active_record,
+              class: value.class.name,
+              id: value.id,
+            } if value.is_a? ActiveRecord::Base
+          end
+
+          module_function :prepare_serialization_specific_original, :prepare_serialization_active_record
+        end
+      end
+    end
+
+    include(WithActiveRecord) if defined? ::ActiveRecord::Base
+
+    def restore(value)
+      case value['type']
+      when 'raw'
+        eval value['value']
+      when 'date'
+        Date.strptime(value['value'], DATE_FORMAT)
+      when 'time'
+        Time.strptime(value['value'], DATE_TIME_FORMAT)
+      when 'class'
+        Module.const_get value['value'].to_sym
+      when 'active_record'
+        klass = const_get value['class'].to_sym
+        klass.find value['id']
+      when 'array'
+        value['values'].map do |value|
+          restore(value)
+        end
+      when 'hash'
+        value['values'].inject({}) do |h, key_value|
+          (key, value) = key_value
+          h[key.to_sym] = restore(key_value)
+          h
+        end
+      else
+        Marshal.restore [value['value']].pack('H*')
+      end
+    end
+  end
+end

data/lib/arsenicum/server.rb

@@ -0,0 +1,40 @@
+module Arsenicum
+  class Server
+    attr_reader :watchdogs
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def self.start(config = Arsenicum::Configuration.instance)
+      Process.daemon(true, true) if config.background
+      File.open(config.pidfile, 'w'){|f|f.puts $$} if config.pidfile
+      new(config).start
+    end
+
+    def start
+      puts "Booting Arsenicum Server..."
+      Signal.trap(:INT, &method(:trap_interruption))
+
+      queue_class = config.queue_class
+      @watchdogs = config.create_queues.map do |queue|
+        Arsenicum::WatchDog.new(queue, config.logger)
+      end
+      @watchdogs.each(&:boot)
+
+      loop { sleep 10 }
+    end
+
+    def shutdown
+      @watchdogs.each(&:shutdown)
+      File.delete config.pidfile if config.pidfile
+      Thread.current.terminate
+    end
+
+    private
+    def trap_interruption(*)
+      shutdown
+    end
+  end
+end

data/lib/arsenicum/sqs.rb

@@ -0,0 +1,5 @@
+module Arsenicum
+  module Sqs
+    autoload :Queue,  'arsenicum/sqs/queue'
+  end
+end

data/lib/arsenicum/sqs/queue.rb

@@ -0,0 +1,76 @@
+require 'aws-sdk'
+
+module Arsenicum::Sqs
+  class Queue < Arsenicum::Queue
+    attr_reader :account
+    attr_reader :sqs
+    attr_reader :wait_timeout
+    attr_reader :failure_queue_name
+    attr_reader :queue_configuration
+
+    def configure(config)
+      @account = config.delete :account
+      @sqs = AWS::SQS.new account
+      @wait_timeout =
+        if config.delete(:long_poll)
+          nil
+        elsif timeout = config.delete(:wait_timeout)
+          timeout.to_i
+        end
+      @failure_queue_name = config.delete :failure_queue_name
+
+      @queue_configuration = config
+    end
+
+    def put_to_queue(json, named: name)
+      sqs_queue = sqs.queues.named(named.to_s)
+      sqs_queue.send_message(json)
+    end
+
+    def poll
+      sqs.queues.named(name.to_s).poll(wait_time_out: wait_timeout) do |message|
+        logger.debug { "RAW_MESSAGE #{message.inspect}" }
+        {
+          message_body: message.body,
+          message_id: message.handle,
+        }
+      end
+    end
+
+    def update_message_status(message_id, successful, json)
+      put_to_queue(json, named: failure_queue_name) unless successful
+
+      sqs_queue = sqs.named(name)
+      sqs.client.delete_message queue_url: sqs_queue.url, receipt_handle: message_id
+    end
+
+    CREATION_OPTIONS = [
+      :visibility_timeout, :maximum_message_size,
+      :delay_seconds, :message_retention_period,
+    ].freeze
+    CREATION_OPTIONS_IN_JSON = [
+      :policy,
+    ].freeze
+
+    def create_queue_backend
+      creation_options = queue_configuration.values_at(*CREATION_OPTIONS).
+        each_with_index.inject({}) do |opts, vi|
+
+        (value, i) = vi
+        opts[CREATION_OPTIONS[i]] = value if value
+        opts
+      end
+      CREATION_OPTIONS_IN_JSON.each do |opt|
+        creation_options[opt] = JSON(queue_configuration[opt]) if queue_configuration[opt]
+      end
+
+      begin
+        sqs.queues.named(name.to_s)
+        puts "Not Created Queue #{name}:Exists"
+      rescue AWS::SQS::Errors::NonExistentQueue
+        sqs.queues.create name.to_s, creation_options
+        puts "Created Queue #{name}"
+      end
+    end
+  end
+end

data/lib/arsenicum/syntax.rb

@@ -0,0 +1,11 @@
+module Arsenucum
+  module Syntax
+    autoload :DelayedJob, 'arsenicum/syntax/delayed_job'
+
+    def self.choose(syntax)
+      syntax_impl =
+        const_get syntax.to_s.gsub(/_([a-z])/){$1.upcase}.gsub(/^([a-z])/){$1.upcase}.to_sym
+      syntax_impl.enable!
+    end
+  end
+end

data/lib/arsenicum/syntax/delayed_job.rb

@@ -0,0 +1,25 @@
+module Arsenicum::Syntax
+  module DelayedJob
+    class DelayedObject < BasicObject
+      def initialize(wrapped_object)
+        @wrapped_object = wrapped_object
+      end
+
+      def method_missing(method_id, *arguments)
+        Arsenicum::QueueProxy.instance.async(@wrapped_object, method_id, *arguments)
+      end
+    end
+
+    module ObjectExt
+      def delay
+        DelayedObject.new(self)
+      end
+    end
+
+    class <<self
+      def enable!
+        Object.__send__(:include, ObjectExt)
+      end
+    end
+  end
+end

data/lib/arsenicum/task.rb

@@ -0,0 +1,53 @@
+require 'json'
+
+module Arsenicum
+  class Task
+    include Serialization
+
+    attr_reader :target, :method, :arguments, :timestamp, :message_id, :exception
+
+    def self.parse(raw_message, message_id)
+      message_content = JSON(raw_message)
+
+      timestamp = message_content['timestamp']
+      method = message_content['method_name'].to_sym
+
+      target = restore(message_content['target'])
+      arguments = message_content['arguments'].nil? ? [] :
+        message_content['arguments'].map{|arg|restore(arg)}
+
+      new(target, method, arguments, timestamp, message_id)
+    end
+
+    def initialize(target, method, arguments, timestamp, message_id)
+      @target     = target
+      @method     = method
+      @arguments  = arguments
+      @timestamp  = timestamp
+      @message_id = message_id
+    end
+
+    def prepare_serialization
+      {
+        target: prepare_serialization(target),
+        timestamp: (Time.now.to_f * 1000000).to_i,
+        method_name: method_name,
+        arguments: arguments.nil? ? nil : arguments.map{|arg|prepare_serialization(arg)},
+      }
+    end
+
+    def serialize
+      JSON(prepare_serialization)
+    end
+
+    def execute!
+      target.__send__ method, *arguments
+    rescue Exception => e
+      @exception = e
+    end
+
+    def successful?
+      !exception
+    end
+  end
+end

data/lib/arsenicum/version.rb

@@ -0,0 +1,3 @@
+module Arsenicum
+  VERSION = "0.0.1"
+end

data/lib/arsenicum/watchdog.rb

@@ -0,0 +1,68 @@
+module Arsenicum
+  class WatchDog
+    attr_reader :queue, :logger
+
+    def initialize(queue, logger)
+      @queue = queue
+      @logger = logger
+
+      @task_queue = Array.new
+      @mutex = Mutex.new
+      @workers = queue.concurrency.times.map{|_|create_worker}
+    end
+
+    def boot
+      @main_thread = Thread.new do
+        loop do
+          message = queue.poll
+          logger.debug { "received message #{message.inspect}" }
+          next unless message
+
+          # FIXME: overtime queue stocking.
+          @mutex.synchronize do
+            @task_queue.push Task.parse(message[:message_body], message[:message_id])
+          end
+        end
+      end
+    end
+
+    def shutdown
+      @workers.each do |worker|
+        begin
+          worker.terminate
+        rescue Exception
+        end
+      end
+      begin
+        @main_thread.terminate
+      rescue Exception
+      end
+    end
+
+    private
+    def create_worker
+      Worker.new(@task_queue, @queue, @mutex)
+    end
+
+    #:nodoc:
+    class Worker < ::Thread
+      attr_reader :running
+
+      def initialize(task_queue, queue, mutex)
+        super do
+          loop do
+            mutex.synchronize { task = task_queue.shift }
+
+            unless task
+              sleep 0.1
+              next
+            end
+
+            task.execute!
+            queue.update_message_status(task.message_id, task.successful?, task.serialize)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/config.example.yml

@@ -0,0 +1,21 @@
+queue_type: sqs
+queues:
+  default:
+    concurrency: 10
+  sample01:
+    concurrency: 3
+    methods:
+      - 'SampleA#hoge1'
+      - 'SampleB.hoge2'
+    classes:
+      - 'ClassInvocation1'
+  sample02:
+    concurrency: 2
+    methods:
+      - 'SampleA#hoge2'
+      - 'SampleB.hoge1'
+sqs:
+  account:
+    access_key_id: 'YOUR_ACCESS_KEY_ID'
+    secret_access_key: 'YOUR SECRET ACCESS KEY'
+    region: 'ap-northeast-1'

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: arsenicum
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- condor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Arsenicum: multi-backend asyncronous processor.'
+email:
+- condor1226@gmail.com
+executables:
+- arsenicum
+- arsenicum_rails
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- arsenicum.gemspec
+- bin/arsenicum
+- bin/arsenicum_rails
+- lib/arsenicum-rails.rb
+- lib/arsenicum.rb
+- lib/arsenicum/actor.rb
+- lib/arsenicum/cli.rb
+- lib/arsenicum/cli/rails.rb
+- lib/arsenicum/configuration.rb
+- lib/arsenicum/queue.rb
+- lib/arsenicum/queue_proxy.rb
+- lib/arsenicum/rake_tasks.rake
+- lib/arsenicum/serialization.rb
+- lib/arsenicum/server.rb
+- lib/arsenicum/sqs.rb
+- lib/arsenicum/sqs/queue.rb
+- lib/arsenicum/syntax.rb
+- lib/arsenicum/syntax/delayed_job.rb
+- lib/arsenicum/task.rb
+- lib/arsenicum/version.rb
+- lib/arsenicum/watchdog.rb
+- spec/config.example.yml
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: 'Arsenicum: multi-backend asyncronous processor.'
+test_files:
+- spec/config.example.yml