arsenicum 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e91e416b5ebf874c42c0f1f5c8e434b7c9018a2
-  data.tar.gz: 44069bbc46faf4e1f8867a5b763773f81c9263d6
+  metadata.gz: 4866821d70fe2b9fe4dff4f307112af69edbfc2d
+  data.tar.gz: fdbaf8062df32ef04e239294e8aea76fafe0718f
 SHA512:
-  metadata.gz: dcf5f344a81eb8ccc9158c62b8a40b5304bc5d0d313c75092b2d059f8df0f1d23bad18528e43cf5cb04de5ee21536d6ea9006fc3688b99689312b18381308666
-  data.tar.gz: a8f8f594bd478b1c484f7a08a3bad399827c392dd470723246f3cfd02a786cf4d62606165c4319c204983cc8ed91c7c89eed8700f3bfe4d9fd0038ec377bc0ee
+  metadata.gz: d5b2b277ddd87b3af7c1878c0da83f35eb9b155f1c5f7f50edd39421ee3b9d5dc82cde8ce43407b08370aa6ceed5a5a871d76346981d308d38bf497fdb5307d0
+  data.tar.gz: 9939c248c291b176fd0368ad4065e341a3a166ae8f13139282f5ab2ed01acd9544503df2cb8deb1b19cd40453c493c5949d1b8c63146dd97fcd151435e937ac1

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/arsenicum/configuration.rb

@@ -4,16 +4,33 @@ module Arsenicum
   class MisconfigurationError < StandardError;end
 
   class Configuration
-    attr_reader :pidfile_path,  :queue_configurations
+    attr_reader :pidfile_path, :daemon, :stdout_path, :stderr_path, :logger_config
 
     def initialize
-      @pidfile_path = 'arsenicum.pid'
+      @pidfile_path   = 'arsenicum.pid'
+      @logger_config  = LoggerConfiguration.new
     end
 
     def queue_configurations
       @queue_configurations ||= []
     end
 
+    def daemonize
+      @daemon = true
+    end
+
+    def stdout(path)
+      @stdout_path = path
+    end
+
+    def stderr(path)
+      @stderr_path = path
+    end
+
+    def logger(&block)
+      logger_config.instance_eval &block
+    end
+
     def queue(name, &block)
       queue_config = QueueConfiguration.new name
       queue_config.instance_eval &block if block_given?
@@ -67,14 +84,20 @@ module Arsenicum
     end
 
     class QueueConfiguration < Arsenicum::Configuration::InstanceConfiguration
-      attr_reader :worker_count,  :task_configurations
-      namespace Arsenicum::Async::Queue
+      attr_reader :worker_count,  :router_class
+      namespace Arsenicum::Queue
 
       def initialize(name)
         super(name)
         @worker_count = 2
       end
 
+      def router(name)
+        @router_class = constantize(classify(name))
+      rescue NameError
+        @router_class = constantize(classify(name), inside: Arsenicum::Routing)
+      end
+
       def workers(count)
         @worker_count = count
       end
@@ -90,7 +113,7 @@ module Arsenicum
       end
 
       def build
-        super.tap do |queue|
+        klass.new(name, init_parameters.merge(router_class: router_class)).tap do |queue|
           task_configurations.each do |task_config|
             queue.register task_config.build
           end
@@ -102,6 +125,41 @@ module Arsenicum
       namespace Arsenicum::Task
     end
 
+    class LoggerConfiguration
+      attr_reader :log_path,  :log_level, :log_format
+
+      def initialize
+        @log_path   = STDOUT
+        @log_level  = :info
+      end
+
+      def path path
+        @log_path = path
+      end
+
+      def level level
+        @log_level = level
+      end
+
+      def format &format
+        @log_format = format
+      end
+
+      def build
+        logger = ::Logger.new(output_stream)
+        logger.level = ::Logger.const_get log_level.to_s.upcase.to_sym
+        logger.formatter = log_format if log_format
+        logger
+      end
+
+      private
+      def output_stream
+        return log_path if log_path.respond_to? :write
+        return File.open(log_path,  'w:UTF-8') if log_path.is_a? String
+        STDOUT
+      end
+    end
+
     class ConfigurationHash < Hash
       def in_configuration?
         @in_configuration

data/lib/arsenicum/core/broker.rb

@@ -1,10 +1,10 @@
 class Arsenicum::Core::Broker
   include Arsenicum::Core::IOHelper
 
-  attr_reader :router
+  attr_reader   :router
 
-  attr_reader :workers,     :available_workers, :mutex
-  attr_reader :worker_count, :worker_options, :tasks
+  attr_reader   :workers,       :available_workers, :mutex
+  attr_reader   :worker_count,  :worker_options,    :tasks
   attr_accessor :default_task
 
   PROCESSOR_COUNT_DEFAULT = 2
@@ -21,10 +21,6 @@ class Arsenicum::Core::Broker
     @worker_options.merge! serializer: serializer,  formatter: formatter
   end
 
-  def register_task(task_id, task)
-    tasks[task_id.to_sym] = task
-  end
-
   def [](task_id)
     tasks[task_id.to_sym] || default_task
   end
@@ -33,48 +29,30 @@ class Arsenicum::Core::Broker
     tasks[task_id.to_sym] = task
   end
 
+  alias_method :register, :[]=
+
   def run
     @workers = {}
     @available_workers = {}
 
-    @worker_count.times do
-      prepare_worker
-    end
+    prepare_workers
   end
 
-  def prepare_worker
-    worker = Arsenicum::Core::Worker.new(self, worker_options)
-    stock(worker)
-  end
+  def broker(success_handler, failure_handler, task_id, *parameters)
+    worker = loop do
+      w = next_worker
+      break w if w
 
-  def stock(worker)
-    mutex.synchronize do
-      pid = worker.run
-      workers[pid] = worker
-      available_workers[pid] = worker
-    end
-  end
-
-  def broker(task_id, *parameters)
-    until (worker = next_worker)
       sleep 0.5
     end
 
-    begin
-      worker.ask task_id, *parameters
-    ensure
-      if worker.active?
-        get_back_worker(worker)
-      else
-        remove(worker)
-        prepare_worker
-      end
-    end
+    Arsenicum::Logger.info { "[broker][Task brokering]id=#{task_id}, params=#{parameters.inspect}" }
+    worker.ask_async success_handler, failure_handler, task_id, *parameters
   end
 
-  def delegate(message)
+  def delegate(message, success_handler, failure_handler)
     (task_id, parameters) = router.route(message)
-    broker task_id, *parameters
+    broker success_handler, failure_handler, task_id, parameters
   end
 
   def remove(worker)
@@ -84,14 +62,46 @@ class Arsenicum::Core::Broker
     end
   end
 
-  def next_worker
-    mutex.synchronize{available_workers.shift.last}
+  def reload
+    workers.each(&:stop)
+
+    workers.clear
+    available_workers.clear
+
+    prepare_workers
   end
 
   def get_back_worker(worker)
     mutex.synchronize{available_workers[worker.pid] = worker}
   end
 
+  private
+  def prepare_workers
+    @worker_count.times do
+      prepare_worker
+    end
+  end
+
+  def prepare_worker
+    worker = Arsenicum::Core::Worker.new(self, worker_options)
+    stock(worker)
+  end
+
+  def stock(worker)
+    mutex.synchronize do
+      pid = worker.run
+      workers[pid] = worker
+      available_workers[pid] = worker
+    end
+  end
+
+  def next_worker
+    mutex.synchronize do
+      (_, worker) = available_workers.shift
+      worker
+    end
+  end
+
   def serialize(value = {})
     serializer.serialize value
   end

data/lib/arsenicum/core/io_helper.rb

@@ -3,7 +3,7 @@ module Arsenicum::Core::IOHelper
     string_to_write = string.dup
     string_to_write.force_encoding 'BINARY'
 
-    io.write [string.length].pack('N')
+    io.write [string.length].pack('N').force_encoding 'BINARY'
     io.write string
   end
 

data/lib/arsenicum/core/worker.rb

@@ -4,19 +4,20 @@ class Arsenicum::Core::Worker
   include Arsenicum::Core::Commands
   include Arsenicum::Core::IOHelper
 
-  attr_reader :pid, :in_parent, :out_parent,
+  attr_reader :pid, :in_parent, :out_parent, :thread,
               :in_child,  :out_child, :active, :broker, :serializer,  :formatter
   alias_method :active?, :active
 
   def initialize(broker, worker_configuration)
-    @broker = WeakRef.new broker # avoiding circular references.
+    @broker     = WeakRef.new broker # avoiding circular references.
     @serializer = worker_configuration[:serializer]
     @formatter  = worker_configuration[:formatter]
+    @thread     = InvokerThread.new(self)
   end
 
   def run
-    (@in_parent, @out_child) = IO.pipe
-    (@in_child, @out_parent) = IO.pipe
+    (@in_parent, @out_child) = open_binary_pipes
+    (@in_child, @out_parent) = open_binary_pipes
 
     @pid = fork do
       [in_parent, out_parent].each(&:close)
@@ -24,7 +25,9 @@ class Arsenicum::Core::Worker
       begin
         loop do
           begin
+            Arsenicum::Logger.debug {log_message_for 'to read initial command'}
             command = read_code in_child
+            Arsenicum::Logger.debug {log_message_for 'Command: 0x%02x' % command}
             break if command == COMMAND_STOP
             task_id_string  = read_string in_child
             content         = read_string in_child
@@ -57,17 +60,33 @@ class Arsenicum::Core::Worker
     pid
   end
 
+  def open_binary_pipes
+    IO.pipe.each do |io|
+      io.set_encoding 'BINARY'
+    end
+  end
+
+  def ask_async(success_handler, failure_handler, task_id, *parameters)
+    thread.ask success_handler, failure_handler, task_id, *parameters
+  end
+
   def ask(task_id, *parameter)
+    Arsenicum::Logger.info {log_message_for "Task ID: #{task_id}"}
+
     write_code    out_parent, COMMAND_TASK
     write_string  out_parent, task_id.to_s
     write_string  out_parent, serialize(parameter)
 
+    Arsenicum::Logger.debug { log_message_for "Task ID: #{task_id}: Request completed. Begin waiting for the reply." }
+
     result = read_code     in_parent
     return if result == 0
     raise Marshal.restore(read_string in_parent, encoding: 'BINARY')
+  ensure
+    return_to_broker
   end
 
-  def terminate
+  def stop
     write_code    out_parent, COMMAND_STOP
     Process.waitpid pid
   end
@@ -88,4 +107,44 @@ class Arsenicum::Core::Worker
       end
     end
   end
+
+  def return_to_broker
+    broker.get_back_worker self
+  end
+
+  def log_message_for(message)
+    "[Worker ##{object_id}]#{message}"
+  end
+
+  class InvokerThread < Thread
+    attr_accessor :task_request
+    private       :task_request,  :task_request=
+
+    def ask(success_handler, failure_handler, task_id, *parameters)
+      self.task_request = [success_handler, failure_handler, task_id, parameters]
+    end
+
+    def initialize(worker)
+      super do
+        loop do
+          next sleep(0.5) unless task_request
+          (success_handler, failure_handler, task_id, parameter) = task_request
+
+          begin
+            worker.ask task_id, *parameter
+            success_handler.call
+          rescue Exception => e
+            Arsenicum::Logger.error {log_message_for worker, "Exception: #{e.class.name}"}
+            failure_handler.call e
+          ensure
+            self.task_request = nil
+          end
+        end
+      end
+    end
+
+    def log_message_for(worker, message)
+      "[Worker ##{worker.object_id}][thread]#{message}"
+    end
+  end
 end

data/lib/arsenicum/logger.rb

@@ -0,0 +1,20 @@
+require 'logger'
+
+module Arsenicum::Logger
+  class << self
+    attr_reader :logger
+
+    def configure(logger_config)
+      @logger = logger_config.build
+    end
+
+    [:debug, :info, :warn, :error, :fatal].each do |method|
+      eval <<-METHOD, binding, __FILE__, __LINE__ + 1
+        def #{method}(*args, &block)
+          return unless logger
+          logger.#{method}(*args, &block)
+        end
+      METHOD
+    end
+  end
+end

data/lib/arsenicum/main.rb

@@ -7,9 +7,17 @@ module Arsenicum
       script = File.read config_file
       config.instance_eval script, config_file, 1
 
-      File.open(config.pidfile_path, 'w:UTF-8') do |f|
-        f.puts $$
+      if config.daemon
+        Process.daemonize
+
+        File.open(config.pidfile_path, 'w:UTF-8') do |f|
+          f.puts $$
+        end
       end
+
+      configure_io  config
+      configure_log config
+
       threads = config.queue_configurations.map{|qc|qc.build.start_async}
 
       begin
@@ -18,6 +26,23 @@ module Arsenicum
       end
     end
 
-    module_function :run
+    private
+    def configure_io(config)
+      $stdout = File.open(config.stdout_path, 'a:UTF-8') if config.stdout_path
+
+      if config.stderr_path
+        if config.stdout_path && config.stdout_path == config.stderr_path
+          $stderr = $stdout
+        else
+          $stderr = File.open(config.stderr_path, 'a:UTF-8')
+        end
+      end
+    end
+
+    def configure_log(config)
+      Arsenicum::Logger.configure config.logger_config
+    end
+
+    module_function :run, :configure_io,  :configure_log
   end
 end

data/lib/arsenicum/{async/queue → queue}/sqs.rb

@@ -1,7 +1,7 @@
 require 'aws-sdk'
 require 'multi_json'
 
-class Arsenicum::Async::Queue::Sqs < Arsenicum::Async::Queue
+class Arsenicum::Queue::Sqs < Arsenicum::Queue
   attr_reader :sqs_queue, :via_sns
 
   def initialize(name, options = {})
@@ -13,15 +13,14 @@ class Arsenicum::Async::Queue::Sqs < Arsenicum::Async::Queue
   end
 
   def pick
-    loop do
-      message = sqs_queue.receive_message
-      next sleep(0.5) unless message
+    message = sqs_message = sqs_queue.receive_message
+    return unless message
+    message = message.as_sns_message if via_sns
 
-      message = message.as_sns_message if via_sns
-      [
-          MultiJson.decode(message.body),
-        -> { message.delete },
-      ]
-    end
+    [MultiJson.decode(message.body), sqs_message]
+  end
+
+  def handle_success(original_message)
+    original_message.delete
   end
 end

data/lib/arsenicum/queue.rb

@@ -0,0 +1,51 @@
+require 'weakref'
+
+class Arsenicum::Queue
+
+  attr_reader   :name,  :worker_count,  :router
+  attr_reader   :broker
+
+  def initialize(name, options)
+    @name         = name
+    @worker_count = options.delete(:worker_count)
+    @router       = build_router options.delete(:router_class)
+    @broker       = Arsenicum::Core::Broker.new worker_count: worker_count, router: router
+  end
+
+  def start
+    Arsenicum::Logger.info "[queue]Queue #{name} is now starting"
+    broker.run
+    Arsenicum::Logger.info "[queue]Queue #{name} start-up completed"
+
+    loop do
+      (message, original_message) = pick
+      next sleep(0.5) unless message
+
+      broker.delegate message, -> { handle_success(original_message) }, -> e { handle_failure(e, original_message) }
+    end
+  end
+
+  def register(task)
+    broker[task.id] = task
+  end
+
+  def handle_success(original_message)
+    #TODO implement correctly in your derived classes.
+  end
+
+  def handle_failure(e, original_message)
+    #TODO implement correctly in your derived classes.
+  end
+
+  def start_async
+    Thread.new{start}
+  end
+
+  private
+  def build_router(router_class)
+    return unless router_class
+    router_class.new self
+  end
+
+  autoload  :Sqs, 'arsenicum/queue/sqs'
+end

data/lib/arsenicum/routing/default.rb

@@ -1,3 +1,5 @@
-class Arsenicum::Routing::Default
-
+class Arsenicum::Routing::Default < Arsenicum::Routing::Router
+  def route(message)
+    return [:default, message]
+  end
 end

data/lib/arsenicum/routing/router.rb

@@ -0,0 +1,9 @@
+require 'weakref'
+
+class Arsenicum::Routing::Router
+  attr_reader :broker
+
+  def initialize(broker)
+    @broker = WeakRef.new broker
+  end
+end

data/lib/arsenicum/routing.rb

@@ -1,3 +1,4 @@
 module Arsenicum::Routing
   autoload  :Default, 'arsenicum/routing/default'
+  autoload  :Router,  'arsenicum/routing/router'
 end

data/lib/arsenicum/task/class_dispatcher.rb

@@ -1,15 +1,18 @@
 class Arsenicum::Task::ClassDispatcher < Arsenicum::Task
+  include Arsenicum::Util
+
   attr_reader :target_class,  :target_method
   private     :target_class,  :target_method
 
   def initialize(id, options)
     super(id)
-    @target_class   = options.delete :type
-    @target_method  = options.delete :target
+    (klass, method) = options[:target].split('#', 2)
+    @target_class   = constantize klass
+    @target_method  = target_class.instance_method method.to_sym
   end
 
   def run(*parameters)
-    target_class.new.__send__ target_method, *parameters
+    target_method.bind(target_class.new).call *parameters
   end
 
 end

data/lib/arsenicum/version.rb

@@ -1,3 +1,3 @@
 module Arsenicum
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/arsenicum.rb

@@ -6,10 +6,12 @@ module Arsenicum
   autoload  :Version,               'arsenicum/version'
   autoload  :Serializer,            'arsenicum/serializer'
   autoload  :Formatter,             'arsenicum/formatter'
-  autoload  :Async,                 'arsenicum/async'
+  autoload  :Queue,                 'arsenicum/queue'
   autoload  :Main,                  'arsenicum/main'
   autoload  :IO,                    'arsenicum/io'
   autoload  :Task,                  'arsenicum/task'
+  autoload  :Routing,               'arsenicum/routing'
+  autoload  :Logger,                'arsenicum/logger'
 
   class << self
     def configure(arg = nil, &block)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arsenicum
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - condor
@@ -135,14 +135,12 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
 - arsenicum.gemspec
 - bin/arsenicum
 - bin/arsenicum_rails
 - lib/arsenicum-rails.rb
 - lib/arsenicum.rb
-- lib/arsenicum/async.rb
-- lib/arsenicum/async/queue.rb
-- lib/arsenicum/async/queue/sqs.rb
 - lib/arsenicum/configuration.rb
 - lib/arsenicum/core.rb
 - lib/arsenicum/core/broker.rb
@@ -151,9 +149,13 @@ files:
 - lib/arsenicum/core/worker.rb
 - lib/arsenicum/formatter.rb
 - lib/arsenicum/io.rb
+- lib/arsenicum/logger.rb
 - lib/arsenicum/main.rb
+- lib/arsenicum/queue.rb
+- lib/arsenicum/queue/sqs.rb
 - lib/arsenicum/routing.rb
 - lib/arsenicum/routing/default.rb
+- lib/arsenicum/routing/router.rb
 - lib/arsenicum/serializer.rb
 - lib/arsenicum/serializer/json.rb
 - lib/arsenicum/task.rb

data/lib/arsenicum/async/queue.rb

@@ -1,37 +0,0 @@
-class Arsenicum::Async::Queue < Arsenicum::Core::Broker
-
-  attr_accessor :broker
-  attr_reader   :name,  :worker_count,  :router
-  attr_reader   :broker
-
-  def initialize(name, options)
-    @name = name
-    @worker_count = options.delete(:worker_count)
-    @router       = options.delete(:router)
-    @broker       = Arsenicum::Core::Broker.new worker_count: worker_count, router: router
-  end
-
-  def start
-    loop do
-      (message, success_handler, failure_handler) = pick
-      next sleep(0.5) unless message
-
-      begin
-        broker.delegate message
-        success_handler.call if success_handler
-      rescue Exception => e
-        failure_handler.call(e) if failure_handler
-      end
-    end
-  end
-
-  def register_task(task)
-    broker[task.id] = task
-  end
-
-  def start_async
-    Thread.new{start}
-  end
-
-  autoload  :Sqs, 'arsenicum/async/queue/sqs'
-end

data/lib/arsenicum/async.rb

@@ -1,3 +0,0 @@
-module Arsenicum::Async
-  autoload  :Queue, 'arsenicum/async/queue'
-end