arsenicum 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51962f53bdcb4a46365d2e73dc6730fb663b1436
-  data.tar.gz: 75cee77e64c8c6620006b6d6cb6452544319cbbe
+  metadata.gz: 5e91e416b5ebf874c42c0f1f5c8e434b7c9018a2
+  data.tar.gz: 44069bbc46faf4e1f8867a5b763773f81c9263d6
 SHA512:
-  metadata.gz: 2307adda63e934e0eab37770afd623cbd5d81d9fe08ed2c9277ec4b5021f039b42230109cb8fdbb1364526bfa58c5ec9da1fa9b0a71f7c7ffd0f9bdd8c05d31e
-  data.tar.gz: 0cabdc6010016709c9b0ca3dde02838812a7eda4af21837ff202af6b9a4b26d6084468486f51a61c952f33612349b368c25b5a39d1e12f9d5ab26a875d3d6433
+  metadata.gz: dcf5f344a81eb8ccc9158c62b8a40b5304bc5d0d313c75092b2d059f8df0f1d23bad18528e43cf5cb04de5ee21536d6ea9006fc3688b99689312b18381308666
+  data.tar.gz: a8f8f594bd478b1c484f7a08a3bad399827c392dd470723246f3cfd02a786cf4d62606165c4319c204983cc8ed91c7c89eed8700f3bfe4d9fd0038ec377bc0ee

data/.gitignore

@@ -12,7 +12,8 @@ lib/bundler/man
 pkg
 rdoc
 spec/reports
-spec/config.yml
+spec/config/config.yml
+spec/config/config.rb
 test/tmp
 test/version_tmp
 tmp

data/arsenicum.gemspec

@@ -19,7 +19,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'aws-sdk'
+  spec.add_dependency 'multi_json'
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rails', '~> 4.0'
+  spec.add_development_dependency 'sqlite3'
 end

data/bin/arsenicum

@@ -4,10 +4,4 @@ require 'arsenicum'
 require 'optparse'
 require 'yaml'
 
-config = {}
-opt = OptionParser.new
-
-opt.on('-c', '--config-file=CONFIG_FILE'){|v|config.merge! YAML.load(File.read(v))}
-# TODO implement other options
-
-Arsenicum::Server.start config
+Arsenicum::CLI.new(ARGV).boot

data/lib/arsenicum.rb

@@ -1,14 +1,25 @@
 module Arsenicum
-  autoload :Version,        'arsenicum/version'
-  autoload :Queue,          'arsenicum/queue'
-  autoload :Task,           'arsenicum/task'
-  autoload :Configuration,  'arsenicum/configuration'
-  autoload :Serialization,  'arsenicum/serialization'
-  autoload :WatchDog,       'arsenicum/watchdog'
-  autoload :QueueProxy,     'arsenicum/queue_proxy'
-  autoload :Syntax,         'arsenicum/syntax'
-  autoload :Sqs,            'arsenicum/sqs'
-  autoload :CLI,            'arsenicum/cli'
-  autoload :Server,         'arsenicum/server'
-  autoload :Actor,          'arsenicum/actor'
+  autoload  :Configuration,         'arsenicum/configuration'
+  autoload  :MisconfigurationError, 'arsenicum/configuration'
+  autoload  :Core,                  'arsenicum/core'
+  autoload  :Util,                  'arsenicum/util'
+  autoload  :Version,               'arsenicum/version'
+  autoload  :Serializer,            'arsenicum/serializer'
+  autoload  :Formatter,             'arsenicum/formatter'
+  autoload  :Async,                 'arsenicum/async'
+  autoload  :Main,                  'arsenicum/main'
+  autoload  :IO,                    'arsenicum/io'
+  autoload  :Task,                  'arsenicum/task'
+
+  class << self
+    def configure(arg = nil, &block)
+      Arsenicum::Configuration.configure arg, &block
+    end
+
+    def configuration
+      Configuration.instance
+    end
+
+    alias_method :config, :configuration
+  end
 end

data/lib/arsenicum/async.rb

@@ -0,0 +1,3 @@
+module Arsenicum::Async
+  autoload  :Queue, 'arsenicum/async/queue'
+end

data/lib/arsenicum/async/queue.rb

@@ -0,0 +1,37 @@
+class Arsenicum::Async::Queue < Arsenicum::Core::Broker
+
+  attr_accessor :broker
+  attr_reader   :name,  :worker_count,  :router
+  attr_reader   :broker
+
+  def initialize(name, options)
+    @name = name
+    @worker_count = options.delete(:worker_count)
+    @router       = options.delete(:router)
+    @broker       = Arsenicum::Core::Broker.new worker_count: worker_count, router: router
+  end
+
+  def start
+    loop do
+      (message, success_handler, failure_handler) = pick
+      next sleep(0.5) unless message
+
+      begin
+        broker.delegate message
+        success_handler.call if success_handler
+      rescue Exception => e
+        failure_handler.call(e) if failure_handler
+      end
+    end
+  end
+
+  def register_task(task)
+    broker[task.id] = task
+  end
+
+  def start_async
+    Thread.new{start}
+  end
+
+  autoload  :Sqs, 'arsenicum/async/queue/sqs'
+end

data/lib/arsenicum/async/queue/sqs.rb

@@ -0,0 +1,27 @@
+require 'aws-sdk'
+require 'multi_json'
+
+class Arsenicum::Async::Queue::Sqs < Arsenicum::Async::Queue
+  attr_reader :sqs_queue, :via_sns
+
+  def initialize(name, options = {})
+    super name, options
+    sqs_args = [options[:aws_account]].tap(&:compact!)
+    sqs = AWS::SQS.new *sqs_args
+    @sqs_queue = sqs.queues.named(name)
+    @via_sns = options[:via_sns]
+  end
+
+  def pick
+    loop do
+      message = sqs_queue.receive_message
+      next sleep(0.5) unless message
+
+      message = message.as_sns_message if via_sns
+      [
+          MultiJson.decode(message.body),
+        -> { message.delete },
+      ]
+    end
+  end
+end

data/lib/arsenicum/configuration.rb

@@ -1,92 +1,143 @@
 require 'logger'
 
 module Arsenicum
+  class MisconfigurationError < StandardError;end
+
   class Configuration
-    attr_accessor :queue_namespace, :queue_type, :queue_configurations, :engine_configuration, :pidfile, :background, :log_level
-    attr_reader :logger
-
-    DEFAULT_QUEUES = {
-      default: {
-        concurrency: 2,
-      },
-    }.freeze
-
-    class << self
-      def configure(configuration)
-        @instance = new(configuration)
-      end
-      attr_reader :instance
+    attr_reader :pidfile_path,  :queue_configurations
 
-      def reconfigure(settings)
-        instance.reconfigure additional_settings: settings
-      end
+    def initialize
+      @pidfile_path = 'arsenicum.pid'
     end
 
-    def initialize(settings)
-      @log_level = Logger::INFO
-      @logger = Logger.new(STDOUT)
+    def queue_configurations
+      @queue_configurations ||= []
+    end
 
-      @original_settings = {queues: DEFAULT_QUEUES}.merge(normalize_hash_key(settings))
-      raise MisconfigurationError, "queue_type is required" unless @original_settings[:queue_type]
+    def queue(name, &block)
+      queue_config = QueueConfiguration.new name
+      queue_config.instance_eval &block if block_given?
+      queue_configurations << queue_config
+    end
 
-      reconfigure
+    def pidfile(path)
+      @pidfile_path = path
     end
 
-    def reconfigure(additional_settings: nil)
-      @original_settings.merge! additional_settings if additional_settings
-      settings = @original_settings.dup
-
-      @pidfile = settings.delete(:pidfile)
-      @background = (settings.delete(:background).to_s.downcase == "true")
-      @queue_type = settings.delete(:queue_type).to_s
-      @engine_configuration = settings[queue_type.to_sym]
-      @queue_namespace = queue_type.gsub(/_([a-z])/){|_|$1.upcase}.gsub(/^([a-z])/){|_|$1.upcase}.to_sym
-
-      queue_settings = settings.delete(:queues)
-      @queue_configurations = queue_settings.inject({}) do |h, kv|
-        (queue_name, queue_setting) = kv
-        h[queue_name] = queue_setting
-        h
+    class InstanceConfiguration
+      include Arsenicum::Util
+
+      attr_reader :name, :init_parameters, :klass
+
+      class << self
+        attr_reader :inside
+        private
+        def namespace(mod)
+          @inside = mod
+        end
       end
 
-      if log_level = settings.delete(:log_level)
-        @log_level = Logger.const_get(log_level.upcase.to_sym)
+      def initialize(name)
+        @name = name
       end
 
-      if log_file = settings.delete(:log_file)
-        self.logger = Logger.new(log_file)
+      def inside
+        self.class.inside
+      end
+
+      def type(type_name)
+        @klass = constantize(classify(type_name))
+      rescue NameError
+        @klass = constantize(classify(type_name), inside: inside)
       end
-    end
 
-    def logger=(new_logger)
-      @logger = new_logger
-      if @logger
-        @logger.level = log_level
+      def init_params(&block)
+        params = ConfigurationHash.new
+        if block
+          params.under_configuration do
+            params.instance_eval(&block)
+          end
+        end
+        @init_parameters = params
+      end
+
+      def build
+        klass.new(name, init_parameters)
       end
     end
 
-    def create_queues
-      queue_configurations.map do |queue_name_config|
-        (queue_name, queue_config) = queue_name_config
-        queue_class.new(queue_name, engine_configuration.merge(queue_config), logger)
+    class QueueConfiguration < Arsenicum::Configuration::InstanceConfiguration
+      attr_reader :worker_count,  :task_configurations
+      namespace Arsenicum::Async::Queue
+
+      def initialize(name)
+        super(name)
+        @worker_count = 2
+      end
+
+      def workers(count)
+        @worker_count = count
+      end
+
+      def task_configurations
+        @task_configurations ||= []
+      end
+
+      def task(name, &block)
+        task_config = TaskConfiguration.new name
+        task_config.instance_eval &block if block_given?
+        task_configurations << task_config
+      end
+
+      def build
+        super.tap do |queue|
+          task_configurations.each do |task_config|
+            queue.register task_config.build
+          end
+        end
       end
     end
 
-    def queue_class
-      Arsenicum.const_get(queue_namespace).const_get(:Queue)
+    class TaskConfiguration  < Arsenicum::Configuration::InstanceConfiguration
+      namespace Arsenicum::Task
     end
 
-    private
-    def normalize_hash_key(hash, to_sym: true)
-      hash.inject({}) do |h, kv|
-        (key, value) = kv
-        value = normalize_hash_key(value, to_sym: to_sym) if value.is_a? Hash
-        key = to_sym ? key.to_sym : key.to_s
-        h[key] = value
-        h
+    class ConfigurationHash < Hash
+      def in_configuration?
+        @in_configuration
+      end
+
+      def under_configuration(&_)
+        @in_configuration = true
+        yield if block_given?
+      ensure
+        @in_configuration = false
+      end
+
+      def method_missing(method_id, *args, &block)
+        case args.length
+          when 0
+            return self[method_id] unless in_configuration?
+
+            if block_given?
+              new_value = ConfigurationHash.new
+              new_value.under_configuration do
+                new_value.instance_eval &block
+              end
+              self[method_id] = new_value
+            else
+              self[method_id] ||= ConfigurationHash.new
+            end
+          when 1
+            if (method_name = method_id.to_s)[-1] == '='
+              return self[method_name[0...-1].to_sym] = args.first
+            end
+
+            self[method_id] = args.first
+          else
+            super
+        end
       end
     end
   end
-
-  class MisconfigurationError < StandardError;end
 end

data/lib/arsenicum/core.rb

@@ -0,0 +1,6 @@
+module Arsenicum::Core
+  autoload  :Broker,              'arsenicum/core/broker'
+  autoload  :Commands,            'arsenicum/core/commands'
+  autoload  :IOHelper,            'arsenicum/core/io_helper'
+  autoload  :Worker,              'arsenicum/core/worker'
+end

data/lib/arsenicum/core/broker.rb

@@ -0,0 +1,98 @@
+class Arsenicum::Core::Broker
+  include Arsenicum::Core::IOHelper
+
+  attr_reader :router
+
+  attr_reader :workers,     :available_workers, :mutex
+  attr_reader :worker_count, :worker_options, :tasks
+  attr_accessor :default_task
+
+  PROCESSOR_COUNT_DEFAULT = 2
+
+  def initialize(options = {})
+    @worker_count = (options.delete(:worker_count) || PROCESSOR_COUNT_DEFAULT).to_i
+    @mutex = Mutex.new
+    @tasks = {}
+    @router = options.delete :router
+
+    serializer = options[:serializer]  ||  Arsenicum::Serializer::JSON.new
+    formatter  = options[:formatter]   ||  Arsenicum::Formatter.new
+    @worker_options = options.delete(:worker_options) || {}
+    @worker_options.merge! serializer: serializer,  formatter: formatter
+  end
+
+  def register_task(task_id, task)
+    tasks[task_id.to_sym] = task
+  end
+
+  def [](task_id)
+    tasks[task_id.to_sym] || default_task
+  end
+
+  def []=(task_id, task)
+    tasks[task_id.to_sym] = task
+  end
+
+  def run
+    @workers = {}
+    @available_workers = {}
+
+    @worker_count.times do
+      prepare_worker
+    end
+  end
+
+  def prepare_worker
+    worker = Arsenicum::Core::Worker.new(self, worker_options)
+    stock(worker)
+  end
+
+  def stock(worker)
+    mutex.synchronize do
+      pid = worker.run
+      workers[pid] = worker
+      available_workers[pid] = worker
+    end
+  end
+
+  def broker(task_id, *parameters)
+    until (worker = next_worker)
+      sleep 0.5
+    end
+
+    begin
+      worker.ask task_id, *parameters
+    ensure
+      if worker.active?
+        get_back_worker(worker)
+      else
+        remove(worker)
+        prepare_worker
+      end
+    end
+  end
+
+  def delegate(message)
+    (task_id, parameters) = router.route(message)
+    broker task_id, *parameters
+  end
+
+  def remove(worker)
+    mutex.synchronize do
+      workers.delete(worker.pid)
+      available_workers.delete(worker.pid)
+    end
+  end
+
+  def next_worker
+    mutex.synchronize{available_workers.shift.last}
+  end
+
+  def get_back_worker(worker)
+    mutex.synchronize{available_workers[worker.pid] = worker}
+  end
+
+  def serialize(value = {})
+    serializer.serialize value
+  end
+end