asynchronic 0.0.1 → 0.1.0

data/lib/asynchronic/queue_engine/ost.rb

@@ -0,0 +1,73 @@
+module Asynchronic
+  module QueueEngine
+    class Ost
+      
+      attr_reader :default_queue
+
+      def initialize(options={})
+        ::Ost.connect options[:redis] if options.key?(:redis)
+        @default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
+        @queues ||= Hash.new { |h,k| h[k] = Queue.new k }
+      end
+
+      def [](name)
+        @queues[name]
+      end
+
+      def queues
+        (@queues.values.map(&:key) | redis.keys('ost:*')).map { |q| q.to_s[4..-1].to_sym }
+      end
+
+      def clear
+        @queues.clear
+        redis.keys('ost:*').each { |k| redis.del k }
+      end
+
+      def listener
+        Listener.new
+      end
+
+      private
+
+      def redis
+        @redis ||= Redis.connect(::Ost.options)
+      end
+
+
+      class Queue < ::Ost::Queue
+
+        def pop
+          key.rpop
+        end
+
+        def empty?
+          !redis.exists(key)
+        end
+
+        def size
+          items.count
+        end
+
+        def to_a
+          items.reverse
+        end
+
+      end
+
+
+      class Listener
+
+        def listen(queue, &block)
+          @current_queue = queue
+          queue.each &block
+        end
+
+        def stop
+          @current_queue.stop
+        end
+
+      end
+
+    end
+  end
+end

data/lib/asynchronic/runtime.rb

@@ -0,0 +1,40 @@
+module Asynchronic
+  class Runtime
+    
+    attr_reader :process
+
+    def initialize(process)
+      @process = process
+    end
+
+    def evaluate
+      begin
+        @data = process.data
+        process.job.local.each { |k,v| define_singleton_method(k) { v } }
+        instance_eval &process.job.class.implementation
+      ensure
+        process.merge @data
+      end
+    end
+
+    def self.evaluate(process)
+      new(process).evaluate
+    end
+
+    private
+
+    def define_job(job_class, options={})
+      defaults = {
+        parent: process.job.lookup.id,
+        queue: job_class.queue || process.queue
+      }
+      
+      process.env.build_job job_class, defaults.merge(options)
+    end
+    
+    def data
+      @data
+    end
+
+  end
+end

data/lib/asynchronic/version.rb

@@ -1,3 +1,3 @@
 module Asynchronic
-  VERSION = "0.0.1"
+  VERSION = '0.1.0'
 end

data/lib/asynchronic/worker.rb

@@ -1,27 +1,36 @@
-module Asynchronic
-  class Worker
+class Asynchronic::Worker
 
-    attr_reader :queue
+  attr_reader :queue
+  attr_reader :queue_name
+  attr_reader :env
+  attr_reader :listener
 
-    def initialize(queue=nil)
-      @queue = queue || Asynchronic.default_queue
-    end
+  def initialize(queue_name, env)
+    @queue_name = queue_name
+    @queue = env.queue_engine[queue_name]
+    @env = env
+    @listener = env.queue_engine.listener
+  end
 
-    def start
-      Signal.trap('INT') { stop }
+  def start
+    Asynchronic.logger.info('Asynchronic') { "Starting worker of #{queue_name} (#{Process.pid})" }
 
-      Ost[@queue].pop do |pid|
-        Process.find(pid).run
-      end
+    Signal.trap('INT') { stop }
+    
+    listener.listen(queue) do |pid|
+      env.load_process(pid).execute
     end
+  end
 
-    def stop
-      Ost[@queue].stop
-    end
+  def stop
+    Asynchronic.logger.info('Asynchronic') { "Stopping worker of #{@queue_name} (#{Process.pid})" }
+    listener.stop
+  end
 
-    def self.start(queue=nil)
-      new(queue).tap(&:start)
-    end
-  
+  def self.start(queue_name, &block)
+    worker = new queue_name, Asynchronic.environment
+    Thread.new { block.call(worker) } if block_given?
+    worker.start
   end
+
 end

data/lib/asynchronic.rb

@@ -1,48 +1,33 @@
-require 'ost'
+require 'forwardable'
 require 'securerandom'
-require 'base64'
+require 'redis'
+require 'ost'
+require 'class_config'
 require 'logger'
-require 'fileutils'
 
-Dir.glob(File.expand_path('asynchronic/*.rb', File.dirname(__FILE__))).sort.each { |f| require f }
+Dir.glob(File.expand_path('asynchronic/**/*.rb', File.dirname(__FILE__))).sort.each { |f| require f }
 
 module Asynchronic
 
-  def self.default_queue
-    @default_queue ||= :asynchronic
-  end
-
-  def self.default_queue=(name)
-    @default_queue = name
-  end
-
-  def self.logger
-    @logger ||= Logger.new($stdout)
-  end
+  UUID_REGEXP = '[a-z\d]{8}-[a-z\d]{4}-[a-z\d]{4}-[a-z\d]{4}-[a-z\d]{12}'
 
-  def self.logger=(logger)
-    @logger = logger
-  end
+  extend ClassConfig
 
-  def self.connect_redis(options)
-    Ost.connect options
-    @redis = Redis.new options
-  end
+  attr_config :default_queue, :asynchronic
+  attr_config :queue_engine, QueueEngine::InMemory.new
+  attr_config :data_store, DataStore::InMemory.new
+  attr_config :logger, Logger.new($stdout)
 
-  def self.redis
-    @redis ||= Redis.current
+  def self.environment
+    Environment.new queue_engine, data_store
   end
 
-  def self.archiving_path
-    @archiving_path ||= File.join(Dir.home, '.asynchronic', 'data')
-  end
-  
-  def self.archiving_path=(path)
-    @archiving_path = path
+  def self.[](pid)
+    environment.load_process pid
   end
 
-  def self.archiving_file(name)
-    File.join archiving_path, "#{name}.bin"
+  def self.processes
+    environment.processes
   end
 
 end

data/spec/coverage_helper.rb

@@ -1,8 +1,2 @@
 require 'simplecov'
-require 'coveralls'
-
-SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
-  SimpleCov::Formatter::HTMLFormatter,
-  Coveralls::SimpleCov::Formatter
-]
 SimpleCov.start

data/spec/data_store/data_store_examples.rb

@@ -0,0 +1,62 @@
+module DataStoreExamples
+  
+  it 'Get/Set value' do
+    data_store.set 'test_key', 123
+    data_store.get('test_key').must_equal 123
+  end
+
+  it 'Key not found' do
+    data_store.get('test_key').must_be_nil
+  end
+
+  it 'Keys' do
+    data_store.keys.must_be_empty
+    data_store.set 'test_key', 123
+    data_store.keys.must_equal ['test_key']
+  end
+
+  it 'Merge' do
+    data_store.set 'a:1', 0
+    data_store.merge 'a', '1' => 1, '2' => 2
+
+    data_store.get('a:1').must_equal 1
+    data_store.get('a:2').must_equal 2
+  end
+
+  it 'To hash' do
+    data_store.set 'a', 0
+    data_store.set 'a:1', 1
+    data_store.set 'a:2', 2
+    data_store.set 'b:3', 3
+
+    data_store.to_hash('a').must_equal '1' => 1, '2' => 2
+  end
+
+  it 'Nested keys' do
+    data_store.set 'a', 0
+    data_store.set 'a:1', 1
+    data_store.set 'a:2', 2
+    data_store.set 'b:3', 3
+
+    data_store.keys('a').must_equal_contents %w(a a:1 a:2)
+    data_store.keys('a:').must_equal_contents %w(a:1 a:2)
+  end
+
+  it 'Clear' do
+    data_store.set 'test_key', 123
+    data_store.clear
+    data_store.keys.must_be_empty
+  end
+
+  it 'Nested clear' do
+    data_store.set 'a', 0
+    data_store.set 'a:1', 1
+    data_store.set 'a:2', 2
+    data_store.set 'b:3', 3
+
+    data_store.clear 'a:'
+
+    data_store.keys.must_equal_contents %w(a b:3)
+  end
+  
+end

data/spec/data_store/in_memory_spec.rb

@@ -0,0 +1,10 @@
+require 'minitest_helper'
+require_relative './data_store_examples'
+
+describe Asynchronic::DataStore::InMemory do
+
+  let(:data_store) { Asynchronic::DataStore::InMemory.new }
+
+  include DataStoreExamples
+
+end

data/spec/data_store/key_spec.rb

@@ -0,0 +1,36 @@
+require 'minitest_helper'
+
+describe Asynchronic::DataStore::Key do
+
+  it 'Return the namespace' do
+    key = Asynchronic::DataStore::Key.new('foo')
+    key.must_equal 'foo'
+  end
+
+  it 'Prepend the namespace' do
+    key = Asynchronic::DataStore::Key.new('foo')
+    key['bar'].must_equal 'foo:bar'
+  end
+
+  it 'Work in more than one level' do
+    key_1 = Asynchronic::DataStore::Key.new('foo')
+    key_2 = Asynchronic::DataStore::Key.new(key_1['bar'])
+    key_2['baz'].must_equal 'foo:bar:baz'
+  end
+
+  it 'Be chainable' do
+    key = Asynchronic::DataStore::Key.new('foo')
+    key['bar']['baz'].must_equal 'foo:bar:baz'
+  end
+
+  it 'Accept symbols' do
+    key = Asynchronic::DataStore::Key.new(:foo)
+    key[:bar].must_equal 'foo:bar'
+  end
+
+  it 'Accept numbers' do
+    key = Asynchronic::DataStore::Key.new('foo')
+    key[3].must_equal 'foo:3'
+  end
+  
+end

data/spec/data_store/lookup_spec.rb

@@ -0,0 +1,92 @@
+require 'minitest_helper'
+
+describe Asynchronic::DataStore::Lookup do
+
+  describe 'One level' do
+
+    let(:job) { Asynchronic::Job.new }
+    let(:lookup) { Asynchronic::DataStore::Lookup.new job }
+
+    it 'Id' do
+      lookup.id.must_equal "job:#{job.id}"
+    end
+
+    it 'Status' do
+      lookup.status.must_equal "job:#{job.id}:status"
+    end
+
+    it 'Data' do
+      lookup.data.must_equal "job:#{job.id}:data"
+    end
+
+    it 'Jobs' do
+      lookup.jobs.must_equal "job:#{job.id}:jobs"
+    end
+
+    it 'Error' do
+      lookup.error.must_equal "job:#{job.id}:error"
+    end
+
+    it 'Created At' do
+      lookup.created_at.must_equal "job:#{job.id}:created_at"
+    end
+
+    it 'Queued At' do
+      lookup.queued_at.must_equal "job:#{job.id}:queued_at"
+    end
+
+    it 'Started At' do
+      lookup.started_at.must_equal "job:#{job.id}:started_at"
+    end
+
+    it 'Finalized At' do
+      lookup.finalized_at.must_equal "job:#{job.id}:finalized_at"
+    end
+
+  end
+
+  describe 'Two levels' do
+
+    let(:parent) { "job:#{SecureRandom.uuid}" }
+    let(:job) { Asynchronic::Job.new parent: parent }
+    let(:lookup) { Asynchronic::DataStore::Lookup.new job }
+
+    it 'Id' do
+      lookup.id.must_equal "#{parent}:jobs:#{job.id}"
+    end
+
+    it 'Status' do
+      lookup.status.must_equal "#{parent}:jobs:#{job.id}:status"
+    end
+
+    it 'Data' do
+      lookup.data.must_equal "#{parent}:jobs:#{job.id}:data"
+    end
+
+    it 'Jobs' do
+      lookup.jobs.must_equal "#{parent}:jobs:#{job.id}:jobs"
+    end
+
+    it 'Error' do
+      lookup.error.must_equal "#{parent}:jobs:#{job.id}:error"
+    end
+
+    it 'Created At' do
+      lookup.created_at.must_equal "#{parent}:jobs:#{job.id}:created_at"
+    end
+
+    it 'Queued At' do
+      lookup.queued_at.must_equal "#{parent}:jobs:#{job.id}:queued_at"
+    end
+
+    it 'Started At' do
+      lookup.started_at.must_equal "#{parent}:jobs:#{job.id}:started_at"
+    end
+
+    it 'Finalized At' do
+      lookup.finalized_at.must_equal "#{parent}:jobs:#{job.id}:finalized_at"
+    end
+
+  end
+
+end

data/spec/data_store/redis_spec.rb

@@ -0,0 +1,14 @@
+require 'minitest_helper'
+require_relative './data_store_examples'
+
+describe Asynchronic::DataStore::Redis do
+
+  let(:data_store) { Asynchronic::DataStore::Redis.new }
+
+  before do
+    data_store.clear
+  end
+
+  include DataStoreExamples
+
+end

data/spec/expectations.rb

@@ -0,0 +1,89 @@
+module MiniTest::Assertions
+
+  def assert_enqueued(expected_processes, queue)
+    messages = Array(expected_processes).map { |p| p.job.lookup.id }
+    queue.to_a.sort.must_equal messages.sort, "Jobs #{Array(expected_processes).map{ |p| p.job.name }}"
+  end
+
+  def assert_have(expected_hash, process)
+    process.data.keys.count.must_equal expected_hash.keys.count, "Missing keys\nExpected keys: #{expected_hash.keys}\n  Actual keys: #{process.data.keys}"
+    expected_hash.each do |k,v|
+      process[k].must_equal v, "Key #{k}"
+    end
+  end
+
+  def assert_be_initialized(process)
+    process.must_be :pending?
+    process.wont_be :finalized?
+    
+    process.processes.must_be_empty
+    process.data.must_be_empty
+    process.error.must_be_nil
+    
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_nil
+    process.started_at.must_be_nil
+    process.finalized_at.must_be_nil
+  end
+
+  def assert_be_pending(process)
+    process.must_be :pending?
+    process.wont_be :finalized?
+
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_nil
+    process.started_at.must_be_nil
+    process.finalized_at.must_be_nil
+  end
+
+  def assert_be_queued(process)
+    process.must_be :queued?
+    process.wont_be :finalized?
+
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_instance_of Time
+    process.started_at.must_be_nil
+    process.finalized_at.must_be_nil
+  end
+
+  def assert_be_waiting(process)
+    process.must_be :waiting?
+    process.wont_be :finalized?
+
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_instance_of Time
+    process.started_at.must_be_instance_of Time
+    process.finalized_at.must_be_nil
+  end
+
+  def assert_be_completed(process)
+    process.must_be :completed?
+    process.must_be :finalized?
+
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_instance_of Time
+    process.started_at.must_be_instance_of Time
+    process.finalized_at.must_be_instance_of Time
+  end
+
+  def assert_be_aborted(process)
+    process.must_be :aborted?
+    process.must_be :finalized?
+
+    process.created_at.must_be_instance_of Time
+    process.queued_at.must_be_instance_of Time
+    process.started_at.must_be_instance_of Time
+    process.finalized_at.must_be_instance_of Time
+  end
+
+end
+
+Asynchronic::QueueEngine::InMemory::Queue.infect_an_assertion :assert_enqueued, :must_enqueued
+Asynchronic::QueueEngine::Ost::Queue.infect_an_assertion :assert_enqueued, :must_enqueued
+Asynchronic::Process.infect_an_assertion :assert_have, :must_have
+Asynchronic::Process.infect_an_assertion :assert_be_initialized, :must_be_initialized, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_pending, :must_be_pending, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_queued, :must_be_queued, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_waiting, :must_be_waiting, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_completed, :must_be_completed, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_aborted, :must_be_aborted, :unary

data/spec/facade_spec.rb

@@ -0,0 +1,61 @@
+require 'minitest_helper'
+
+describe Asynchronic, 'Facade' do
+
+  before do
+    Asynchronic.environment.data_store.clear
+    Asynchronic.environment.queue_engine.clear
+  end
+  
+  it 'Default queue' do
+    Asynchronic.default_queue.must_equal :asynchronic
+  end
+
+  it 'Default queue_engine' do
+    Asynchronic.queue_engine.must_be_instance_of Asynchronic::QueueEngine::InMemory
+  end
+
+  it 'Default data store' do
+    Asynchronic.data_store.must_be_instance_of Asynchronic::DataStore::InMemory
+  end
+
+  it 'Default logger' do
+    Asynchronic.logger.must_be_instance_of Logger
+  end
+
+  it 'Environment' do
+    Asynchronic.environment.tap do |env|
+      env.queue_engine.must_equal Asynchronic.queue_engine
+      env.data_store.must_equal Asynchronic.data_store
+    end
+  end
+
+  it 'Load process' do
+    process = Asynchronic.environment.build_process BasicJob
+    Asynchronic[process.pid].tap do |p|
+      p.pid.must_equal process.pid
+      p.job.must_equal process.job
+    end
+  end
+
+  it 'List processes' do
+    pids = 3.times.map do 
+      process = Asynchronic.environment.build_process SequentialJob
+      process.pid
+    end
+
+    Asynchronic.processes.count.must_equal 3
+    Asynchronic.processes.map(&:pid).each { |pid| pids.must_include pid }
+  end
+
+  it 'Enqueue' do
+    pid = BasicJob.enqueue input: 100
+    
+    Asynchronic.environment.tap do |env|
+      env.default_queue.to_a.must_equal [pid]
+      env[pid].must_be_instance_of BasicJob
+      env.load_process(pid)[:input].must_equal 100
+    end
+  end
+
+end