asynchronic 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7a82bc550c9f5c81ebf6ec87c33051ab31d645d
-  data.tar.gz: 0571213c9a3a35d9770f4f81032fb9e35c8c1dd4
+  metadata.gz: 15caa715564ca133dd45cad9dce01db2de3cf7a6
+  data.tar.gz: 006fe0ebb38821c8cd1842e4b8060e14779d71c5
 SHA512:
-  metadata.gz: 0b8a197203e07e67f36af15d8dd4736517f0aba1fff0696f462a76a9e61a881f1935dc4401f3d0e330c77d4b382ae762fa71ae3d327ee7b7599726ac67105a34
-  data.tar.gz: 4ecbd031498315ae41331ddc09c8a1a72e6eba232f194c695a3feb638c1bb1820e7d225cedaae8899e291a1207984ed8b2bb84d9706a82fc71cbda9569eea05d
+  metadata.gz: c41cf2f761f07c3b17a0c05099361bbe90f2f33506bb29a349074f079827bd1ee588e12319158a1c59e399bbd0f641c616579b401f4aac2a82175be59916e73e
+  data.tar.gz: a64a7b0e1a5422bee3fa8a465d166ef37342ef5294865c24208a211423aed838f9f031b0320755cfde35788c2e6d34691b1fc762173af33bd3ed430359ffa112

data/.travis.yml

@@ -2,6 +2,7 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0
+  - 2.1
   - jruby
 services:
   - redis-server

data/README.md

@@ -24,76 +24,6 @@ Or install it yourself as:
 
 ## Usage
 
-### Basic usage
-
-    class Job
-      extend Asynchronic::Pipeline
-      
-      step :step_name do
-        ...
-      end
-    end
-
-    Job.run
-
-    Asynchronic::Worker.start
-
-### Enque job in specific queue
-
-    class Job
-      extend Asynchronic::Pipeline
-      
-      queue :queue_name
-      
-      step :step_name do
-        ...
-      end
-    end
-
-    Job.run
-
-    Asynchronic::Worker.start :queue_name
-
-### Pipeline with shared context
-
-    class Job
-      extend Asynchronic::Pipeline
-
-      step :first do |ctx|
-        ctx[:c] = ctx[:a] + ctx[:b]
-        100
-      end
-
-      step :second do |ctx, input|
-        input * ctx[:c] # 300
-      end
-    end
-
-    Job.run a: 1, b: 2
-
-    Asynchronic::Worker.start
-
-### Specify queue for each step
-
-    class Job
-      extend Asynchronic::Pipeline
-      
-      step :first_queue, queue: :queue1 do
-        ...
-      end
-      
-      step :second_queue, queue: ->(ctx){ctx[:dynamic_queue]} do
-        ...
-      end
-    end
-
-    Job.run dynamic_queue: :queue2
-
-    [:queue1, :queue2].map do |queue|
-      Thread.new do
-        Asynchronic::Worker.start queue
-      end
-    end
 
 ## Contributing
 

data/Rakefile

@@ -7,4 +7,11 @@ Rake::TestTask.new(:spec) do |t|
   t.verbose = false
 end
 
+task :console do
+  require 'pry'
+  require 'asynchronic'
+  ARGV.clear
+  Pry.start
+end
+
 task default: :spec

data/asynchronic.gemspec

@@ -18,11 +18,15 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'redis', '~> 3.0'
   spec.add_dependency 'ost', '~> 0.1'
-  
+  spec.add_dependency 'class_config', '~> 0.0'
+
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'minitest', '~> 4.7'
+  spec.add_development_dependency 'minitest-great_expectations', '~> 0.0'
   spec.add_development_dependency 'turn', '~> 0.9'
   spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'pry'
 end

data/lib/asynchronic/data_store/in_memory.rb

@@ -0,0 +1,47 @@
+module Asynchronic
+  module DataStore
+    class InMemory
+
+      def initialize
+        @hash = {}
+        @mutex = Mutex.new
+      end
+
+      def get(key)
+        @hash[key.to_s]
+      end
+
+      def set(key, value)
+        @mutex.synchronize { @hash[key.to_s] = value }
+      end
+
+      def merge(key, hash)
+        scoped_key = Key.new key
+        hash.each do |k,v|
+          set scoped_key[k].to_s, v
+        end
+      end
+
+      def to_hash(key)
+        children_key = "#{key}:"
+        keys(children_key).inject({}) do |hash, k|
+          hash[k[children_key.size..-1]] = get k
+          hash
+        end
+      end
+
+      def keys(key=nil)
+        key ? keys.select { |k| k.start_with? key.to_s } : @hash.keys
+      end
+
+      def clear(key=nil)
+        if key
+          @hash.delete_if { |k,v| k.start_with? key.to_s }
+        else
+          @hash.clear
+        end
+      end
+
+    end
+  end
+end

data/lib/asynchronic/data_store/key.rb

@@ -0,0 +1,15 @@
+module Asynchronic
+  module DataStore
+    class Key < String
+    
+      def initialize(key)
+        super key.to_s
+      end
+
+      def [](key)
+        self.class.new "#{self}:#{key}"
+      end
+
+    end
+  end
+end

data/lib/asynchronic/data_store/lookup.rb

@@ -0,0 +1,27 @@
+module Asynchronic
+  module DataStore
+    class Lookup
+
+      KEYS = [:status, :data, :jobs, :error, :created_at, :queued_at, :started_at, :finalized_at]
+
+      def initialize(job)
+        @job = job
+      end
+
+      def id
+        if @job.parent 
+          DataStore::Key.new(@job.parent)[:jobs][@job.id]
+        else
+          DataStore::Key.new(:job)[@job.id]
+        end
+      end
+
+      KEYS.each do |key|
+        define_method key do
+          id[key]
+        end
+      end
+
+    end
+  end
+end

data/lib/asynchronic/data_store/redis.rb

@@ -0,0 +1,52 @@
+module Asynchronic
+  module DataStore
+    class Redis
+
+      attr_reader :connection
+
+      def initialize(*args)
+        @connection = ::Redis.new *args
+      end
+
+      def get(key)
+        value = connection.get root[key]
+        value ? Marshal.load(value) : nil
+      end
+
+      def set(key, value)
+        connection.set root[key], Marshal.dump(value)
+      end
+
+      def merge(key, hash)
+        scoped_key = Key.new key
+        hash.each do |k,v|
+          set scoped_key[k], v
+        end
+      end
+
+      def to_hash(key)
+        children_key = "#{key}:"
+        keys(children_key).inject({}) do |hash, k|
+          hash[k[children_key.size..-1]] = get k
+          hash
+        end
+      end
+
+      def keys(key=nil)
+        keys = key ? connection.keys("#{root[key]}*") : connection.keys
+        keys.map { |k| k[(root.size + 1)..-1] }
+      end
+
+      def clear(key=nil)
+        keys(key).each { |k| connection.del root[k] }
+      end
+
+      private
+
+      def root
+        Key.new :asynchronic
+      end
+      
+    end
+  end
+end

data/lib/asynchronic/environment.rb

@@ -0,0 +1,57 @@
+module Asynchronic
+  class Environment
+
+    attr_reader :queue_engine
+    attr_reader :data_store
+    
+    def initialize(queue_engine, data_store)
+      @queue_engine = queue_engine
+      @data_store = data_store
+    end
+
+    def [](key)
+      data_store.get key
+    end
+
+    def []=(key, value)
+      data_store.set key, value
+    end
+
+    def queue(name)
+      queue_engine[name]
+    end
+
+    def default_queue
+      queue(queue_engine.default_queue)
+    end
+
+    def enqueue(msg, queue=nil)
+      queue(queue || queue_engine.default_queue).push msg
+    end
+
+    def build_job(job_class, options={})
+      Asynchronic.logger.debug('Asynchronic') { "Building job #{job_class} - #{options}" }
+      job_class.new(options).tap do |job|
+        self[job.lookup.id] = job
+        self[job.lookup.created_at] = Time.now
+      end
+    end
+
+    def build_process(job_class, options={})
+      Process.new build_job(job_class, options), self
+    end
+
+    def load_process(pid)
+      Process.new self[pid], self
+    end
+
+    def processes
+      data_store.keys.
+        select { |k| k.match Regexp.new("job:#{Asynchronic::UUID_REGEXP}:created_at$") }.
+        sort_by {|k| data_store.get k }.
+        reverse.
+        map { |k| load_process k.gsub(':created_at', '') }
+    end
+
+  end
+end

data/lib/asynchronic/error.rb

@@ -0,0 +1,13 @@
+module Asynchronic
+  class Error
+
+    attr_reader :message
+    attr_reader :backtrace
+    
+    def initialize(source)
+      @message = source.respond_to?(:message) ? source.message : source.to_s
+      @backtrace = source.respond_to?(:backtrace) ? source.backtrace : []
+    end
+
+  end
+end

data/lib/asynchronic/hash.rb

@@ -0,0 +1,31 @@
+class Hash
+  def with_indiferent_access
+    HashWithIndiferentAccess.new self
+  end
+end
+
+class HashWithIndiferentAccess < Hash
+
+  def initialize(hash=nil)
+    merge! hash if hash
+  end
+
+  def [](key)
+    if key?(key) || !transformable_key?(key)
+      super
+    else
+      super transform_key(key)
+    end
+  end
+
+  private
+
+  def transformable_key?(key)
+    key.is_a?(String) || key.is_a?(Symbol)
+  end
+
+  def transform_key(key)
+    key.is_a?(String) ? key.to_sym : key.to_s
+  end
+
+end

data/lib/asynchronic/job.rb

@@ -0,0 +1,46 @@
+module Asynchronic
+  class Job
+
+    attr_reader :id
+    attr_reader :name
+    attr_reader :queue
+    attr_reader :parent
+    attr_reader :dependencies
+    attr_reader :local
+
+    def initialize(options={})
+      @id = SecureRandom.uuid
+      @name = options.key?(:alias) ? options[:alias].to_s : self.class.to_s
+      @queue = options[:queue] || self.class.queue
+      @parent = options[:parent]
+      @dependencies = Array(options[:dependencies] || options[:dependency]).map(&:to_s)
+      @local = options[:local] || {}
+
+      raise 'Cant have dependencies without parent job' if dependencies.any? && parent.nil?
+    end
+
+    def lookup
+      DataStore::Lookup.new self
+    end
+
+    def self.queue(queue=nil)
+      queue ? @queue = queue : @queue
+    end
+
+    def self.implementation
+      @implementation
+    end
+
+    def self.enqueue(data={})
+      process = Asynchronic.environment.build_process self
+      process.enqueue data
+    end
+
+    private
+
+    def self.define(&block)
+      @implementation = block
+    end
+
+  end
+end

data/lib/asynchronic/process.rb

@@ -1,79 +1,148 @@
 module Asynchronic
   class Process
 
-    include Persistent
+    STATUSES = [:pending, :queued, :running, :waiting, :completed, :aborted]
 
-    Child = Struct.new :status, :output
+    TIME_TRACKING_MAP = {
+      queued: :queued_at,
+      running: :started_at,
+      completed: :finalized_at,
+      aborted: :finalized_at
+    }
 
-    attr_reader :pipeline
-    attr_reader :context
-    attr_reader :children
-  
-    def initialize(pipeline, context={})
-      @pipeline = pipeline
-      @context = context
-      @children = pipeline.steps.map { Child.new :pending }
+    extend Forwardable
+
+    def_delegators :job, :id, :name, :queue
+    def_delegators :data, :[]
+
+    attr_reader :job
+    attr_reader :env
+
+    def initialize(job, env)
+      @job = job
+      @env = env
     end
 
-    def enqueue(queue=nil)
-      q = queue || pipeline.queue || Asynchronic.default_queue
-      Ost[q.is_a?(Proc) ? q.call(context) : q].push id
+    def pid
+      lookup.id
     end
 
-    def run
-      current_child.tap do |i|
-        log "Running: #{id} (child: #{i})" do
-          children[i].status = :running
-          save
-
-          current_input = previous_child?(i) ? children[previous_child(i)].output : nil
-          children[i].output = pipeline.steps[i].block.call(context, current_input)
-          children[i].status = :finalized
-          save
-
-          if next_child?(i)
-            enqueue(pipeline.steps[next_child(i)].options[:queue])
+    def data
+      parent ? parent.data : env.data_store.to_hash(lookup.data).with_indiferent_access
+    end
+
+    def merge(data)
+      parent ? parent.merge(data) : env.data_store.merge(lookup.data, data)
+    end
+
+    def enqueue(data={})
+      merge data
+      env.enqueue lookup.id, queue
+      update_status :queued
+
+      lookup.id
+    end
+
+    def execute
+      run
+      wakeup
+    end
+
+    def wakeup
+      if waiting?
+        if processes.any?(&:aborted?)
+          abort Error.new "Error caused by #{processes.select(&:aborted?).map{|p| p.job.name}.join(', ')}"
+        else
+          if processes.all?(&:completed?)
+            update_status :completed 
           else
-            archive
+            processes.select(&:ready?).each { |p| p.enqueue }
           end
         end
       end
+
+      parent.wakeup if parent && finalized?
     end
 
-    def self.enqueue(pipeline, context={})
-      process = Process.create pipeline, context
-      process.enqueue(pipeline.steps.first.options[:queue])
-      process.id
+    def error
+      env[lookup.error]
     end
 
-    private
+    def status
+      env[lookup.status] || :pending
+    end
+
+    STATUSES.each do |status|
+      define_method "#{status}?" do
+        self.status == status
+      end
+    end
 
-    def current_child
-      children.index { |c| c.status == :pending }
+    def ready?
+      pending? && dependencies.all?(&:completed?)
     end
 
-    def previous_child(index=current_step)
-      index - 1
+    def finalized?
+      completed? || aborted?
     end
 
-    def previous_child?(index=current_step)
-      previous_child(index) >= 0
+    def processes(name=nil)
+      processes = env.data_store.keys(lookup.jobs).
+        select { |k| k.match Regexp.new("^#{lookup.jobs[Asynchronic::UUID_REGEXP]}$") }.
+        map { |k| env.load_process k }
+
+      name ? processes.detect { |p| p.name == name.to_s } : processes
+    end
+
+    def parent
+      @parent ||= Process.new env[job.parent], env if job.parent
+    end
+
+    def dependencies
+      @dependencies ||= parent.processes.select { |p| job.dependencies.include? p.name }
+    end
+
+    def created_at
+      env[lookup.created_at]
+    end
+
+    def queued_at
+      env[lookup.queued_at]
+    end
+
+    def started_at
+      env[lookup.started_at]
+    end
+
+    def finalized_at
+      env[lookup.finalized_at]
+    end
+
+    private
+
+    def run
+      update_status :running
+      Runtime.evaluate self
+      update_status :waiting
+    rescue Exception => ex
+      message = "Failed job #{job.name} (#{lookup.id})\n#{ex.class} #{ex.message}\n#{ex.backtrace.join("\n")}"
+      Asynchronic.logger.error('Asynchronic') { message }
+      abort ex
     end
 
-    def next_child(index=current_step)
-      index + 1
+    def update_status(status)
+      Asynchronic.logger.info('Asynchronic') { "#{status.to_s.capitalize} #{job.name} (#{lookup.id})" }
+      env[lookup.status] = status
+      env[lookup.send(TIME_TRACKING_MAP[status])] = Time.now if TIME_TRACKING_MAP.key? status
     end
 
-    def next_child?(index=current_step)
-      next_child(index) < children.count
+    def abort(exception)
+      env[lookup.error] = Error.new(exception)
+      update_status :aborted
     end
 
-    def log(message)
-      start = Time.now
-      Asynchronic.logger.info('Asynchronic') { "#{message} - Start" }
-      result = yield
-      Asynchronic.logger.info('Asynchronic') { "#{message} - End (Time: #{Time.now - start})" }
-      result
+    def lookup
+      @lookup ||= job.lookup
     end
 
   end

data/lib/asynchronic/queue_engine/in_memory.rb

@@ -0,0 +1,72 @@
+module Asynchronic
+  module QueueEngine
+    class InMemory
+
+      attr_reader :default_queue
+
+      def initialize(options={})
+        @default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
+        @queues ||= Hash.new { |h,k| h[k] = Queue.new }
+      end
+
+      def [](name)
+        @queues[name]
+      end
+
+      def queues
+        @queues.keys.map(&:to_sym)
+      end
+
+      def clear
+        @queues.clear
+      end
+
+      def listener
+        Listener.new
+      end
+
+
+      class Queue
+
+        extend Forwardable
+
+        def_delegators :@queue, :size, :empty?, :to_a
+
+        def initialize
+          @queue = []
+          @mutex = Mutex.new
+        end
+
+        def pop
+          @mutex.synchronize { @queue.shift }
+        end
+
+        def push(message)
+          @mutex.synchronize { @queue.push message }
+        end
+
+      end
+
+
+      class Listener
+
+        def listen(queue, &block)
+          @stopping = false
+
+          loop do
+            break if @stopping
+            item = queue.pop
+            next unless item
+            block.call item
+          end
+        end
+
+        def stop
+          @stopping = true
+        end
+
+      end
+
+    end
+  end
+end