burstflow 0.1.0

data/lib/burst/job.rb

@@ -0,0 +1,187 @@
+class Burst::Job
+
+  include Burst::Model
+
+  define_stored_attributes :id, :workflow_id, :klass, :params, :incoming, :outgoing, :payloads, :output
+  define_stored_attributes :enqueued_at, :started_at, :finished_at, :failed_at, :suspended_at, :resumed_at
+
+  SUSPEND = 'suspend'.freeze
+
+  class Error < ::RuntimeError; end
+
+  def initialize(workflow, hash_store = {})
+    @workflow = workflow
+    assign_default_values(hash_store)
+  end
+
+  def assign_default_values(hash_store)
+    set_model(hash_store.deep_dup)
+
+    self.id ||= SecureRandom.uuid
+    self.workflow_id ||= @workflow.id
+    self.klass ||= self.class.to_s
+    self.incoming ||= []
+    self.outgoing ||= []
+  end
+
+  def reload
+    assign_default_values(@workflow.get_job_hash(self.id))
+  end
+
+  def save!
+    @workflow.with_lock do
+      @workflow.set_job(self)
+      @workflow.save!
+      yield if block_given?
+    end
+  end
+
+  def self.from_hash(workflow, hash_store)
+    hash_store[:klass].constantize.new(workflow, hash_store)
+  end
+
+  # execute this code by ActiveJob. You may return Burst::Job::SUSPEND to suspend job, or call suspend method
+  def perform; end
+
+  # execute this code when resumes after suspending
+  def resume(data)
+    set_output(data)
+  end
+
+  # store data to be available for next jobs
+  def set_output(data)
+    self.output = data
+  end
+
+  # mark execution as suspended
+  def suspend
+    set_output(SUSPEND)
+  end
+
+  def configure
+    @workflow.with_lock do
+      yield
+      @workflow.resolve_dependencies
+      @workflow.save!
+      @workflow.all_jobs.to_a.each(&:save!)
+      reload
+    end
+  end
+
+  def run(klass, opts = {})
+    opts[:after] = [*opts[:after], self.id].uniq
+    opts[:before] = [*opts[:before], *self.outgoing].uniq
+    @workflow.run(klass, opts)
+  end
+
+  def attributes
+    {
+      workflow_id: self.workflow_id,
+      id: self.id,
+      klass: self.klass,
+      params: params,
+      incoming: self.incoming,
+      outgoing: self.outgoing,
+      output: output,
+      started_at: started_at,
+      enqueued_at: enqueued_at,
+      finished_at: finished_at,
+      failed_at: failed_at,
+      suspended_at: suspended_at,
+      resumed_at: resumed_at
+    }
+  end
+
+  # mark job as enqueued when it is scheduled to queue
+  def enqueue!
+    raise Error.new('Already enqueued') if enqueued?
+    self.enqueued_at = current_timestamp
+    self.started_at = nil
+    self.finished_at = nil
+    self.failed_at = nil
+    self.suspended_at = nil
+    self.resumed_at = nil
+  end
+
+  # mark job as started when it is start performing
+  def start!
+    raise Error.new('Already started') if started?
+    self.started_at = current_timestamp
+  end
+
+  # mark job as finished when it is finish performing
+  def finish!
+    raise Error.new('Already finished') if finished?
+    self.finished_at = current_timestamp
+  end
+
+  # mark job as failed when it is failed
+  def fail!
+    raise Error.new('Already failed') if failed?
+    self.finished_at = self.failed_at = current_timestamp
+  end
+
+  # mark job as suspended
+  def suspend!
+    self.suspended_at = current_timestamp
+  end
+
+  # mark job as resumed
+  def resume!
+    raise Error.new('Not suspended ') unless suspended?
+    raise Error.new('Already resumed ') if resumed?
+    self.resumed_at = current_timestamp
+  end
+
+  def enqueued?
+    !enqueued_at.nil?
+  end
+
+  def started?
+    !started_at.nil?
+  end
+
+  def finished?
+    !finished_at.nil?
+  end
+
+  def running?
+    started? && !finished? && !suspended?
+  end
+
+  def failed?
+    !failed_at.nil?
+  end
+
+  def suspended?
+    !suspended_at.nil? && !resumed?
+  end
+
+  def resumed?
+    !resumed_at.nil?
+  end
+
+  def succeeded?
+    finished? && !failed?
+  end
+
+  def ready_to_start?
+    !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+  end
+
+  def initial?
+    incoming.empty?
+  end
+
+  def current_timestamp
+    Time.now.to_i
+  end
+
+  def parents_succeeded?
+    incoming.all? do |id|
+      @workflow.get_job(id).succeeded?
+    end
+  end
+
+
+end

data/lib/burst/manager.rb

@@ -0,0 +1,79 @@
+class Burst::Manager
+
+  attr_accessor :workflow
+
+  def initialize(workflow)
+    @workflow = workflow
+  end
+
+  def start
+    workflow.with_lock do
+      raise 'Already started' unless workflow.initial?
+
+      workflow.initial_jobs.each do |job|
+        enqueue_job!(job)
+      end
+    end
+  end
+
+  def enqueue_job!(job)
+    job.enqueue!
+    job.save! do
+      Burst::Worker.perform_later(workflow.id, job.id)
+    end
+  end
+
+  def resume!(job, data)
+    job.resume!
+    job.save! do
+      Burst::Worker.perform_later(workflow.id, job.id, data)
+    end
+  end
+
+  def start_job!(job)
+    job.start!
+    job.save!
+
+    job.perform
+  end
+
+  def resume_job!(job, data)
+    job.resume(data)
+  end
+
+  def suspend_job!(job)
+    job.suspend!
+    job.save!
+  end
+
+  def finish_job!(job)
+    job.finish!
+    job.save!
+
+    workflow.with_lock do
+      enqueue_outgoing_jobs(job)
+    end
+  end
+
+  def job_performed!(job, result)
+    if result == Burst::Job::SUSPEND || job.output == Burst::Job::SUSPEND
+      suspend_job!(job)
+    else
+      finish_job!(job)
+    end
+  end
+
+  def fail_job!(job)
+    job.fail!
+    job.save!
+  end
+
+  def enqueue_outgoing_jobs(job)
+    job.outgoing.each do |job_id|
+      out = workflow.get_job(job_id)
+
+      enqueue_job!(out) if out.ready_to_start?
+    end
+  end
+
+end

data/lib/burst/model.rb

@@ -0,0 +1,49 @@
+module Burst::Model
+
+  extend ActiveSupport::Concern
+
+  included do |klass|
+    klass.include ActiveModel::Model
+    klass.include ActiveModel::Dirty
+    klass.include ActiveModel::Serialization
+    klass.extend ActiveModel::Callbacks
+
+    attr_accessor :model
+
+    def set_model(model)
+      @model = model
+    end
+
+    def attributes=(hash)
+      hash.each do |key, value|
+        send("#{key}=", value)
+      end
+    end
+
+    def ==(other)
+      instance_variable_get('@model') == other.instance_variable_get('@model')
+    end
+
+    def as_json(*_args)
+      serializable_hash
+    end
+  end
+
+  class_methods do
+    def define_stored_attributes(*keys)
+      keys.each do |key|
+        define_attribute_methods key.to_sym
+
+        define_method key.to_sym do
+          return @model[key.to_s]
+        end
+
+        define_method "#{key}=".to_sym do |v|
+          send("#{key}_will_change!") if v != send(key)
+          return @model[key.to_s] = v
+        end
+      end
+    end
+  end
+
+end

data/lib/burst/worker.rb

@@ -0,0 +1,42 @@
+class Burst::Worker < ::ActiveJob::Base
+
+  def perform(workflow_id, job_id, resume_data = nil)
+    setup(workflow_id, job_id)
+
+    job.payloads = incoming_payloads
+
+    result = if resume_data.nil?
+               @manager.start_job!(job)
+             else
+               @manager.resume_job!(job, resume_data)
+    end
+
+    @manager.job_performed!(job, result)
+  rescue StandardError => e
+    @manager.fail_job!(job)
+    raise e
+  end
+
+
+  private
+
+    attr_reader :workflow, :job
+
+    def setup(workflow_id, job_id)
+      @workflow = Burst::Workflow.find(workflow_id)
+      @job = @workflow.get_job(job_id)
+      @manager = @workflow.manager
+    end
+
+    def incoming_payloads
+      job.incoming.map do |job_id|
+        incoming = workflow.get_job(job_id)
+        {
+          id: incoming.id,
+          class: incoming.klass.to_s,
+          payload: incoming.output
+        }
+      end
+    end
+
+end

data/lib/burst/workflow.rb

@@ -0,0 +1,148 @@
+class Burst::Workflow < ActiveRecord::Base
+
+  self.table_name_prefix = 'burst_'
+
+  INITIAL   = 'initial'.freeze
+  RUNNING   = 'running'.freeze
+  FINISHED  = 'finished'.freeze
+  FAILED    = 'failed'.freeze
+  SUSPENDED = 'suspended'.freeze
+
+  include Burst::WorkflowHelper
+  include Burst::Builder
+
+  attr_accessor :manager, :job_cache
+  define_flow_attributes :jobs, :klass
+
+  after_initialize do
+    initialize_builder
+
+    @job_cache = {}
+
+    self.id ||= SecureRandom.uuid
+    self.jobs ||= {}.with_indifferent_access
+    self.klass ||= self.class.to_s
+
+    @manager = Burst::Manager.new(self)
+  end
+
+  def attributes
+    {
+      id: self.id,
+      jobs: self.jobs,
+      klass: self.klass,
+      status: status
+    }
+  end
+
+  def self.build(*args)
+    wf = new
+    wf.configure(*args)
+    wf.resolve_dependencies
+    wf
+  end
+
+  def reload(options = nil)
+    self.job_cache = {}
+    super
+  end
+
+  def start!
+    save!
+    manager.start
+  end
+
+  def resume!(job_id, data)
+    manager.resume!(get_job(job_id), data)
+  end
+
+  def status
+    if failed?
+      FAILED
+    elsif suspended?
+      SUSPENDED
+    elsif running?
+      RUNNING
+    elsif finished?
+      FINISHED
+    else
+      INITIAL
+    end
+  end
+
+  def initial?
+    status == INITIAL
+  end
+
+  def finished?
+    all_jobs.all?(&:finished?)
+  end
+
+  def started?
+    !!started_at
+  end
+
+  def running?
+    started? && !finished?
+  end
+
+  def failed?
+    all_jobs.any?(&:failed?)
+  end
+
+  def suspended?
+    !failed? && all_jobs.any?(&:suspended?)
+  end
+
+  def all_jobs
+    Enumerator.new do |y|
+      jobs.keys.each do |id|
+        y << get_job(id)
+      end
+    end
+  end
+
+  def get_job(id)
+    if job = @job_cache[id]
+      job
+    else
+      job = Burst::Job.from_hash(self, jobs[id].deep_dup)
+      @job_cache[job.id] = job
+      job
+    end
+  end
+
+  def set_job(job)
+    jobs[job.id] = job.as_json
+  end
+
+  def initial_jobs
+    all_jobs.select(&:initial?)
+  end
+
+  def find_job(id_or_klass)
+    id = if jobs.key?(id_or_klass)
+           id_or_klass
+         else
+           find_id_by_klass(id_or_klass)
+    end
+
+    get_job(id)
+  end
+
+  def get_job_hash(id)
+    jobs[id]
+  end
+
+  private
+
+    def find_id_by_klass(klass)
+      finded = jobs.select do |_, job|
+        job[:klass].to_s == klass.to_s
+      end
+
+      raise 'Duplicat job detected' if finded.count > 1
+      finded.first.second[:id]
+    end
+
+end

data/lib/burst/workflow_helper.rb

@@ -0,0 +1,86 @@
+module Burst::WorkflowHelper
+
+  extend ActiveSupport::Concern
+
+  class JSONBWithIndifferentAccess
+
+    def self.dump(hash)
+      hash.as_json
+    end
+
+    def self.load(hash)
+      hash ||= {}
+      hash = JSON.parse(hash) if hash.is_a? String
+      hash.with_indifferent_access
+    end
+
+  end
+
+  included do |_klass|
+    serialize :flow, JSONBWithIndifferentAccess
+
+    def first_job
+      all_jobs.min_by{|n| n.started_at || Time.now.to_i }
+    end
+
+    def last_job
+      all_jobs.max_by{|n| n.finished_at || 0 } if finished?
+    end
+
+    def started_at
+      first_job&.started_at
+    end
+
+    def finished_at
+      last_job&.finished_at
+    end
+
+    def configure(*args)
+      instance_exec *args, &self.class.configuration
+    end
+  end
+
+  class_methods do
+    def define_flow_attributes(*keys)
+      keys.each do |key|
+        define_method key.to_sym do
+          return flow[key.to_s]
+        end
+
+        define_method "#{key}=".to_sym do |v|
+          return flow[key.to_s] = v
+        end
+      end
+    end
+
+    def configure(&block)
+      @configuration = block
+    end
+
+    def configuration
+      @configuration
+    end
+  end
+
+end
+
+
+# class Test
+#   def self.configure &block
+#     @c = block
+#   end
+
+#   def self.c
+#     @c
+#   end
+
+#   def go
+#     puts "111111"
+#   end
+
+#   def run
+#     instance_eval &Test.c
+#   end
+
+
+# end

data/lib/burst.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'bundler'
+require 'bundler/setup'
+Bundler.require(:default)
+
+require 'active_support/all'
+require 'active_support/dependencies'
+require 'active_record'
+require 'active_job'
+
+require 'pathname'
+require 'securerandom'
+
+require 'burst/configuration'
+require 'burst/model'
+require 'burst/builder'
+require 'burst/manager'
+require 'burst/job'
+require 'burst/workflow_helper'
+require 'burst/workflow'
+require 'burst/worker'
+
+module Burst
+
+  def self.root
+    Pathname.new(__FILE__).parent.parent
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+end

data/spec/burst_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe Burst do
+end