burstflow 0.1.1 → 0.2.0

data/lib/{burst → burstflow/job}/model.rb

@@ -1,12 +1,10 @@
-module Burst::Model
+module Burstflow::Job::Model
 
   extend ActiveSupport::Concern
 
   included do |klass|
-    klass.include ActiveModel::Model
     klass.include ActiveModel::Dirty
     klass.include ActiveModel::Serialization
-    klass.extend ActiveModel::Callbacks
 
     attr_accessor :model
 

data/lib/burstflow/job/state.rb

@@ -0,0 +1,125 @@
+module Burstflow::Job::State
+  extend  ActiveSupport::Concern
+
+  included do
+
+    # mark job as enqueued when it is scheduled to queue
+    def enqueue!
+      raise Burstflow::Job::InternalError.new(self, "Can't enqueue: already enqueued") if enqueued?
+      self.enqueued_at = current_timestamp
+      self.started_at = nil
+      self.finished_at = nil
+      self.failed_at = nil
+      self.suspended_at = nil
+      self.resumed_at = nil
+    end
+
+    # mark job as started when it is start performing
+    def start!
+      raise Burstflow::Job::InternalError.new(self, "Can't start: already started") if started?
+      raise Burstflow::Job::InternalError.new(self, "Can't start: not enqueued") if !enqueued?
+      self.started_at = current_timestamp
+    end
+
+    # mark job as finished when it is finish performing
+    def finish!
+      raise Burstflow::Job::InternalError.new(self, "Can't finish: already finished") if finished?
+      raise Burstflow::Job::InternalError.new(self, "Can't finish: not started") if !started?
+      self.finished_at = current_timestamp
+    end
+
+    # mark job as failed when it is failed
+    def fail! msg_or_exception
+      #raise Burstflow::Job::InternalError.new(self, "Can't fail: already failed") if failed?
+      #raise Burstflow::Job::InternalError.new(self, Can't fail: already finished") if finished?
+      raise Burstflow::Job::InternalError.new(self, "Can't fail: not started") if !started?
+      self.finished_at = self.failed_at = current_timestamp
+
+      context = {}
+      if msg_or_exception.is_a?(::Exception)
+        context[:message]   = msg_or_exception.message
+        context[:klass]     = msg_or_exception.class.to_s
+        context[:backtrace] = msg_or_exception.backtrace.first(10)
+        context[:cause]     = msg_or_exception.cause.try(:inspect)
+      else
+        context[:message]   = msg_or_exception
+      end
+
+      self.failure = context
+    end
+
+    # mark job as suspended
+    def suspend!
+      raise Burstflow::Job::InternalError.new(self, "Can't suspend: already suspended") if suspended?
+      raise Burstflow::Job::InternalError.new(self, "Can't suspend: not runnig") if !running?
+      self.suspended_at = current_timestamp
+    end
+
+    # mark job as resumed
+    def resume!
+      raise Burstflow::Job::InternalError.new(self, "Can't resume: already resumed") if resumed?
+      raise Burstflow::Job::InternalError.new(self, "Can't resume: not suspended") if !suspended?
+      self.resumed_at = current_timestamp
+    end
+
+    def enqueued?
+      !enqueued_at.nil?
+    end
+
+    def started?
+      !started_at.nil?
+    end
+
+    def finished?
+      !finished_at.nil?
+    end
+
+    def running?
+      started? && !finished? && !suspended?
+    end
+
+    def scheduled?
+      enqueued? && !finished? && !suspended?
+    end
+
+    def failed?
+      !failed_at.nil?
+    end
+
+    def suspended?
+      !suspended_at.nil? && !resumed?
+    end
+
+    def resumed?
+      !resumed_at.nil?
+    end
+
+    def succeeded?
+      finished? && !failed?
+    end
+
+    def ready_to_start?
+      !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+    end
+
+    def initial?
+      incoming.empty?
+    end
+
+    def parents_succeeded?
+      incoming.all? do |id|
+        workflow.job(id).succeeded?
+      end
+    end
+
+    def current_timestamp
+      Time.now.to_i
+    end
+
+  end
+
+  class_methods do
+
+  end
+
+end

data/lib/burstflow/manager.rb

@@ -0,0 +1,123 @@
+module Burstflow
+class Manager
+
+  attr_accessor :workflow
+
+  def initialize(workflow)
+    @workflow = workflow
+  end
+
+  #workflow management
+
+  def start_workflow!
+    workflow.with_lock do
+      workflow.runnig!
+
+      workflow.initial_jobs.each do |job|
+        enqueue_job!(job)
+      end
+    end
+  end
+
+  def resume_workflow! job_id, data
+    workflow.with_lock do
+      workflow.resumed!
+
+      job = workflow.job(job_id)
+      resume_job!(job, data)
+    end
+  end
+
+
+  #Mark job enqueued and enqueue it
+  def enqueue_job!(job)
+    job.run_callbacks :enqueue do
+      job.enqueue!
+      job.save! do
+        Burstflow::Worker.perform_later(workflow.id, job.id)
+      end
+    end
+  end
+
+  #Enqueue job for resuming
+  def resume_job!(job, data)
+    job.save! do
+      Burstflow::Worker.perform_later(workflow.id, job.id, data)
+    end
+  end
+
+
+  #Mark job suspended and forget it until resume
+  def suspend_job!(job)
+    job.run_callbacks :suspend do
+      job.suspend!
+      job.save! do
+        analyze_workflow_state(job)
+      end
+    end
+  end
+
+
+  #Mark job finished and make further actions 
+  def finish_job!(job)
+    job.finish!
+    job.save! do
+      analyze_workflow_state(job)
+    end
+  end
+
+  #Mark job failed and make further actions 
+  def fail_job!(job, exception)
+    job.run_callbacks :failure do
+      job.fail! exception
+      workflow.add_error(job)
+      workflow.save!
+
+      job.save! do
+        analyze_workflow_state(job)
+      end
+    end    
+  end
+
+  #Mark job finished or suspended depends on result or output
+  def job_performed!(job, result)
+    if result == Burstflow::Job::SUSPEND || job.output == Burstflow::Job::SUSPEND
+      suspend_job!(job)
+    else
+      finish_job!(job)
+    end
+  rescue => e
+    raise Burstflow::Workflow::InternalError.new(workflow, e.message)
+  end
+
+private
+
+  #analyze job completition, current workflow state and perform futher actions
+  def analyze_workflow_state job
+    unless ActiveRecord::Base.connection.open_transactions > 0
+      raise Burstflow::Workflow::InternalError.new(workflow, "analyze_workflow_state must be called in transaction with lock!")
+    end
+
+    if job.succeeded? && job.outgoing.any? && !workflow.has_errors?
+      return enqueue_outgoing_jobs(job)
+    else
+      if workflow.has_scheduled_jobs?
+        #do nothing
+        #scheduled jobs will perform finish action
+      else
+        workflow.complete!
+      end
+    end
+  end
+
+  #enqueue outgoing jobs if all requirements are met
+  def enqueue_outgoing_jobs(job)
+    job.outgoing.each do |job_id|
+      out = workflow.job(job_id)
+
+      enqueue_job!(out) if out.ready_to_start?
+    end
+  end
+
+end
+end

data/lib/burstflow/railtie.rb

@@ -0,0 +1,6 @@
+
+module Burstflow
+  class Railtie < Rails::Engine #:nodoc:
+    engine_name 'burstflow'
+  end
+end

data/lib/burstflow/version.rb

@@ -0,0 +1,3 @@
+module Burstflow
+  VERSION = '0.2.0'
+end

data/lib/burstflow/worker.rb

@@ -0,0 +1,59 @@
+require 'active_job'
+
+class Burstflow::Worker < ::ActiveJob::Base
+
+  attr_reader :workflow, :job
+
+  rescue_from(Exception) do |exception|
+    @manager.fail_job!(job, exception)
+  end
+
+  rescue_from(Burstflow::Job::InternalError) do |exception|
+    @manager.fail_job!(exception.job, exception)
+  end
+
+  rescue_from(Burstflow::Workflow::InternalError) do |exception|
+    exception.workflow.add_error(exception)
+    exception.workflow.save!
+  end
+
+  before_perform do
+    workflow_id, job_id, resume_data = arguments
+
+    @workflow = Burstflow::Workflow.find(workflow_id)
+    @job = @workflow.job(job_id)
+    @manager = @workflow.manager
+
+    set_incoming_payloads(job)
+  end
+
+  def perform(workflow_id, job_id, resume_data = nil)
+    result = if resume_data.nil?
+      job.start!
+      job.save!
+  
+      job.perform_now
+    else
+      job.resume!
+      job.save!
+      
+      job.resume_now(resume_data)
+    end
+
+    @manager.job_performed!(job, result)
+  end
+
+private
+
+  def set_incoming_payloads job
+    job.payloads = job.incoming.map do |job_id|
+      incoming = workflow.job(job_id)
+      {
+        id: incoming.id,
+        class: incoming.klass.to_s,
+        value: incoming.output
+      }
+    end
+  end
+
+end

data/lib/burstflow/workflow.rb

@@ -0,0 +1,207 @@
+require 'active_record'
+
+require 'burstflow/manager'
+
+module Burstflow
+class Workflow < ActiveRecord::Base
+  require 'burstflow/workflow/exception'
+  require 'burstflow/workflow/builder'
+  require 'burstflow/workflow/configuration'
+  require 'burstflow/workflow/callbacks'
+  
+  self.table_name_prefix = 'burstflow_'
+
+  INITIAL   = 'initial'.freeze
+  RUNNING   = 'running'.freeze
+  FINISHED  = 'finished'.freeze
+  FAILED    = 'failed'.freeze
+  SUSPENDED = 'suspended'.freeze
+
+  STATUSES = [INITIAL, RUNNING, FINISHED, FAILED, SUSPENDED].freeze
+
+  include Burstflow::Workflow::Configuration
+  include Burstflow::Workflow::Callbacks
+
+  attr_accessor :manager, :cache
+  define_flow_attributes :jobs_config, :failures
+
+  after_initialize do
+    @cache = {}
+
+    self.status ||= INITIAL
+    self.id ||= SecureRandom.uuid
+    self.jobs_config ||= {}.with_indifferent_access
+    self.failures ||= []
+
+    @manager = Burstflow::Manager.new(self)
+  end
+
+  STATUSES.each do |name|
+    define_method "#{name}?".to_sym do
+      self.status == name
+    end
+  end
+
+  def attributes
+    {
+      id: self.id,
+      jobs_config: self.jobs_config,
+      type: self.class.to_s,
+      status: status,
+      failures: failures
+    }
+  end
+
+  def self.build(*args)
+    new.tap do |wf|
+      builder = Burstflow::Workflow::Builder.new(wf, *args, &configuration)
+      wf.flow = {'jobs_config' => builder.as_json}
+    end
+  end
+
+  def reload(*)
+    self.cache = {}
+    super
+  end
+
+  def start!
+    manager.start_workflow!
+    self
+  end
+
+  def resume!(job_id, data)
+    manager.resume_workflow!(job_id, data)
+    self
+  end
+
+  def jobs
+    Enumerator.new do |y|
+      jobs_config.keys.each do |id|
+        y << job(id)
+      end
+    end
+  end
+
+  def job_hash(id)
+    jobs_config[id].deep_dup
+  end
+
+  def job(id)
+    Burstflow::Job.from_hash(self, job_hash(id))
+  end
+
+  def set_job(job)
+    jobs_config[job.id] = job.as_json
+  end
+
+  def initial_jobs
+    cache[:initial_jobs] ||= jobs.select(&:initial?)
+  end
+
+  def add_error job_orexception
+    context = {
+      created_at: Time.now.to_i
+    }
+    if job_orexception.is_a?(::Exception)
+      context[:message]     = job_orexception.message
+      context[:klass]       = job_orexception.class.to_s
+      context[:backtrace]   = job_orexception.backtrace.first(10)
+      context[:cause]       = job_orexception.cause
+    else
+      context[:job] = job_orexception.id
+    end
+
+    failures.push(context)
+  end
+
+  def has_errors?
+    failures.any?
+  end
+
+  def has_scheduled_jobs?
+    cache[:has_scheduled_jobs] ||= jobs.any? do |job|
+      job.scheduled? || (job.initial? && !job.enqueued?)
+    end
+  end
+
+  def has_suspended_jobs?
+    cache[:has_suspended_jobs] ||= jobs.any?(&:suspended?)
+  end
+
+  def complete!
+    if has_errors?
+      failed!
+    elsif has_suspended_jobs?
+      suspended!
+    else
+      finished!
+    end
+  end
+
+  def first_job
+    all_jobs.min_by{|n| n.started_at || Time.now.to_i }
+  end
+
+  def last_job
+    all_jobs.max_by{|n| n.finished_at || 0 } if finished?
+  end
+
+  def started_at
+    first_job&.started_at
+  end
+
+  def finished_at
+    last_job&.finished_at
+  end
+
+  def runnig!
+    raise InternalError.new(self, "Can't start: workflow already running") if (running? || suspended?)
+    raise InternalError.new(self, "Can't start: workflow already failed") if failed?
+    raise InternalError.new(self, "Can't start: workflow already finished") if finished?
+    self.status = RUNNING
+    save!
+  end
+
+  def failed!
+    run_callbacks :failure do
+      raise InternalError.new(self, "Can't fail: workflow already failed") if failed?
+      raise InternalError.new(self, "Can't fail: workflow already finished") if finished?
+      raise InternalError.new(self, "Can't fail: workflow in not runnig") if !(running? || suspended?)
+      self.status = FAILED
+      save!
+    end
+  end
+
+  def finished!
+    run_callbacks :finish do
+      raise InternalError.new(self, "Can't finish: workflow already finished") if finished?
+      raise InternalError.new(self, "Can't finish: workflow already failed") if failed?
+      raise InternalError.new(self, "Can't finish: workflow in not runnig") if !running?
+      self.status = FINISHED
+      save!
+    end 
+  end
+
+  def suspended!
+    run_callbacks :suspend do
+      raise InternalError.new(self, "Can't suspend: workflow already finished") if finished?
+      raise InternalError.new(self, "Can't suspend: workflow already failed") if failed?
+      raise InternalError.new(self, "Can't suspend: workflow in not runnig") if !running?
+      self.status = SUSPENDED
+      save!
+    end
+  end
+
+  def resumed!
+    run_callbacks :resume do
+      raise InternalError.new(self, "Can't resume: workflow already running") if running?
+      raise InternalError.new(self, "Can't resume: workflow already finished") if finished?
+      raise InternalError.new(self, "Can't resume: workflow already failed") if failed?
+      raise InternalError.new(self, "Can't resume: workflow in not suspended") if !suspended?
+      self.status = RUNNING
+      save!
+    end
+  end
+
+end
+end