burstflow 0.1.0

data/spec/cases_spec.rb

@@ -0,0 +1,180 @@
+require 'spec_helper'
+
+describe Burst::Manager do
+  class JobHandler
+
+    attr_accessor :jobs
+
+    def initialize
+      @jobs = []
+    end
+
+    def add(json)
+      @jobs.push json
+    end
+
+    def find_job(klass)
+      @jobs.detect do |json|
+        json['klass'].to_s == klass.to_s
+      end
+    end
+
+  end
+
+  around :each do |ex|
+    begin
+      $job_handler = JobHandler.new
+      ex.run
+    ensure
+      $job_handler = nil
+    end
+  end
+
+  class TestCaseJob < Burst::Job
+
+    def perform
+      # puts "#{self.class.to_s}"
+      set_output(self.class.to_s)
+      $job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
+    end
+
+  end
+
+  class CaseJob1 < TestCaseJob; end
+  class CaseJob2 < TestCaseJob; end
+  class CaseJob3 < TestCaseJob; end
+  class CaseAsyncJob < TestCaseJob
+
+    def perform
+      suspend
+      $job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
+    end
+
+  end
+
+  class CaseW1 < Burst::Workflow
+
+    configure do |*_args|
+      id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
+      run CaseJob2, id: 'job2', after: id1, params: { p2: 2 }
+      run CaseJob3, after: [CaseJob1, 'job2']
+    end
+
+  end
+
+
+  it 'case1 no async job state' do
+    w = CaseW1.build
+
+    perform_enqueued_jobs do
+      w.start!
+    end
+
+    w.reload
+
+    expect(w.started?).to eq true
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq true
+    expect(w.running?).to eq false
+    expect(w.status).to eq Burst::Workflow::FINISHED
+
+    expect($job_handler.jobs.count).to eq 3
+    expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
+
+    expect($job_handler.find_job(CaseJob2)).to include(output: 'CaseJob2', params: { p2: 2 })
+    expect($job_handler.find_job(CaseJob2)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
+
+    expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
+    expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
+    expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseJob2', payload: 'CaseJob2')
+  end
+
+  class CaseW2 < Burst::Workflow
+
+    configure do |*_args|
+      id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
+      run CaseAsyncJob, id: 'job2', after: id1, params: { p2: 2 }
+      run CaseJob3, after: [CaseJob1, 'job2']
+    end
+
+  end
+
+  it 'case2 with async job state' do
+    w = CaseW2.build
+
+    perform_enqueued_jobs do
+      w.start!
+    end
+
+    w.reload
+
+    expect(w.started?).to eq true
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq false
+    expect(w.running?).to eq true
+    expect(w.suspended?).to eq true
+    expect(w.status).to eq Burst::Workflow::SUSPENDED
+
+    expect($job_handler.jobs.count).to eq 2
+    expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
+
+    expect($job_handler.find_job(CaseAsyncJob)).to include(output: Burst::Job::SUSPEND, params: { p2: 2 })
+    expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
+
+    w = CaseW2.find(w.id)
+
+    perform_enqueued_jobs do
+      w.resume!('job2', 'result')
+    end
+
+    w.reload
+
+    expect($job_handler.jobs.count).to eq 3
+    expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
+
+    # expect($job_handler.find_job(AsyncJob)).to include(output: 'result', params: {p2: 2})
+    expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
+
+    expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
+    expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
+    expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseAsyncJob', payload: 'result')
+  end
+
+  describe 'dynamic jobs with payloads' do
+    class DynJob1 < TestCaseJob
+
+      def perform
+        configure do
+          run DynJob2, params: { a: 1 }
+          run DynJob2, params: { a: 2 }
+          run DynJob2, params: { a: 3 }
+        end
+        super
+      end
+
+    end
+
+    class DynJob2 < TestCaseJob; end
+    class DynJob3 < TestCaseJob; end
+
+    class DynFlow1 < Burst::Workflow
+
+      configure do |*_args|
+        run DynJob1
+        run DynJob3, after: DynJob1
+      end
+
+    end
+
+    it 'create intemediate jobs' do
+      w = DynFlow1.build
+
+      perform_enqueued_jobs do
+        w.start!
+      end
+
+      expect($job_handler.jobs.count).to eq 5
+      expect($job_handler.jobs.last['payloads'].count).to eq 4
+    end
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+describe Burst::Job do
+  let(:w) { Burst::Workflow.new }
+
+  context 'initializing' do
+    class TestJob1 < Burst::Job
+    end
+
+    subject{ TestJob1.new(w, {}.with_indifferent_access) }
+
+    it 'empty' do
+      expect(subject.workflow_id).to eq w.id
+      expect(subject.klass).to eq TestJob1.to_s
+
+      expect(subject.incoming).to eq []
+      expect(subject.outgoing).to eq []
+
+      expect(subject.initial?).to eq true
+
+      expect(subject.enqueued?).to eq false
+      expect(subject.started?).to eq false
+      expect(subject.finished?).to eq false
+      expect(subject.failed?).to eq false
+      expect(subject.suspended?).to eq false
+      expect(subject.resumed?).to eq false
+
+      expect(subject.ready_to_start?).to eq true
+    end
+
+    it '#enqueue!' do
+      subject.enqueue!
+
+      expect(subject.enqueued?).to eq true
+      expect(subject.ready_to_start?).to eq false
+    end
+
+    it '#start!' do
+      subject.start!
+
+      expect(subject.started?).to eq true
+      expect(subject.ready_to_start?).to eq false
+    end
+
+    it '#finish!' do
+      subject.finish!
+
+      expect(subject.finished?).to eq true
+      expect(subject.succeeded?).to eq true
+      expect(subject.failed?).to eq false
+      expect(subject.ready_to_start?).to eq false
+    end
+
+    it '#fail!' do
+      subject.fail!
+
+      expect(subject.finished?).to eq true
+      expect(subject.succeeded?).to eq false
+      expect(subject.failed?).to eq true
+      expect(subject.ready_to_start?).to eq false
+    end
+
+    it '#suspend!' do
+      subject.suspend!
+
+      expect(subject.finished?).to eq false
+      expect(subject.suspended?).to eq true
+      expect(subject.ready_to_start?).to eq true
+    end
+
+    it '#resume!' do
+      subject.suspend!
+      subject.resume!
+
+      expect(subject.finished?).to eq false
+      expect(subject.resumed?).to eq true
+      expect(subject.ready_to_start?).to eq true
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'bundler'
+require 'bundler/setup'
+Bundler.require(:default)
+
+require 'burst'
+
+
+ActiveJob::Base.queue_adapter = :test
+ActiveJob::Base.logger = nil
+
+$root = File.join(File.dirname(__dir__), 'spec')
+Dir[File.join($root, 'support', '**', '*.rb')].each {|f| require f }
+
+
+
+
+RSpec::Matchers.define :have_jobs do |flow, jobs|
+  match do |_actual|
+    expected = jobs.map do |job|
+      hash_including(args: include(flow, job))
+    end
+    expect(ActiveJob::Base.queue_adapter.enqueued_jobs).to match_array(expected)
+  end
+
+  failure_message do |_actual|
+    "expected queue to have #{jobs}, but instead has: #{ActiveJob::Base.queue_adapter.enqueued_jobs.map{|j| j[:args][1] }}"
+  end
+end
+
+RSpec.configure do |config|
+  config.include ActiveJob::TestHelper
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.before(:each) do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+
+  config.after(:each) do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+end

data/spec/support/database_clean.rb

@@ -0,0 +1,16 @@
+require 'database_cleaner'
+
+RSpec.configure do |config|
+  config.before(:all) do
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end

data/spec/support/runner.rb

@@ -0,0 +1,12 @@
+require 'otr-activerecord'
+require 'yaml'
+require 'erb'
+
+RSpec.configure do |config|
+  # ActiveSupport::Dependencies.autoload_paths += [ 'lib' ].map {|f| File.join($root, '..', f)}
+
+  config.before(:suite) do
+    OTR::ActiveRecord.configure_from_file! 'config/database.yml'
+    ActiveRecord::Base.logger = nil
+  end
+end

data/spec/workflow_spec.rb

@@ -0,0 +1,185 @@
+require 'spec_helper'
+
+describe Burst::Workflow do
+  class TestJob < Burst::Job
+
+    def perform
+      # puts "#{self.class} perform"
+    end
+
+  end
+
+  class WfJob1 < TestJob
+  end
+
+  class WfJob2 < TestJob
+  end
+
+  class WfJob3 < TestJob
+  end
+
+
+  class W1 < Burst::Workflow
+
+    configure do |*_args|
+      id1 = run WfJob1, id: 'job1'
+      run WfJob2, id: 'job2', after: id1
+      run WfJob3, after: [WfJob1, 'job2']
+    end
+
+  end
+
+  def expect_jobs(*jobs)
+    j1, j2, j3 = *jobs
+
+    expect(j1.finished?).to eq false
+    expect(j2.finished?).to eq false
+
+    expect(j1.klass).to eq 'WfJob1'
+    expect(j2.klass).to eq 'WfJob2'
+    expect(j3.klass).to eq 'WfJob3'
+
+    expect(j1.outgoing).to include(j2.id, j3.id)
+    expect(j2.outgoing).to include(j3.id)
+    expect(j3.outgoing).to include
+
+    expect(j1.incoming).to include
+    expect(j2.incoming).to include(j1.id)
+    expect(j3.incoming).to include(j1.id, j2.id)
+
+    expect(j1.initial?).to eq true
+    expect(j2.initial?).to eq false
+    expect(j3.initial?).to eq false
+  end
+
+  context 'model' do
+    it 'default store' do
+      w = Burst::Workflow.new
+
+      expect(w.attributes).to include(:id, jobs: {}, klass: Burst::Workflow.to_s)
+
+      expect(w.started?).to eq false
+      expect(w.failed?).to eq false
+      expect(w.finished?).to eq true
+      expect(w.running?).to eq false
+      expect(w.status).to eq Burst::Workflow::FINISHED
+    end
+
+    it 'store persistance' do
+      w = Burst::Workflow.new
+      w.save!
+
+      w2 = Burst::Workflow.find(w.id)
+      expect(w2.attributes).to include(w.attributes)
+    end
+
+    it 'builded store' do
+      w = W1.build
+
+      jobs = w.jobs
+      expect_jobs(*jobs.values.map{|json| Burst::Job.new(w, json) })
+
+      expect(w.attributes).to include(:id, jobs: jobs, klass: W1.to_s)
+    end
+
+    it 'builded persistance' do
+      w = W1.build
+      w.save!
+
+      jobs = w.jobs
+      expect_jobs(*jobs.values.map{|json| Burst::Job.new(w, json) })
+
+      w2 = Burst::Workflow.find(w.id)
+      expect(w2.attributes).to include(w.attributes)
+    end
+  end
+
+  def expect_wf(w)
+    j1 = w.find_job('job1')
+    j2 = w.find_job('job2')
+    j3 = w.find_job(WfJob3)
+
+    expect(j1.finished?).to eq false
+    expect(j2.finished?).to eq false
+
+    expect(j1.klass).to eq 'WfJob1'
+    expect(j2.klass).to eq 'WfJob2'
+    expect(j3.klass).to eq 'WfJob3'
+
+    expect(j1.outgoing).to include(j2.id, j3.id)
+    expect(j2.outgoing).to include(j3.id)
+    expect(j3.outgoing).to include
+
+    expect(j1.incoming).to include
+    expect(j2.incoming).to include(j1.id)
+    expect(j3.incoming).to include(j1.id, j2.id)
+
+    expect(j1.initial?).to eq true
+    expect(j2.initial?).to eq false
+    expect(j3.initial?).to eq false
+
+    expect(w.initial_jobs).to include(j1)
+  end
+
+  it 'check builder' do
+    w = W1.build
+    expect_wf(w)
+
+    expect(w.status)
+  end
+
+  it 'check persistance' do
+    w = W1.build
+    w.save!
+
+    w2 = W1.find(w.id)
+    expect_wf(w2)
+  end
+
+  it 'start without perform' do
+    w = W1.build
+    w.save!
+
+    expect(w.started?).to eq false
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq false
+    expect(w.running?).to eq false
+    expect(w.status).to eq Burst::Workflow::INITIAL
+
+    w.start!
+
+    expect(Burst::Worker).to have_jobs(w.id, ['job1'])
+    expect(Burst::Worker).not_to have_jobs(w.id, ['job2'])
+
+    w = W1.find(w.id)
+
+    expect(w.started?).to eq false
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq false
+    expect(w.running?).to eq false
+    expect(w.status).to eq Burst::Workflow::INITIAL
+  end
+
+  it 'start with perform' do
+    w = W1.build
+    w.save!
+
+    expect(w.started?).to eq false
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq false
+    expect(w.running?).to eq false
+    expect(w.status).to eq Burst::Workflow::INITIAL
+
+    perform_enqueued_jobs do
+      w.start!
+    end
+
+    w = W1.find(w.id)
+
+    expect(w.started?).to eq true
+    expect(w.failed?).to eq false
+    expect(w.finished?).to eq true
+    expect(w.running?).to eq false
+    expect(w.status).to eq Burst::Workflow::FINISHED
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: burstflow
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Samoilenko Yuri
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: It has dependency, result pipelining and suspend/resume ability
+email:
+- kinnalru@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- burst.gemspec
+- config/database.yml
+- db/migrate/20180101000001_create_workflow.rb
+- db/schema.rb
+- db/seeds.rb
+- lib/burst.rb
+- lib/burst/builder.rb
+- lib/burst/configuration.rb
+- lib/burst/job.rb
+- lib/burst/manager.rb
+- lib/burst/model.rb
+- lib/burst/worker.rb
+- lib/burst/workflow.rb
+- lib/burst/workflow_helper.rb
+- spec/burst_spec.rb
+- spec/cases_spec.rb
+- spec/job_spec.rb
+- spec/spec_helper.rb
+- spec/support/database_clean.rb
+- spec/support/runner.rb
+- spec/workflow_spec.rb
+homepage: https://github.com/RnD-Soft/burst
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Burst is a parallel workflow runner using ActiveRecord and ActiveJob
+test_files:
+- spec/burst_spec.rb
+- spec/cases_spec.rb
+- spec/job_spec.rb
+- spec/spec_helper.rb
+- spec/support/database_clean.rb
+- spec/support/runner.rb
+- spec/workflow_spec.rb