burstflow 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -3
- data/Gemfile.lock +119 -0
- data/burstflow.gemspec +10 -6
- data/config/database.yml +4 -3
- data/db/migrate/20180101000001_create_workflow.rb +1 -0
- data/db/schema.rb +13 -7
- data/lib/burstflow.rb +11 -0
- data/lib/burstflow/job.rb +102 -0
- data/lib/burstflow/job/callbacks.rb +55 -0
- data/lib/burstflow/job/exception.rb +8 -0
- data/lib/burstflow/job/initialization.rb +35 -0
- data/lib/{burst → burstflow/job}/model.rb +1 -3
- data/lib/burstflow/job/state.rb +125 -0
- data/lib/burstflow/manager.rb +123 -0
- data/lib/burstflow/railtie.rb +6 -0
- data/lib/burstflow/version.rb +3 -0
- data/lib/burstflow/worker.rb +59 -0
- data/lib/burstflow/workflow.rb +207 -0
- data/lib/burstflow/workflow/builder.rb +91 -0
- data/lib/burstflow/workflow/callbacks.rb +66 -0
- data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
- data/lib/burstflow/workflow/exception.rb +8 -0
- data/lib/generators/burstflow/install/install_generator.rb +22 -0
- data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
- data/spec/builder_spec.rb +63 -0
- data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
- data/spec/generators/install_generator_spec.rb +27 -0
- data/spec/job_spec.rb +18 -8
- data/spec/spec_helper.rb +4 -1
- data/spec/support/database_clean.rb +4 -1
- data/spec/workflow_spec.rb +397 -147
- metadata +45 -21
- data/db/migrate/20180101000001_create_workflow.rb +0 -13
- data/db/seeds.rb +0 -1
- data/lib/burst.rb +0 -37
- data/lib/burst/builder.rb +0 -48
- data/lib/burst/configuration.rb +0 -27
- data/lib/burst/job.rb +0 -187
- data/lib/burst/manager.rb +0 -79
- data/lib/burst/worker.rb +0 -42
- data/lib/burst/workflow.rb +0 -148
- data/spec/cases_spec.rb +0 -180
@@ -0,0 +1,91 @@
|
|
1
|
+
module Burstflow
|
2
|
+
|
3
|
+
class Workflow::Builder
|
4
|
+
def initialize workflow, *args, &block
|
5
|
+
@workflow = workflow
|
6
|
+
@deps = []
|
7
|
+
@jobs_by_class = {}
|
8
|
+
@jobs_by_id = {}
|
9
|
+
|
10
|
+
@workflow.jobs_config.each_pair do |id, job_hash|
|
11
|
+
job = Burstflow::Job.from_hash(@workflow, job_hash)
|
12
|
+
|
13
|
+
@jobs_by_class[job.klass.to_s] ||= []
|
14
|
+
@jobs_by_class[job.klass.to_s] << job
|
15
|
+
@jobs_by_id[id] = job
|
16
|
+
job.incoming.each do |from|
|
17
|
+
@deps << { from: from, to: id }
|
18
|
+
end
|
19
|
+
|
20
|
+
job.outgoing.each do |to|
|
21
|
+
@deps << { from: id, to: to }
|
22
|
+
end
|
23
|
+
|
24
|
+
@deps.uniq!
|
25
|
+
end
|
26
|
+
|
27
|
+
instance_exec *args, &block
|
28
|
+
resolve_dependencies
|
29
|
+
end
|
30
|
+
|
31
|
+
def run(klass, opts = {})
|
32
|
+
opts = opts.with_indifferent_access
|
33
|
+
|
34
|
+
before_deps = opts.delete(:before) || []
|
35
|
+
after_deps = opts.delete(:after) || []
|
36
|
+
|
37
|
+
job = klass.new(@workflow, opts)
|
38
|
+
|
39
|
+
[*before_deps].each do |dep|
|
40
|
+
@deps << { from: job.id, to: dep.to_s }
|
41
|
+
end
|
42
|
+
|
43
|
+
[*after_deps].each do |dep|
|
44
|
+
@deps << { from: dep.to_s, to: job.id }
|
45
|
+
end
|
46
|
+
|
47
|
+
@jobs_by_class[klass.to_s] ||= []
|
48
|
+
@jobs_by_class[klass.to_s] << job
|
49
|
+
|
50
|
+
raise "Job id duplication" if @jobs_by_id.key?(job.id)
|
51
|
+
@jobs_by_id[job.id] = job
|
52
|
+
|
53
|
+
job.id
|
54
|
+
end
|
55
|
+
|
56
|
+
def find_job id_or_klass
|
57
|
+
id = if @jobs_by_id.key?(id_or_klass)
|
58
|
+
id_or_klass
|
59
|
+
else
|
60
|
+
jobs = @jobs_by_class[id_or_klass.to_s]
|
61
|
+
|
62
|
+
raise "No job with #{id_or_klass.to_s} klass or id found" if jobs.count == 0
|
63
|
+
raise "Duplicated jobs with #{id_or_klass.to_s} klass or id detected" if jobs.count > 1
|
64
|
+
|
65
|
+
jobs.first.id
|
66
|
+
end
|
67
|
+
|
68
|
+
@jobs_by_id[id]
|
69
|
+
end
|
70
|
+
|
71
|
+
def resolve_dependencies
|
72
|
+
@deps.each do |dependency|
|
73
|
+
from = find_job(dependency[:from].to_s)
|
74
|
+
to = find_job(dependency[:to].to_s)
|
75
|
+
|
76
|
+
to.incoming << from.id
|
77
|
+
from.outgoing << to.id
|
78
|
+
|
79
|
+
to.incoming.uniq!
|
80
|
+
from.outgoing.uniq!
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def as_json
|
85
|
+
@jobs_by_id.each_with_object({}) do |(id, job), json|
|
86
|
+
json[job.id] = job.as_json
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module Burstflow::Workflow::Callbacks
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
include ActiveSupport::Callbacks
|
4
|
+
|
5
|
+
included do
|
6
|
+
|
7
|
+
define_callbacks :failure, :finish, :suspend, :resume
|
8
|
+
|
9
|
+
end
|
10
|
+
|
11
|
+
class_methods do
|
12
|
+
|
13
|
+
def before_failure(*filters, &blk)
|
14
|
+
set_callback(:failure, :before, *filters, &blk)
|
15
|
+
end
|
16
|
+
|
17
|
+
def after_failure(*filters, &blk)
|
18
|
+
set_callback(:failure, :after, *filters, &blk)
|
19
|
+
end
|
20
|
+
|
21
|
+
def around_failure(*filters, &blk)
|
22
|
+
set_callback(:failure, :around, *filters, &blk)
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def before_finish(*filters, &blk)
|
27
|
+
set_callback(:finish, :before, *filters, &blk)
|
28
|
+
end
|
29
|
+
|
30
|
+
def after_finish(*filters, &blk)
|
31
|
+
set_callback(:finish, :after, *filters, &blk)
|
32
|
+
end
|
33
|
+
|
34
|
+
def around_finish(*filters, &blk)
|
35
|
+
set_callback(:finish, :around, *filters, &blk)
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
def before_suspend(*filters, &blk)
|
40
|
+
set_callback(:suspend, :before, *filters, &blk)
|
41
|
+
end
|
42
|
+
|
43
|
+
def after_suspend(*filters, &blk)
|
44
|
+
set_callback(:suspend, :after, *filters, &blk)
|
45
|
+
end
|
46
|
+
|
47
|
+
def around_suspend(*filters, &blk)
|
48
|
+
set_callback(:suspend, :around, *filters, &blk)
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
def before_resume(*filters, &blk)
|
53
|
+
set_callback(:resume, :before, *filters, &blk)
|
54
|
+
end
|
55
|
+
|
56
|
+
def after_resume(*filters, &blk)
|
57
|
+
set_callback(:resume, :after, *filters, &blk)
|
58
|
+
end
|
59
|
+
|
60
|
+
def around_resume(*filters, &blk)
|
61
|
+
set_callback(:resume, :around, *filters, &blk)
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
|
-
module
|
1
|
+
module Burstflow
|
2
2
|
|
3
|
+
module Workflow::Configuration
|
3
4
|
extend ActiveSupport::Concern
|
4
5
|
|
5
6
|
class JSONBWithIndifferentAccess
|
@@ -18,25 +19,12 @@ module Burst::WorkflowHelper
|
|
18
19
|
|
19
20
|
included do |_klass|
|
20
21
|
serialize :flow, JSONBWithIndifferentAccess
|
21
|
-
|
22
|
-
def first_job
|
23
|
-
all_jobs.min_by{|n| n.started_at || Time.now.to_i }
|
24
|
-
end
|
25
|
-
|
26
|
-
def last_job
|
27
|
-
all_jobs.max_by{|n| n.finished_at || 0 } if finished?
|
28
|
-
end
|
29
|
-
|
30
|
-
def started_at
|
31
|
-
first_job&.started_at
|
32
|
-
end
|
33
|
-
|
34
|
-
def finished_at
|
35
|
-
last_job&.finished_at
|
36
|
-
end
|
37
|
-
|
22
|
+
|
38
23
|
def configure(*args)
|
39
|
-
|
24
|
+
builder = Builder.new()
|
25
|
+
builder.instance_exec *args, &self.class.configuration
|
26
|
+
builder.resolve_dependencies
|
27
|
+
builder.as_json
|
40
28
|
end
|
41
29
|
end
|
42
30
|
|
@@ -64,23 +52,4 @@ module Burst::WorkflowHelper
|
|
64
52
|
|
65
53
|
end
|
66
54
|
|
67
|
-
|
68
|
-
# class Test
|
69
|
-
# def self.configure &block
|
70
|
-
# @c = block
|
71
|
-
# end
|
72
|
-
|
73
|
-
# def self.c
|
74
|
-
# @c
|
75
|
-
# end
|
76
|
-
|
77
|
-
# def go
|
78
|
-
# puts "111111"
|
79
|
-
# end
|
80
|
-
|
81
|
-
# def run
|
82
|
-
# instance_eval &Test.c
|
83
|
-
# end
|
84
|
-
|
85
|
-
|
86
|
-
# end
|
55
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
require 'rails/generators/migration'
|
3
|
+
|
4
|
+
module Burstflow
|
5
|
+
module Generators
|
6
|
+
class InstallGenerator < ::Rails::Generators::Base
|
7
|
+
include Rails::Generators::Migration
|
8
|
+
source_root File.expand_path('../templates', __FILE__)
|
9
|
+
desc "Add the migrations for Burstflow"
|
10
|
+
|
11
|
+
def self.next_migration_number(path)
|
12
|
+
next_migration_number = current_migration_number(path) + 1
|
13
|
+
ActiveRecord::Migration.next_migration_number(next_migration_number)
|
14
|
+
end
|
15
|
+
|
16
|
+
def copy_migrations
|
17
|
+
migration_template "create_workflow.rb",
|
18
|
+
"db/migrate/create_workflow.rb"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class CreateWorkflow < ActiveRecord::Migration[5.1]
|
2
|
+
|
3
|
+
def change
|
4
|
+
enable_extension 'pgcrypto'
|
5
|
+
|
6
|
+
create_table :burstflow_workflows, id: :uuid do |t|
|
7
|
+
t.string :type, index: true
|
8
|
+
t.string :status, index: true
|
9
|
+
t.jsonb :flow, null: false, default: {}
|
10
|
+
|
11
|
+
t.timestamps
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Burstflow::Workflow::Builder do
|
4
|
+
context 'initializing' do
|
5
|
+
BuilderJob1 = Class.new(Burstflow::Job)
|
6
|
+
BuilderJob2 = Class.new(Burstflow::Job)
|
7
|
+
BuilderJob3 = Class.new(Burstflow::Job)
|
8
|
+
|
9
|
+
let(:workflow){double(:workflow, id: 'id1', jobs_config: {})}
|
10
|
+
|
11
|
+
it 'without dependencies' do
|
12
|
+
builder = Burstflow::Workflow::Builder.new workflow do
|
13
|
+
$jobid1 = run BuilderJob1, params: { param1: true }
|
14
|
+
end
|
15
|
+
|
16
|
+
flow = builder.as_json
|
17
|
+
expect(flow.count).to eq 1
|
18
|
+
|
19
|
+
expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: {'param1' => true})
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'with dependencies' do
|
23
|
+
builder = Burstflow::Workflow::Builder.new workflow, :arg1, :arg2 do |arg1, arg2|
|
24
|
+
$jobid1 = run BuilderJob1, params: { param1: true, arg: arg1}
|
25
|
+
$jobid2 = run BuilderJob2, params: { param2: true, arg: arg2}, after: BuilderJob1
|
26
|
+
$jobid3 = run BuilderJob3, before: BuilderJob2, after: $jobid1
|
27
|
+
$jobid4 = run BuilderJob3, after: $jobid3
|
28
|
+
end
|
29
|
+
|
30
|
+
flow = builder.as_json
|
31
|
+
expect(flow.count).to eq 4
|
32
|
+
|
33
|
+
expect(flow[$jobid1]).to include(:id,
|
34
|
+
klass: BuilderJob1.to_s,
|
35
|
+
incoming: [],
|
36
|
+
outgoing: [$jobid2, $jobid3],
|
37
|
+
workflow_id: 'id1',
|
38
|
+
params: {'param1' => true, 'arg' => :arg1})
|
39
|
+
|
40
|
+
expect(flow[$jobid2]).to include(:id,
|
41
|
+
klass: BuilderJob2.to_s,
|
42
|
+
incoming: [$jobid1, $jobid3],
|
43
|
+
outgoing: [],
|
44
|
+
workflow_id: 'id1',
|
45
|
+
params: {'param2' => true, 'arg' => :arg2})
|
46
|
+
|
47
|
+
expect(flow[$jobid3]).to include(:id,
|
48
|
+
klass: BuilderJob3.to_s,
|
49
|
+
incoming: [$jobid1],
|
50
|
+
outgoing: [$jobid2, $jobid4],
|
51
|
+
workflow_id: 'id1',
|
52
|
+
params: nil)
|
53
|
+
|
54
|
+
expect(flow[$jobid4]).to include(:id,
|
55
|
+
klass: BuilderJob3.to_s,
|
56
|
+
incoming: [$jobid3],
|
57
|
+
outgoing: [],
|
58
|
+
workflow_id: 'id1',
|
59
|
+
params: nil)
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require "generator_spec"
|
3
|
+
require 'tmpdir'
|
4
|
+
|
5
|
+
require 'generators/burstflow/install/install_generator'
|
6
|
+
|
7
|
+
module Burstflow
|
8
|
+
module Generators
|
9
|
+
describe InstallGenerator, type: :generator do
|
10
|
+
root_dir = File.expand_path(Dir.tmpdir(), __FILE__)
|
11
|
+
destination root_dir
|
12
|
+
|
13
|
+
before :all do
|
14
|
+
prepare_destination
|
15
|
+
run_generator
|
16
|
+
end
|
17
|
+
|
18
|
+
it "creates the installation db migration" do
|
19
|
+
migration_file =
|
20
|
+
Dir.glob("#{root_dir}/db/migrate/*create_workflow.rb")
|
21
|
+
|
22
|
+
assert_file migration_file[0],
|
23
|
+
/CreateWorkflow < ActiveRecord::Migration/
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/spec/job_spec.rb
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe
|
4
|
-
let(:w) {
|
3
|
+
describe Burstflow::Job do
|
4
|
+
let(:w) { Burstflow::Workflow.new }
|
5
5
|
|
6
6
|
context 'initializing' do
|
7
|
-
class
|
7
|
+
class JobJob1 < Burstflow::Job
|
8
8
|
end
|
9
9
|
|
10
|
-
subject{
|
10
|
+
subject{ JobJob1.new(w, {}.with_indifferent_access) }
|
11
11
|
|
12
12
|
it 'empty' do
|
13
13
|
expect(subject.workflow_id).to eq w.id
|
14
|
-
expect(subject.klass).to eq
|
14
|
+
expect(subject.klass).to eq JobJob1.to_s
|
15
15
|
|
16
16
|
expect(subject.incoming).to eq []
|
17
17
|
expect(subject.outgoing).to eq []
|
@@ -36,6 +36,7 @@ describe Burst::Job do
|
|
36
36
|
end
|
37
37
|
|
38
38
|
it '#start!' do
|
39
|
+
subject.enqueue!
|
39
40
|
subject.start!
|
40
41
|
|
41
42
|
expect(subject.started?).to eq true
|
@@ -43,6 +44,8 @@ describe Burst::Job do
|
|
43
44
|
end
|
44
45
|
|
45
46
|
it '#finish!' do
|
47
|
+
subject.enqueue!
|
48
|
+
subject.start!
|
46
49
|
subject.finish!
|
47
50
|
|
48
51
|
expect(subject.finished?).to eq true
|
@@ -52,29 +55,36 @@ describe Burst::Job do
|
|
52
55
|
end
|
53
56
|
|
54
57
|
it '#fail!' do
|
55
|
-
subject.
|
58
|
+
subject.enqueue!
|
59
|
+
subject.start!
|
60
|
+
subject.fail! "error"
|
56
61
|
|
57
62
|
expect(subject.finished?).to eq true
|
58
63
|
expect(subject.succeeded?).to eq false
|
59
64
|
expect(subject.failed?).to eq true
|
60
65
|
expect(subject.ready_to_start?).to eq false
|
66
|
+
expect(subject.failure).to include(message: 'error')
|
61
67
|
end
|
62
68
|
|
63
69
|
it '#suspend!' do
|
70
|
+
subject.enqueue!
|
71
|
+
subject.start!
|
64
72
|
subject.suspend!
|
65
73
|
|
66
74
|
expect(subject.finished?).to eq false
|
67
75
|
expect(subject.suspended?).to eq true
|
68
|
-
expect(subject.ready_to_start?).to eq
|
76
|
+
expect(subject.ready_to_start?).to eq false
|
69
77
|
end
|
70
78
|
|
71
79
|
it '#resume!' do
|
80
|
+
subject.enqueue!
|
81
|
+
subject.start!
|
72
82
|
subject.suspend!
|
73
83
|
subject.resume!
|
74
84
|
|
75
85
|
expect(subject.finished?).to eq false
|
76
86
|
expect(subject.resumed?).to eq true
|
77
|
-
expect(subject.ready_to_start?).to eq
|
87
|
+
expect(subject.ready_to_start?).to eq false
|
78
88
|
end
|
79
89
|
end
|
80
90
|
end
|