burstflow 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +1 -1
- data/README.md +29 -21
- data/burstflow.gemspec +4 -4
- data/db/schema.rb +11 -13
- data/lib/burstflow/job.rb +23 -26
- data/lib/burstflow/job/callbacks.rb +4 -9
- data/lib/burstflow/job/exception.rb +3 -1
- data/lib/burstflow/job/initialization.rb +3 -6
- data/lib/burstflow/job/state.rb +17 -13
- data/lib/burstflow/manager.rb +85 -86
- data/lib/burstflow/railtie.rb +5 -2
- data/lib/burstflow/version.rb +3 -1
- data/lib/burstflow/worker.rb +21 -21
- data/lib/burstflow/workflow.rb +164 -156
- data/lib/burstflow/workflow/builder.rb +67 -64
- data/lib/burstflow/workflow/callbacks.rb +10 -16
- data/lib/burstflow/workflow/configuration.rb +37 -36
- data/lib/burstflow/workflow/exception.rb +3 -1
- data/lib/generators/burstflow/install/install_generator.rb +11 -5
- data/spec/builder_spec.rb +26 -27
- data/spec/generators/install_generator_spec.rb +11 -7
- data/spec/job_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- data/spec/workflow_spec.rb +128 -101
- metadata +4 -4
@@ -1,91 +1,94 @@
|
|
1
1
|
module Burstflow
|
2
|
-
|
3
|
-
class Workflow::Builder
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
2
|
+
|
3
|
+
class Workflow::Builder
|
4
|
+
|
5
|
+
def initialize(workflow, *args, &block)
|
6
|
+
@workflow = workflow
|
7
|
+
@deps = []
|
8
|
+
@jobs_by_class = {}
|
9
|
+
@jobs_by_id = {}
|
10
|
+
|
11
|
+
@workflow.jobs_config.each_pair do |id, job_hash|
|
12
|
+
job = Burstflow::Job.from_hash(@workflow, job_hash)
|
13
|
+
|
14
|
+
@jobs_by_class[job.klass.to_s] ||= []
|
15
|
+
@jobs_by_class[job.klass.to_s] << job
|
16
|
+
@jobs_by_id[id] = job
|
17
|
+
job.incoming.each do |from|
|
18
|
+
@deps << { from: from, to: id }
|
19
|
+
end
|
20
|
+
|
21
|
+
job.outgoing.each do |to|
|
22
|
+
@deps << { from: id, to: to }
|
23
|
+
end
|
24
|
+
|
25
|
+
@deps.uniq!
|
18
26
|
end
|
19
27
|
|
20
|
-
|
21
|
-
|
28
|
+
instance_exec *args, &block
|
29
|
+
resolve_dependencies
|
22
30
|
end
|
23
31
|
|
24
|
-
|
25
|
-
|
32
|
+
def run(klass, opts = {})
|
33
|
+
opts = opts.with_indifferent_access
|
26
34
|
|
27
|
-
|
28
|
-
|
29
|
-
end
|
35
|
+
before_deps = opts.delete(:before) || []
|
36
|
+
after_deps = opts.delete(:after) || []
|
30
37
|
|
31
|
-
|
32
|
-
opts = opts.with_indifferent_access
|
38
|
+
job = klass.new(@workflow, opts)
|
33
39
|
|
34
|
-
|
35
|
-
|
40
|
+
[*before_deps].each do |dep|
|
41
|
+
@deps << { from: job.id, to: dep.to_s }
|
42
|
+
end
|
36
43
|
|
37
|
-
|
44
|
+
[*after_deps].each do |dep|
|
45
|
+
@deps << { from: dep.to_s, to: job.id }
|
46
|
+
end
|
38
47
|
|
39
|
-
|
40
|
-
@
|
41
|
-
end
|
48
|
+
@jobs_by_class[klass.to_s] ||= []
|
49
|
+
@jobs_by_class[klass.to_s] << job
|
42
50
|
|
43
|
-
|
44
|
-
@deps << { from: dep.to_s, to: job.id }
|
45
|
-
end
|
51
|
+
raise 'Job id duplication' if @jobs_by_id.key?(job.id)
|
46
52
|
|
47
|
-
|
48
|
-
@jobs_by_class[klass.to_s] << job
|
53
|
+
@jobs_by_id[job.id] = job
|
49
54
|
|
50
|
-
|
51
|
-
|
55
|
+
job.id
|
56
|
+
end
|
52
57
|
|
53
|
-
|
54
|
-
|
58
|
+
def find_job(id_or_klass)
|
59
|
+
id = if @jobs_by_id.key?(id_or_klass)
|
60
|
+
id_or_klass
|
61
|
+
else
|
62
|
+
jobs = @jobs_by_class[id_or_klass.to_s]
|
55
63
|
|
56
|
-
|
57
|
-
|
58
|
-
id_or_klass
|
59
|
-
else
|
60
|
-
jobs = @jobs_by_class[id_or_klass.to_s]
|
64
|
+
raise "No job with #{id_or_klass} klass or id found" if jobs.count == 0
|
65
|
+
raise "Duplicated jobs with #{id_or_klass} klass or id detected" if jobs.count > 1
|
61
66
|
|
62
|
-
|
63
|
-
|
67
|
+
jobs.first.id
|
68
|
+
end
|
64
69
|
|
65
|
-
|
70
|
+
@jobs_by_id[id]
|
66
71
|
end
|
67
72
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
@deps.each do |dependency|
|
73
|
-
from = find_job(dependency[:from].to_s)
|
74
|
-
to = find_job(dependency[:to].to_s)
|
73
|
+
def resolve_dependencies
|
74
|
+
@deps.each do |dependency|
|
75
|
+
from = find_job(dependency[:from].to_s)
|
76
|
+
to = find_job(dependency[:to].to_s)
|
75
77
|
|
76
|
-
|
77
|
-
|
78
|
+
to.incoming << from.id
|
79
|
+
from.outgoing << to.id
|
78
80
|
|
79
|
-
|
80
|
-
|
81
|
+
to.incoming.uniq!
|
82
|
+
from.outgoing.uniq!
|
83
|
+
end
|
81
84
|
end
|
82
|
-
end
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
-
|
86
|
+
def as_json
|
87
|
+
@jobs_by_id.each_with_object({}) do |(_id, job), json|
|
88
|
+
json[job.id] = job.as_json
|
89
|
+
end
|
87
90
|
end
|
91
|
+
|
88
92
|
end
|
89
93
|
|
90
94
|
end
|
91
|
-
end
|
@@ -1,66 +1,60 @@
|
|
1
1
|
module Burstflow::Workflow::Callbacks
|
2
|
+
|
2
3
|
extend ActiveSupport::Concern
|
3
4
|
include ActiveSupport::Callbacks
|
4
5
|
|
5
6
|
included do
|
6
|
-
|
7
7
|
define_callbacks :failure, :finish, :suspend, :resume
|
8
|
-
|
9
8
|
end
|
10
9
|
|
11
10
|
class_methods do
|
12
|
-
|
13
11
|
def before_failure(*filters, &blk)
|
14
12
|
set_callback(:failure, :before, *filters, &blk)
|
15
13
|
end
|
16
|
-
|
14
|
+
|
17
15
|
def after_failure(*filters, &blk)
|
18
16
|
set_callback(:failure, :after, *filters, &blk)
|
19
17
|
end
|
20
|
-
|
18
|
+
|
21
19
|
def around_failure(*filters, &blk)
|
22
20
|
set_callback(:failure, :around, *filters, &blk)
|
23
21
|
end
|
24
22
|
|
25
|
-
|
26
23
|
def before_finish(*filters, &blk)
|
27
24
|
set_callback(:finish, :before, *filters, &blk)
|
28
25
|
end
|
29
|
-
|
26
|
+
|
30
27
|
def after_finish(*filters, &blk)
|
31
28
|
set_callback(:finish, :after, *filters, &blk)
|
32
29
|
end
|
33
|
-
|
30
|
+
|
34
31
|
def around_finish(*filters, &blk)
|
35
32
|
set_callback(:finish, :around, *filters, &blk)
|
36
33
|
end
|
37
34
|
|
38
|
-
|
39
35
|
def before_suspend(*filters, &blk)
|
40
36
|
set_callback(:suspend, :before, *filters, &blk)
|
41
37
|
end
|
42
|
-
|
38
|
+
|
43
39
|
def after_suspend(*filters, &blk)
|
44
40
|
set_callback(:suspend, :after, *filters, &blk)
|
45
41
|
end
|
46
|
-
|
42
|
+
|
47
43
|
def around_suspend(*filters, &blk)
|
48
44
|
set_callback(:suspend, :around, *filters, &blk)
|
49
45
|
end
|
50
46
|
|
51
|
-
|
52
47
|
def before_resume(*filters, &blk)
|
53
48
|
set_callback(:resume, :before, *filters, &blk)
|
54
49
|
end
|
55
|
-
|
50
|
+
|
56
51
|
def after_resume(*filters, &blk)
|
57
52
|
set_callback(:resume, :after, *filters, &blk)
|
58
53
|
end
|
59
|
-
|
54
|
+
|
60
55
|
def around_resume(*filters, &blk)
|
61
56
|
set_callback(:resume, :around, *filters, &blk)
|
62
57
|
end
|
63
|
-
|
64
58
|
end
|
65
59
|
|
66
|
-
end
|
60
|
+
end
|
@@ -1,55 +1,56 @@
|
|
1
1
|
module Burstflow
|
2
2
|
|
3
|
-
module Workflow::Configuration
|
4
|
-
extend ActiveSupport::Concern
|
3
|
+
module Workflow::Configuration
|
5
4
|
|
6
|
-
|
5
|
+
extend ActiveSupport::Concern
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
class JSONBWithIndifferentAccess
|
8
|
+
|
9
|
+
def self.dump(hash)
|
10
|
+
hash.as_json
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.load(hash)
|
14
|
+
hash ||= {}
|
15
|
+
hash = JSON.parse(hash) if hash.is_a? String
|
16
|
+
hash.with_indifferent_access
|
17
|
+
end
|
11
18
|
|
12
|
-
def self.load(hash)
|
13
|
-
hash ||= {}
|
14
|
-
hash = JSON.parse(hash) if hash.is_a? String
|
15
|
-
hash.with_indifferent_access
|
16
19
|
end
|
17
20
|
|
18
|
-
|
21
|
+
included do |_klass|
|
22
|
+
serialize :flow, JSONBWithIndifferentAccess
|
19
23
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
builder.resolve_dependencies
|
27
|
-
builder.as_json
|
24
|
+
def configure(*args)
|
25
|
+
builder = Builder.new
|
26
|
+
builder.instance_exec *args, &self.class.configuration
|
27
|
+
builder.resolve_dependencies
|
28
|
+
builder.as_json
|
29
|
+
end
|
28
30
|
end
|
29
|
-
end
|
30
31
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
32
|
+
class_methods do
|
33
|
+
def define_flow_attributes(*keys)
|
34
|
+
keys.each do |key|
|
35
|
+
define_method key.to_sym do
|
36
|
+
return flow[key.to_s]
|
37
|
+
end
|
37
38
|
|
38
|
-
|
39
|
-
|
39
|
+
define_method "#{key}=".to_sym do |v|
|
40
|
+
return flow[key.to_s] = v
|
41
|
+
end
|
40
42
|
end
|
41
43
|
end
|
42
|
-
end
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
45
|
+
def configure(&block)
|
46
|
+
@configuration = block
|
47
|
+
end
|
47
48
|
|
48
|
-
|
49
|
-
|
49
|
+
def configuration
|
50
|
+
@configuration
|
51
|
+
end
|
50
52
|
end
|
53
|
+
|
51
54
|
end
|
52
55
|
|
53
56
|
end
|
54
|
-
|
55
|
-
end
|
@@ -2,11 +2,14 @@ require 'rails/generators'
|
|
2
2
|
require 'rails/generators/migration'
|
3
3
|
|
4
4
|
module Burstflow
|
5
|
+
|
5
6
|
module Generators
|
7
|
+
|
6
8
|
class InstallGenerator < ::Rails::Generators::Base
|
9
|
+
|
7
10
|
include Rails::Generators::Migration
|
8
|
-
source_root File.expand_path('
|
9
|
-
desc
|
11
|
+
source_root File.expand_path('templates', __dir__)
|
12
|
+
desc 'Add the migrations for Burstflow'
|
10
13
|
|
11
14
|
def self.next_migration_number(path)
|
12
15
|
next_migration_number = current_migration_number(path) + 1
|
@@ -14,9 +17,12 @@ module Burstflow
|
|
14
17
|
end
|
15
18
|
|
16
19
|
def copy_migrations
|
17
|
-
migration_template
|
18
|
-
|
20
|
+
migration_template 'create_workflow.rb',
|
21
|
+
'db/migrate/create_workflow.rb'
|
19
22
|
end
|
23
|
+
|
20
24
|
end
|
25
|
+
|
21
26
|
end
|
22
|
-
|
27
|
+
|
28
|
+
end
|
data/spec/builder_spec.rb
CHANGED
@@ -6,23 +6,23 @@ describe Burstflow::Workflow::Builder do
|
|
6
6
|
BuilderJob2 = Class.new(Burstflow::Job)
|
7
7
|
BuilderJob3 = Class.new(Burstflow::Job)
|
8
8
|
|
9
|
-
let(:workflow){double(:workflow, id: 'id1', jobs_config: {})}
|
9
|
+
let(:workflow){ double(:workflow, id: 'id1', jobs_config: {}) }
|
10
10
|
|
11
11
|
it 'without dependencies' do
|
12
|
-
builder = Burstflow::Workflow::Builder.new workflow do
|
12
|
+
builder = Burstflow::Workflow::Builder.new workflow do
|
13
13
|
$jobid1 = run BuilderJob1, params: { param1: true }
|
14
14
|
end
|
15
15
|
|
16
16
|
flow = builder.as_json
|
17
17
|
expect(flow.count).to eq 1
|
18
18
|
|
19
|
-
expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: {'param1' => true})
|
19
|
+
expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: { 'param1' => true })
|
20
20
|
end
|
21
21
|
|
22
22
|
it 'with dependencies' do
|
23
23
|
builder = Burstflow::Workflow::Builder.new workflow, :arg1, :arg2 do |arg1, arg2|
|
24
|
-
$jobid1 = run BuilderJob1, params: { param1: true, arg: arg1}
|
25
|
-
$jobid2 = run BuilderJob2, params: { param2: true, arg: arg2}, after: BuilderJob1
|
24
|
+
$jobid1 = run BuilderJob1, params: { param1: true, arg: arg1 }
|
25
|
+
$jobid2 = run BuilderJob2, params: { param2: true, arg: arg2 }, after: BuilderJob1
|
26
26
|
$jobid3 = run BuilderJob3, before: BuilderJob2, after: $jobid1
|
27
27
|
$jobid4 = run BuilderJob3, after: $jobid3
|
28
28
|
end
|
@@ -30,34 +30,33 @@ describe Burstflow::Workflow::Builder do
|
|
30
30
|
flow = builder.as_json
|
31
31
|
expect(flow.count).to eq 4
|
32
32
|
|
33
|
-
expect(flow[$jobid1]).to include(:id,
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
33
|
+
expect(flow[$jobid1]).to include(:id,
|
34
|
+
klass: BuilderJob1.to_s,
|
35
|
+
incoming: [],
|
36
|
+
outgoing: [$jobid2, $jobid3],
|
37
|
+
workflow_id: 'id1',
|
38
|
+
params: { 'param1' => true, 'arg' => :arg1 })
|
39
39
|
|
40
40
|
expect(flow[$jobid2]).to include(:id,
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
41
|
+
klass: BuilderJob2.to_s,
|
42
|
+
incoming: [$jobid1, $jobid3],
|
43
|
+
outgoing: [],
|
44
|
+
workflow_id: 'id1',
|
45
|
+
params: { 'param2' => true, 'arg' => :arg2 })
|
46
46
|
|
47
47
|
expect(flow[$jobid3]).to include(:id,
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
48
|
+
klass: BuilderJob3.to_s,
|
49
|
+
incoming: [$jobid1],
|
50
|
+
outgoing: [$jobid2, $jobid4],
|
51
|
+
workflow_id: 'id1',
|
52
|
+
params: nil)
|
53
53
|
|
54
54
|
expect(flow[$jobid4]).to include(:id,
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
klass: BuilderJob3.to_s,
|
56
|
+
incoming: [$jobid3],
|
57
|
+
outgoing: [],
|
58
|
+
workflow_id: 'id1',
|
59
|
+
params: nil)
|
60
60
|
end
|
61
|
-
|
62
61
|
end
|
63
62
|
end
|