burstflow 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +1 -1
- data/README.md +29 -21
- data/burstflow.gemspec +4 -4
- data/db/schema.rb +11 -13
- data/lib/burstflow/job.rb +23 -26
- data/lib/burstflow/job/callbacks.rb +4 -9
- data/lib/burstflow/job/exception.rb +3 -1
- data/lib/burstflow/job/initialization.rb +3 -6
- data/lib/burstflow/job/state.rb +17 -13
- data/lib/burstflow/manager.rb +85 -86
- data/lib/burstflow/railtie.rb +5 -2
- data/lib/burstflow/version.rb +3 -1
- data/lib/burstflow/worker.rb +21 -21
- data/lib/burstflow/workflow.rb +164 -156
- data/lib/burstflow/workflow/builder.rb +67 -64
- data/lib/burstflow/workflow/callbacks.rb +10 -16
- data/lib/burstflow/workflow/configuration.rb +37 -36
- data/lib/burstflow/workflow/exception.rb +3 -1
- data/lib/generators/burstflow/install/install_generator.rb +11 -5
- data/spec/builder_spec.rb +26 -27
- data/spec/generators/install_generator_spec.rb +11 -7
- data/spec/job_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- data/spec/workflow_spec.rb +128 -101
- metadata +4 -4
@@ -1,91 +1,94 @@
|
|
1
1
|
module Burstflow
|
2
|
-
|
3
|
-
class Workflow::Builder
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
2
|
+
|
3
|
+
class Workflow::Builder
|
4
|
+
|
5
|
+
def initialize(workflow, *args, &block)
|
6
|
+
@workflow = workflow
|
7
|
+
@deps = []
|
8
|
+
@jobs_by_class = {}
|
9
|
+
@jobs_by_id = {}
|
10
|
+
|
11
|
+
@workflow.jobs_config.each_pair do |id, job_hash|
|
12
|
+
job = Burstflow::Job.from_hash(@workflow, job_hash)
|
13
|
+
|
14
|
+
@jobs_by_class[job.klass.to_s] ||= []
|
15
|
+
@jobs_by_class[job.klass.to_s] << job
|
16
|
+
@jobs_by_id[id] = job
|
17
|
+
job.incoming.each do |from|
|
18
|
+
@deps << { from: from, to: id }
|
19
|
+
end
|
20
|
+
|
21
|
+
job.outgoing.each do |to|
|
22
|
+
@deps << { from: id, to: to }
|
23
|
+
end
|
24
|
+
|
25
|
+
@deps.uniq!
|
18
26
|
end
|
19
27
|
|
20
|
-
|
21
|
-
|
28
|
+
instance_exec *args, &block
|
29
|
+
resolve_dependencies
|
22
30
|
end
|
23
31
|
|
24
|
-
|
25
|
-
|
32
|
+
def run(klass, opts = {})
|
33
|
+
opts = opts.with_indifferent_access
|
26
34
|
|
27
|
-
|
28
|
-
|
29
|
-
end
|
35
|
+
before_deps = opts.delete(:before) || []
|
36
|
+
after_deps = opts.delete(:after) || []
|
30
37
|
|
31
|
-
|
32
|
-
opts = opts.with_indifferent_access
|
38
|
+
job = klass.new(@workflow, opts)
|
33
39
|
|
34
|
-
|
35
|
-
|
40
|
+
[*before_deps].each do |dep|
|
41
|
+
@deps << { from: job.id, to: dep.to_s }
|
42
|
+
end
|
36
43
|
|
37
|
-
|
44
|
+
[*after_deps].each do |dep|
|
45
|
+
@deps << { from: dep.to_s, to: job.id }
|
46
|
+
end
|
38
47
|
|
39
|
-
|
40
|
-
@
|
41
|
-
end
|
48
|
+
@jobs_by_class[klass.to_s] ||= []
|
49
|
+
@jobs_by_class[klass.to_s] << job
|
42
50
|
|
43
|
-
|
44
|
-
@deps << { from: dep.to_s, to: job.id }
|
45
|
-
end
|
51
|
+
raise 'Job id duplication' if @jobs_by_id.key?(job.id)
|
46
52
|
|
47
|
-
|
48
|
-
@jobs_by_class[klass.to_s] << job
|
53
|
+
@jobs_by_id[job.id] = job
|
49
54
|
|
50
|
-
|
51
|
-
|
55
|
+
job.id
|
56
|
+
end
|
52
57
|
|
53
|
-
|
54
|
-
|
58
|
+
def find_job(id_or_klass)
|
59
|
+
id = if @jobs_by_id.key?(id_or_klass)
|
60
|
+
id_or_klass
|
61
|
+
else
|
62
|
+
jobs = @jobs_by_class[id_or_klass.to_s]
|
55
63
|
|
56
|
-
|
57
|
-
|
58
|
-
id_or_klass
|
59
|
-
else
|
60
|
-
jobs = @jobs_by_class[id_or_klass.to_s]
|
64
|
+
raise "No job with #{id_or_klass} klass or id found" if jobs.count == 0
|
65
|
+
raise "Duplicated jobs with #{id_or_klass} klass or id detected" if jobs.count > 1
|
61
66
|
|
62
|
-
|
63
|
-
|
67
|
+
jobs.first.id
|
68
|
+
end
|
64
69
|
|
65
|
-
|
70
|
+
@jobs_by_id[id]
|
66
71
|
end
|
67
72
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
@deps.each do |dependency|
|
73
|
-
from = find_job(dependency[:from].to_s)
|
74
|
-
to = find_job(dependency[:to].to_s)
|
73
|
+
def resolve_dependencies
|
74
|
+
@deps.each do |dependency|
|
75
|
+
from = find_job(dependency[:from].to_s)
|
76
|
+
to = find_job(dependency[:to].to_s)
|
75
77
|
|
76
|
-
|
77
|
-
|
78
|
+
to.incoming << from.id
|
79
|
+
from.outgoing << to.id
|
78
80
|
|
79
|
-
|
80
|
-
|
81
|
+
to.incoming.uniq!
|
82
|
+
from.outgoing.uniq!
|
83
|
+
end
|
81
84
|
end
|
82
|
-
end
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
-
|
86
|
+
def as_json
|
87
|
+
@jobs_by_id.each_with_object({}) do |(_id, job), json|
|
88
|
+
json[job.id] = job.as_json
|
89
|
+
end
|
87
90
|
end
|
91
|
+
|
88
92
|
end
|
89
93
|
|
90
94
|
end
|
91
|
-
end
|
@@ -1,66 +1,60 @@
|
|
1
1
|
module Burstflow::Workflow::Callbacks
|
2
|
+
|
2
3
|
extend ActiveSupport::Concern
|
3
4
|
include ActiveSupport::Callbacks
|
4
5
|
|
5
6
|
included do
|
6
|
-
|
7
7
|
define_callbacks :failure, :finish, :suspend, :resume
|
8
|
-
|
9
8
|
end
|
10
9
|
|
11
10
|
class_methods do
|
12
|
-
|
13
11
|
def before_failure(*filters, &blk)
|
14
12
|
set_callback(:failure, :before, *filters, &blk)
|
15
13
|
end
|
16
|
-
|
14
|
+
|
17
15
|
def after_failure(*filters, &blk)
|
18
16
|
set_callback(:failure, :after, *filters, &blk)
|
19
17
|
end
|
20
|
-
|
18
|
+
|
21
19
|
def around_failure(*filters, &blk)
|
22
20
|
set_callback(:failure, :around, *filters, &blk)
|
23
21
|
end
|
24
22
|
|
25
|
-
|
26
23
|
def before_finish(*filters, &blk)
|
27
24
|
set_callback(:finish, :before, *filters, &blk)
|
28
25
|
end
|
29
|
-
|
26
|
+
|
30
27
|
def after_finish(*filters, &blk)
|
31
28
|
set_callback(:finish, :after, *filters, &blk)
|
32
29
|
end
|
33
|
-
|
30
|
+
|
34
31
|
def around_finish(*filters, &blk)
|
35
32
|
set_callback(:finish, :around, *filters, &blk)
|
36
33
|
end
|
37
34
|
|
38
|
-
|
39
35
|
def before_suspend(*filters, &blk)
|
40
36
|
set_callback(:suspend, :before, *filters, &blk)
|
41
37
|
end
|
42
|
-
|
38
|
+
|
43
39
|
def after_suspend(*filters, &blk)
|
44
40
|
set_callback(:suspend, :after, *filters, &blk)
|
45
41
|
end
|
46
|
-
|
42
|
+
|
47
43
|
def around_suspend(*filters, &blk)
|
48
44
|
set_callback(:suspend, :around, *filters, &blk)
|
49
45
|
end
|
50
46
|
|
51
|
-
|
52
47
|
def before_resume(*filters, &blk)
|
53
48
|
set_callback(:resume, :before, *filters, &blk)
|
54
49
|
end
|
55
|
-
|
50
|
+
|
56
51
|
def after_resume(*filters, &blk)
|
57
52
|
set_callback(:resume, :after, *filters, &blk)
|
58
53
|
end
|
59
|
-
|
54
|
+
|
60
55
|
def around_resume(*filters, &blk)
|
61
56
|
set_callback(:resume, :around, *filters, &blk)
|
62
57
|
end
|
63
|
-
|
64
58
|
end
|
65
59
|
|
66
|
-
end
|
60
|
+
end
|
@@ -1,55 +1,56 @@
|
|
1
1
|
module Burstflow
|
2
2
|
|
3
|
-
module Workflow::Configuration
|
4
|
-
extend ActiveSupport::Concern
|
3
|
+
module Workflow::Configuration
|
5
4
|
|
6
|
-
|
5
|
+
extend ActiveSupport::Concern
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
class JSONBWithIndifferentAccess
|
8
|
+
|
9
|
+
def self.dump(hash)
|
10
|
+
hash.as_json
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.load(hash)
|
14
|
+
hash ||= {}
|
15
|
+
hash = JSON.parse(hash) if hash.is_a? String
|
16
|
+
hash.with_indifferent_access
|
17
|
+
end
|
11
18
|
|
12
|
-
def self.load(hash)
|
13
|
-
hash ||= {}
|
14
|
-
hash = JSON.parse(hash) if hash.is_a? String
|
15
|
-
hash.with_indifferent_access
|
16
19
|
end
|
17
20
|
|
18
|
-
|
21
|
+
included do |_klass|
|
22
|
+
serialize :flow, JSONBWithIndifferentAccess
|
19
23
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
builder.resolve_dependencies
|
27
|
-
builder.as_json
|
24
|
+
def configure(*args)
|
25
|
+
builder = Builder.new
|
26
|
+
builder.instance_exec *args, &self.class.configuration
|
27
|
+
builder.resolve_dependencies
|
28
|
+
builder.as_json
|
29
|
+
end
|
28
30
|
end
|
29
|
-
end
|
30
31
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
32
|
+
class_methods do
|
33
|
+
def define_flow_attributes(*keys)
|
34
|
+
keys.each do |key|
|
35
|
+
define_method key.to_sym do
|
36
|
+
return flow[key.to_s]
|
37
|
+
end
|
37
38
|
|
38
|
-
|
39
|
-
|
39
|
+
define_method "#{key}=".to_sym do |v|
|
40
|
+
return flow[key.to_s] = v
|
41
|
+
end
|
40
42
|
end
|
41
43
|
end
|
42
|
-
end
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
45
|
+
def configure(&block)
|
46
|
+
@configuration = block
|
47
|
+
end
|
47
48
|
|
48
|
-
|
49
|
-
|
49
|
+
def configuration
|
50
|
+
@configuration
|
51
|
+
end
|
50
52
|
end
|
53
|
+
|
51
54
|
end
|
52
55
|
|
53
56
|
end
|
54
|
-
|
55
|
-
end
|
@@ -2,11 +2,14 @@ require 'rails/generators'
|
|
2
2
|
require 'rails/generators/migration'
|
3
3
|
|
4
4
|
module Burstflow
|
5
|
+
|
5
6
|
module Generators
|
7
|
+
|
6
8
|
class InstallGenerator < ::Rails::Generators::Base
|
9
|
+
|
7
10
|
include Rails::Generators::Migration
|
8
|
-
source_root File.expand_path('
|
9
|
-
desc
|
11
|
+
source_root File.expand_path('templates', __dir__)
|
12
|
+
desc 'Add the migrations for Burstflow'
|
10
13
|
|
11
14
|
def self.next_migration_number(path)
|
12
15
|
next_migration_number = current_migration_number(path) + 1
|
@@ -14,9 +17,12 @@ module Burstflow
|
|
14
17
|
end
|
15
18
|
|
16
19
|
def copy_migrations
|
17
|
-
migration_template
|
18
|
-
|
20
|
+
migration_template 'create_workflow.rb',
|
21
|
+
'db/migrate/create_workflow.rb'
|
19
22
|
end
|
23
|
+
|
20
24
|
end
|
25
|
+
|
21
26
|
end
|
22
|
-
|
27
|
+
|
28
|
+
end
|
data/spec/builder_spec.rb
CHANGED
@@ -6,23 +6,23 @@ describe Burstflow::Workflow::Builder do
|
|
6
6
|
BuilderJob2 = Class.new(Burstflow::Job)
|
7
7
|
BuilderJob3 = Class.new(Burstflow::Job)
|
8
8
|
|
9
|
-
let(:workflow){double(:workflow, id: 'id1', jobs_config: {})}
|
9
|
+
let(:workflow){ double(:workflow, id: 'id1', jobs_config: {}) }
|
10
10
|
|
11
11
|
it 'without dependencies' do
|
12
|
-
builder = Burstflow::Workflow::Builder.new workflow do
|
12
|
+
builder = Burstflow::Workflow::Builder.new workflow do
|
13
13
|
$jobid1 = run BuilderJob1, params: { param1: true }
|
14
14
|
end
|
15
15
|
|
16
16
|
flow = builder.as_json
|
17
17
|
expect(flow.count).to eq 1
|
18
18
|
|
19
|
-
expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: {'param1' => true})
|
19
|
+
expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: { 'param1' => true })
|
20
20
|
end
|
21
21
|
|
22
22
|
it 'with dependencies' do
|
23
23
|
builder = Burstflow::Workflow::Builder.new workflow, :arg1, :arg2 do |arg1, arg2|
|
24
|
-
$jobid1 = run BuilderJob1, params: { param1: true, arg: arg1}
|
25
|
-
$jobid2 = run BuilderJob2, params: { param2: true, arg: arg2}, after: BuilderJob1
|
24
|
+
$jobid1 = run BuilderJob1, params: { param1: true, arg: arg1 }
|
25
|
+
$jobid2 = run BuilderJob2, params: { param2: true, arg: arg2 }, after: BuilderJob1
|
26
26
|
$jobid3 = run BuilderJob3, before: BuilderJob2, after: $jobid1
|
27
27
|
$jobid4 = run BuilderJob3, after: $jobid3
|
28
28
|
end
|
@@ -30,34 +30,33 @@ describe Burstflow::Workflow::Builder do
|
|
30
30
|
flow = builder.as_json
|
31
31
|
expect(flow.count).to eq 4
|
32
32
|
|
33
|
-
expect(flow[$jobid1]).to include(:id,
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
33
|
+
expect(flow[$jobid1]).to include(:id,
|
34
|
+
klass: BuilderJob1.to_s,
|
35
|
+
incoming: [],
|
36
|
+
outgoing: [$jobid2, $jobid3],
|
37
|
+
workflow_id: 'id1',
|
38
|
+
params: { 'param1' => true, 'arg' => :arg1 })
|
39
39
|
|
40
40
|
expect(flow[$jobid2]).to include(:id,
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
41
|
+
klass: BuilderJob2.to_s,
|
42
|
+
incoming: [$jobid1, $jobid3],
|
43
|
+
outgoing: [],
|
44
|
+
workflow_id: 'id1',
|
45
|
+
params: { 'param2' => true, 'arg' => :arg2 })
|
46
46
|
|
47
47
|
expect(flow[$jobid3]).to include(:id,
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
48
|
+
klass: BuilderJob3.to_s,
|
49
|
+
incoming: [$jobid1],
|
50
|
+
outgoing: [$jobid2, $jobid4],
|
51
|
+
workflow_id: 'id1',
|
52
|
+
params: nil)
|
53
53
|
|
54
54
|
expect(flow[$jobid4]).to include(:id,
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
klass: BuilderJob3.to_s,
|
56
|
+
incoming: [$jobid3],
|
57
|
+
outgoing: [],
|
58
|
+
workflow_id: 'id1',
|
59
|
+
params: nil)
|
60
60
|
end
|
61
|
-
|
62
61
|
end
|
63
62
|
end
|