burstflow 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -3
- data/Gemfile.lock +119 -0
- data/burstflow.gemspec +10 -6
- data/config/database.yml +4 -3
- data/db/migrate/20180101000001_create_workflow.rb +1 -0
- data/db/schema.rb +13 -7
- data/lib/burstflow.rb +11 -0
- data/lib/burstflow/job.rb +102 -0
- data/lib/burstflow/job/callbacks.rb +55 -0
- data/lib/burstflow/job/exception.rb +8 -0
- data/lib/burstflow/job/initialization.rb +35 -0
- data/lib/{burst → burstflow/job}/model.rb +1 -3
- data/lib/burstflow/job/state.rb +125 -0
- data/lib/burstflow/manager.rb +123 -0
- data/lib/burstflow/railtie.rb +6 -0
- data/lib/burstflow/version.rb +3 -0
- data/lib/burstflow/worker.rb +59 -0
- data/lib/burstflow/workflow.rb +207 -0
- data/lib/burstflow/workflow/builder.rb +91 -0
- data/lib/burstflow/workflow/callbacks.rb +66 -0
- data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
- data/lib/burstflow/workflow/exception.rb +8 -0
- data/lib/generators/burstflow/install/install_generator.rb +22 -0
- data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
- data/spec/builder_spec.rb +63 -0
- data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
- data/spec/generators/install_generator_spec.rb +27 -0
- data/spec/job_spec.rb +18 -8
- data/spec/spec_helper.rb +4 -1
- data/spec/support/database_clean.rb +4 -1
- data/spec/workflow_spec.rb +397 -147
- metadata +45 -21
- data/db/migrate/20180101000001_create_workflow.rb +0 -13
- data/db/seeds.rb +0 -1
- data/lib/burst.rb +0 -37
- data/lib/burst/builder.rb +0 -48
- data/lib/burst/configuration.rb +0 -27
- data/lib/burst/job.rb +0 -187
- data/lib/burst/manager.rb +0 -79
- data/lib/burst/worker.rb +0 -42
- data/lib/burst/workflow.rb +0 -148
- data/spec/cases_spec.rb +0 -180
data/lib/burst/worker.rb
DELETED
@@ -1,42 +0,0 @@
|
|
1
|
-
class Burst::Worker < ::ActiveJob::Base
|
2
|
-
|
3
|
-
def perform(workflow_id, job_id, resume_data = nil)
|
4
|
-
setup(workflow_id, job_id)
|
5
|
-
|
6
|
-
job.payloads = incoming_payloads
|
7
|
-
|
8
|
-
result = if resume_data.nil?
|
9
|
-
@manager.start_job!(job)
|
10
|
-
else
|
11
|
-
@manager.resume_job!(job, resume_data)
|
12
|
-
end
|
13
|
-
|
14
|
-
@manager.job_performed!(job, result)
|
15
|
-
rescue StandardError => e
|
16
|
-
@manager.fail_job!(job)
|
17
|
-
raise e
|
18
|
-
end
|
19
|
-
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
attr_reader :workflow, :job
|
24
|
-
|
25
|
-
def setup(workflow_id, job_id)
|
26
|
-
@workflow = Burst::Workflow.find(workflow_id)
|
27
|
-
@job = @workflow.get_job(job_id)
|
28
|
-
@manager = @workflow.manager
|
29
|
-
end
|
30
|
-
|
31
|
-
def incoming_payloads
|
32
|
-
job.incoming.map do |job_id|
|
33
|
-
incoming = workflow.get_job(job_id)
|
34
|
-
{
|
35
|
-
id: incoming.id,
|
36
|
-
class: incoming.klass.to_s,
|
37
|
-
payload: incoming.output
|
38
|
-
}
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
data/lib/burst/workflow.rb
DELETED
@@ -1,148 +0,0 @@
|
|
1
|
-
class Burst::Workflow < ActiveRecord::Base
|
2
|
-
|
3
|
-
self.table_name_prefix = 'burst_'
|
4
|
-
|
5
|
-
INITIAL = 'initial'.freeze
|
6
|
-
RUNNING = 'running'.freeze
|
7
|
-
FINISHED = 'finished'.freeze
|
8
|
-
FAILED = 'failed'.freeze
|
9
|
-
SUSPENDED = 'suspended'.freeze
|
10
|
-
|
11
|
-
include Burst::WorkflowHelper
|
12
|
-
include Burst::Builder
|
13
|
-
|
14
|
-
attr_accessor :manager, :job_cache
|
15
|
-
define_flow_attributes :jobs, :klass
|
16
|
-
|
17
|
-
after_initialize do
|
18
|
-
initialize_builder
|
19
|
-
|
20
|
-
@job_cache = {}
|
21
|
-
|
22
|
-
self.id ||= SecureRandom.uuid
|
23
|
-
self.jobs ||= {}.with_indifferent_access
|
24
|
-
self.klass ||= self.class.to_s
|
25
|
-
|
26
|
-
@manager = Burst::Manager.new(self)
|
27
|
-
end
|
28
|
-
|
29
|
-
def attributes
|
30
|
-
{
|
31
|
-
id: self.id,
|
32
|
-
jobs: self.jobs,
|
33
|
-
klass: self.klass,
|
34
|
-
status: status
|
35
|
-
}
|
36
|
-
end
|
37
|
-
|
38
|
-
def self.build(*args)
|
39
|
-
wf = new
|
40
|
-
wf.configure(*args)
|
41
|
-
wf.resolve_dependencies
|
42
|
-
wf
|
43
|
-
end
|
44
|
-
|
45
|
-
def reload(options = nil)
|
46
|
-
self.job_cache = {}
|
47
|
-
super
|
48
|
-
end
|
49
|
-
|
50
|
-
def start!
|
51
|
-
save!
|
52
|
-
manager.start
|
53
|
-
end
|
54
|
-
|
55
|
-
def resume!(job_id, data)
|
56
|
-
manager.resume!(get_job(job_id), data)
|
57
|
-
end
|
58
|
-
|
59
|
-
def status
|
60
|
-
if failed?
|
61
|
-
FAILED
|
62
|
-
elsif suspended?
|
63
|
-
SUSPENDED
|
64
|
-
elsif running?
|
65
|
-
RUNNING
|
66
|
-
elsif finished?
|
67
|
-
FINISHED
|
68
|
-
else
|
69
|
-
INITIAL
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def initial?
|
74
|
-
status == INITIAL
|
75
|
-
end
|
76
|
-
|
77
|
-
def finished?
|
78
|
-
all_jobs.all?(&:finished?)
|
79
|
-
end
|
80
|
-
|
81
|
-
def started?
|
82
|
-
!!started_at
|
83
|
-
end
|
84
|
-
|
85
|
-
def running?
|
86
|
-
started? && !finished?
|
87
|
-
end
|
88
|
-
|
89
|
-
def failed?
|
90
|
-
all_jobs.any?(&:failed?)
|
91
|
-
end
|
92
|
-
|
93
|
-
def suspended?
|
94
|
-
!failed? && all_jobs.any?(&:suspended?)
|
95
|
-
end
|
96
|
-
|
97
|
-
def all_jobs
|
98
|
-
Enumerator.new do |y|
|
99
|
-
jobs.keys.each do |id|
|
100
|
-
y << get_job(id)
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
def get_job(id)
|
106
|
-
if job = @job_cache[id]
|
107
|
-
job
|
108
|
-
else
|
109
|
-
job = Burst::Job.from_hash(self, jobs[id].deep_dup)
|
110
|
-
@job_cache[job.id] = job
|
111
|
-
job
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
def set_job(job)
|
116
|
-
jobs[job.id] = job.as_json
|
117
|
-
end
|
118
|
-
|
119
|
-
def initial_jobs
|
120
|
-
all_jobs.select(&:initial?)
|
121
|
-
end
|
122
|
-
|
123
|
-
def find_job(id_or_klass)
|
124
|
-
id = if jobs.key?(id_or_klass)
|
125
|
-
id_or_klass
|
126
|
-
else
|
127
|
-
find_id_by_klass(id_or_klass)
|
128
|
-
end
|
129
|
-
|
130
|
-
get_job(id)
|
131
|
-
end
|
132
|
-
|
133
|
-
def get_job_hash(id)
|
134
|
-
jobs[id]
|
135
|
-
end
|
136
|
-
|
137
|
-
private
|
138
|
-
|
139
|
-
def find_id_by_klass(klass)
|
140
|
-
finded = jobs.select do |_, job|
|
141
|
-
job[:klass].to_s == klass.to_s
|
142
|
-
end
|
143
|
-
|
144
|
-
raise 'Duplicat job detected' if finded.count > 1
|
145
|
-
finded.first.second[:id]
|
146
|
-
end
|
147
|
-
|
148
|
-
end
|
data/spec/cases_spec.rb
DELETED
@@ -1,180 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Burst::Manager do
|
4
|
-
class JobHandler
|
5
|
-
|
6
|
-
attr_accessor :jobs
|
7
|
-
|
8
|
-
def initialize
|
9
|
-
@jobs = []
|
10
|
-
end
|
11
|
-
|
12
|
-
def add(json)
|
13
|
-
@jobs.push json
|
14
|
-
end
|
15
|
-
|
16
|
-
def find_job(klass)
|
17
|
-
@jobs.detect do |json|
|
18
|
-
json['klass'].to_s == klass.to_s
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
around :each do |ex|
|
25
|
-
begin
|
26
|
-
$job_handler = JobHandler.new
|
27
|
-
ex.run
|
28
|
-
ensure
|
29
|
-
$job_handler = nil
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
class TestCaseJob < Burst::Job
|
34
|
-
|
35
|
-
def perform
|
36
|
-
# puts "#{self.class.to_s}"
|
37
|
-
set_output(self.class.to_s)
|
38
|
-
$job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
class CaseJob1 < TestCaseJob; end
|
44
|
-
class CaseJob2 < TestCaseJob; end
|
45
|
-
class CaseJob3 < TestCaseJob; end
|
46
|
-
class CaseAsyncJob < TestCaseJob
|
47
|
-
|
48
|
-
def perform
|
49
|
-
suspend
|
50
|
-
$job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
54
|
-
|
55
|
-
class CaseW1 < Burst::Workflow
|
56
|
-
|
57
|
-
configure do |*_args|
|
58
|
-
id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
|
59
|
-
run CaseJob2, id: 'job2', after: id1, params: { p2: 2 }
|
60
|
-
run CaseJob3, after: [CaseJob1, 'job2']
|
61
|
-
end
|
62
|
-
|
63
|
-
end
|
64
|
-
|
65
|
-
|
66
|
-
it 'case1 no async job state' do
|
67
|
-
w = CaseW1.build
|
68
|
-
|
69
|
-
perform_enqueued_jobs do
|
70
|
-
w.start!
|
71
|
-
end
|
72
|
-
|
73
|
-
w.reload
|
74
|
-
|
75
|
-
expect(w.started?).to eq true
|
76
|
-
expect(w.failed?).to eq false
|
77
|
-
expect(w.finished?).to eq true
|
78
|
-
expect(w.running?).to eq false
|
79
|
-
expect(w.status).to eq Burst::Workflow::FINISHED
|
80
|
-
|
81
|
-
expect($job_handler.jobs.count).to eq 3
|
82
|
-
expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
|
83
|
-
|
84
|
-
expect($job_handler.find_job(CaseJob2)).to include(output: 'CaseJob2', params: { p2: 2 })
|
85
|
-
expect($job_handler.find_job(CaseJob2)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
|
86
|
-
|
87
|
-
expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
|
88
|
-
expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
|
89
|
-
expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseJob2', payload: 'CaseJob2')
|
90
|
-
end
|
91
|
-
|
92
|
-
class CaseW2 < Burst::Workflow
|
93
|
-
|
94
|
-
configure do |*_args|
|
95
|
-
id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
|
96
|
-
run CaseAsyncJob, id: 'job2', after: id1, params: { p2: 2 }
|
97
|
-
run CaseJob3, after: [CaseJob1, 'job2']
|
98
|
-
end
|
99
|
-
|
100
|
-
end
|
101
|
-
|
102
|
-
it 'case2 with async job state' do
|
103
|
-
w = CaseW2.build
|
104
|
-
|
105
|
-
perform_enqueued_jobs do
|
106
|
-
w.start!
|
107
|
-
end
|
108
|
-
|
109
|
-
w.reload
|
110
|
-
|
111
|
-
expect(w.started?).to eq true
|
112
|
-
expect(w.failed?).to eq false
|
113
|
-
expect(w.finished?).to eq false
|
114
|
-
expect(w.running?).to eq true
|
115
|
-
expect(w.suspended?).to eq true
|
116
|
-
expect(w.status).to eq Burst::Workflow::SUSPENDED
|
117
|
-
|
118
|
-
expect($job_handler.jobs.count).to eq 2
|
119
|
-
expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
|
120
|
-
|
121
|
-
expect($job_handler.find_job(CaseAsyncJob)).to include(output: Burst::Job::SUSPEND, params: { p2: 2 })
|
122
|
-
expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
|
123
|
-
|
124
|
-
w = CaseW2.find(w.id)
|
125
|
-
|
126
|
-
perform_enqueued_jobs do
|
127
|
-
w.resume!('job2', 'result')
|
128
|
-
end
|
129
|
-
|
130
|
-
w.reload
|
131
|
-
|
132
|
-
expect($job_handler.jobs.count).to eq 3
|
133
|
-
expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
|
134
|
-
|
135
|
-
# expect($job_handler.find_job(AsyncJob)).to include(output: 'result', params: {p2: 2})
|
136
|
-
expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
|
137
|
-
|
138
|
-
expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
|
139
|
-
expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
|
140
|
-
expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseAsyncJob', payload: 'result')
|
141
|
-
end
|
142
|
-
|
143
|
-
describe 'dynamic jobs with payloads' do
|
144
|
-
class DynJob1 < TestCaseJob
|
145
|
-
|
146
|
-
def perform
|
147
|
-
configure do
|
148
|
-
run DynJob2, params: { a: 1 }
|
149
|
-
run DynJob2, params: { a: 2 }
|
150
|
-
run DynJob2, params: { a: 3 }
|
151
|
-
end
|
152
|
-
super
|
153
|
-
end
|
154
|
-
|
155
|
-
end
|
156
|
-
|
157
|
-
class DynJob2 < TestCaseJob; end
|
158
|
-
class DynJob3 < TestCaseJob; end
|
159
|
-
|
160
|
-
class DynFlow1 < Burst::Workflow
|
161
|
-
|
162
|
-
configure do |*_args|
|
163
|
-
run DynJob1
|
164
|
-
run DynJob3, after: DynJob1
|
165
|
-
end
|
166
|
-
|
167
|
-
end
|
168
|
-
|
169
|
-
it 'create intemediate jobs' do
|
170
|
-
w = DynFlow1.build
|
171
|
-
|
172
|
-
perform_enqueued_jobs do
|
173
|
-
w.start!
|
174
|
-
end
|
175
|
-
|
176
|
-
expect($job_handler.jobs.count).to eq 5
|
177
|
-
expect($job_handler.jobs.last['payloads'].count).to eq 4
|
178
|
-
end
|
179
|
-
end
|
180
|
-
end
|