burstflow 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -3
  3. data/Gemfile.lock +119 -0
  4. data/burstflow.gemspec +10 -6
  5. data/config/database.yml +4 -3
  6. data/db/migrate/20180101000001_create_workflow.rb +1 -0
  7. data/db/schema.rb +13 -7
  8. data/lib/burstflow.rb +11 -0
  9. data/lib/burstflow/job.rb +102 -0
  10. data/lib/burstflow/job/callbacks.rb +55 -0
  11. data/lib/burstflow/job/exception.rb +8 -0
  12. data/lib/burstflow/job/initialization.rb +35 -0
  13. data/lib/{burst → burstflow/job}/model.rb +1 -3
  14. data/lib/burstflow/job/state.rb +125 -0
  15. data/lib/burstflow/manager.rb +123 -0
  16. data/lib/burstflow/railtie.rb +6 -0
  17. data/lib/burstflow/version.rb +3 -0
  18. data/lib/burstflow/worker.rb +59 -0
  19. data/lib/burstflow/workflow.rb +207 -0
  20. data/lib/burstflow/workflow/builder.rb +91 -0
  21. data/lib/burstflow/workflow/callbacks.rb +66 -0
  22. data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
  23. data/lib/burstflow/workflow/exception.rb +8 -0
  24. data/lib/generators/burstflow/install/install_generator.rb +22 -0
  25. data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
  26. data/spec/builder_spec.rb +63 -0
  27. data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
  28. data/spec/generators/install_generator_spec.rb +27 -0
  29. data/spec/job_spec.rb +18 -8
  30. data/spec/spec_helper.rb +4 -1
  31. data/spec/support/database_clean.rb +4 -1
  32. data/spec/workflow_spec.rb +397 -147
  33. metadata +45 -21
  34. data/db/migrate/20180101000001_create_workflow.rb +0 -13
  35. data/db/seeds.rb +0 -1
  36. data/lib/burst.rb +0 -37
  37. data/lib/burst/builder.rb +0 -48
  38. data/lib/burst/configuration.rb +0 -27
  39. data/lib/burst/job.rb +0 -187
  40. data/lib/burst/manager.rb +0 -79
  41. data/lib/burst/worker.rb +0 -42
  42. data/lib/burst/workflow.rb +0 -148
  43. data/spec/cases_spec.rb +0 -180
@@ -1,42 +0,0 @@
1
- class Burst::Worker < ::ActiveJob::Base
2
-
3
- def perform(workflow_id, job_id, resume_data = nil)
4
- setup(workflow_id, job_id)
5
-
6
- job.payloads = incoming_payloads
7
-
8
- result = if resume_data.nil?
9
- @manager.start_job!(job)
10
- else
11
- @manager.resume_job!(job, resume_data)
12
- end
13
-
14
- @manager.job_performed!(job, result)
15
- rescue StandardError => e
16
- @manager.fail_job!(job)
17
- raise e
18
- end
19
-
20
-
21
- private
22
-
23
- attr_reader :workflow, :job
24
-
25
- def setup(workflow_id, job_id)
26
- @workflow = Burst::Workflow.find(workflow_id)
27
- @job = @workflow.get_job(job_id)
28
- @manager = @workflow.manager
29
- end
30
-
31
- def incoming_payloads
32
- job.incoming.map do |job_id|
33
- incoming = workflow.get_job(job_id)
34
- {
35
- id: incoming.id,
36
- class: incoming.klass.to_s,
37
- payload: incoming.output
38
- }
39
- end
40
- end
41
-
42
- end
@@ -1,148 +0,0 @@
1
- class Burst::Workflow < ActiveRecord::Base
2
-
3
- self.table_name_prefix = 'burst_'
4
-
5
- INITIAL = 'initial'.freeze
6
- RUNNING = 'running'.freeze
7
- FINISHED = 'finished'.freeze
8
- FAILED = 'failed'.freeze
9
- SUSPENDED = 'suspended'.freeze
10
-
11
- include Burst::WorkflowHelper
12
- include Burst::Builder
13
-
14
- attr_accessor :manager, :job_cache
15
- define_flow_attributes :jobs, :klass
16
-
17
- after_initialize do
18
- initialize_builder
19
-
20
- @job_cache = {}
21
-
22
- self.id ||= SecureRandom.uuid
23
- self.jobs ||= {}.with_indifferent_access
24
- self.klass ||= self.class.to_s
25
-
26
- @manager = Burst::Manager.new(self)
27
- end
28
-
29
- def attributes
30
- {
31
- id: self.id,
32
- jobs: self.jobs,
33
- klass: self.klass,
34
- status: status
35
- }
36
- end
37
-
38
- def self.build(*args)
39
- wf = new
40
- wf.configure(*args)
41
- wf.resolve_dependencies
42
- wf
43
- end
44
-
45
- def reload(options = nil)
46
- self.job_cache = {}
47
- super
48
- end
49
-
50
- def start!
51
- save!
52
- manager.start
53
- end
54
-
55
- def resume!(job_id, data)
56
- manager.resume!(get_job(job_id), data)
57
- end
58
-
59
- def status
60
- if failed?
61
- FAILED
62
- elsif suspended?
63
- SUSPENDED
64
- elsif running?
65
- RUNNING
66
- elsif finished?
67
- FINISHED
68
- else
69
- INITIAL
70
- end
71
- end
72
-
73
- def initial?
74
- status == INITIAL
75
- end
76
-
77
- def finished?
78
- all_jobs.all?(&:finished?)
79
- end
80
-
81
- def started?
82
- !!started_at
83
- end
84
-
85
- def running?
86
- started? && !finished?
87
- end
88
-
89
- def failed?
90
- all_jobs.any?(&:failed?)
91
- end
92
-
93
- def suspended?
94
- !failed? && all_jobs.any?(&:suspended?)
95
- end
96
-
97
- def all_jobs
98
- Enumerator.new do |y|
99
- jobs.keys.each do |id|
100
- y << get_job(id)
101
- end
102
- end
103
- end
104
-
105
- def get_job(id)
106
- if job = @job_cache[id]
107
- job
108
- else
109
- job = Burst::Job.from_hash(self, jobs[id].deep_dup)
110
- @job_cache[job.id] = job
111
- job
112
- end
113
- end
114
-
115
- def set_job(job)
116
- jobs[job.id] = job.as_json
117
- end
118
-
119
- def initial_jobs
120
- all_jobs.select(&:initial?)
121
- end
122
-
123
- def find_job(id_or_klass)
124
- id = if jobs.key?(id_or_klass)
125
- id_or_klass
126
- else
127
- find_id_by_klass(id_or_klass)
128
- end
129
-
130
- get_job(id)
131
- end
132
-
133
- def get_job_hash(id)
134
- jobs[id]
135
- end
136
-
137
- private
138
-
139
- def find_id_by_klass(klass)
140
- finded = jobs.select do |_, job|
141
- job[:klass].to_s == klass.to_s
142
- end
143
-
144
- raise 'Duplicat job detected' if finded.count > 1
145
- finded.first.second[:id]
146
- end
147
-
148
- end
@@ -1,180 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Burst::Manager do
4
- class JobHandler
5
-
6
- attr_accessor :jobs
7
-
8
- def initialize
9
- @jobs = []
10
- end
11
-
12
- def add(json)
13
- @jobs.push json
14
- end
15
-
16
- def find_job(klass)
17
- @jobs.detect do |json|
18
- json['klass'].to_s == klass.to_s
19
- end
20
- end
21
-
22
- end
23
-
24
- around :each do |ex|
25
- begin
26
- $job_handler = JobHandler.new
27
- ex.run
28
- ensure
29
- $job_handler = nil
30
- end
31
- end
32
-
33
- class TestCaseJob < Burst::Job
34
-
35
- def perform
36
- # puts "#{self.class.to_s}"
37
- set_output(self.class.to_s)
38
- $job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
39
- end
40
-
41
- end
42
-
43
- class CaseJob1 < TestCaseJob; end
44
- class CaseJob2 < TestCaseJob; end
45
- class CaseJob3 < TestCaseJob; end
46
- class CaseAsyncJob < TestCaseJob
47
-
48
- def perform
49
- suspend
50
- $job_handler.add as_json.with_indifferent_access.merge(payloads: payloads)
51
- end
52
-
53
- end
54
-
55
- class CaseW1 < Burst::Workflow
56
-
57
- configure do |*_args|
58
- id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
59
- run CaseJob2, id: 'job2', after: id1, params: { p2: 2 }
60
- run CaseJob3, after: [CaseJob1, 'job2']
61
- end
62
-
63
- end
64
-
65
-
66
- it 'case1 no async job state' do
67
- w = CaseW1.build
68
-
69
- perform_enqueued_jobs do
70
- w.start!
71
- end
72
-
73
- w.reload
74
-
75
- expect(w.started?).to eq true
76
- expect(w.failed?).to eq false
77
- expect(w.finished?).to eq true
78
- expect(w.running?).to eq false
79
- expect(w.status).to eq Burst::Workflow::FINISHED
80
-
81
- expect($job_handler.jobs.count).to eq 3
82
- expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
83
-
84
- expect($job_handler.find_job(CaseJob2)).to include(output: 'CaseJob2', params: { p2: 2 })
85
- expect($job_handler.find_job(CaseJob2)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
86
-
87
- expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
88
- expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
89
- expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseJob2', payload: 'CaseJob2')
90
- end
91
-
92
- class CaseW2 < Burst::Workflow
93
-
94
- configure do |*_args|
95
- id1 = run CaseJob1, id: 'job1', params: { p1: 1 }
96
- run CaseAsyncJob, id: 'job2', after: id1, params: { p2: 2 }
97
- run CaseJob3, after: [CaseJob1, 'job2']
98
- end
99
-
100
- end
101
-
102
- it 'case2 with async job state' do
103
- w = CaseW2.build
104
-
105
- perform_enqueued_jobs do
106
- w.start!
107
- end
108
-
109
- w.reload
110
-
111
- expect(w.started?).to eq true
112
- expect(w.failed?).to eq false
113
- expect(w.finished?).to eq false
114
- expect(w.running?).to eq true
115
- expect(w.suspended?).to eq true
116
- expect(w.status).to eq Burst::Workflow::SUSPENDED
117
-
118
- expect($job_handler.jobs.count).to eq 2
119
- expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
120
-
121
- expect($job_handler.find_job(CaseAsyncJob)).to include(output: Burst::Job::SUSPEND, params: { p2: 2 })
122
- expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
123
-
124
- w = CaseW2.find(w.id)
125
-
126
- perform_enqueued_jobs do
127
- w.resume!('job2', 'result')
128
- end
129
-
130
- w.reload
131
-
132
- expect($job_handler.jobs.count).to eq 3
133
- expect($job_handler.find_job(CaseJob1)).to include(output: 'CaseJob1', params: { p1: 1 })
134
-
135
- # expect($job_handler.find_job(AsyncJob)).to include(output: 'result', params: {p2: 2})
136
- expect($job_handler.find_job(CaseAsyncJob)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
137
-
138
- expect($job_handler.find_job(CaseJob3)).to include(output: 'CaseJob3')
139
- expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job1', class: 'CaseJob1', payload: 'CaseJob1')
140
- expect($job_handler.find_job(CaseJob3)[:payloads]).to include(id: 'job2', class: 'CaseAsyncJob', payload: 'result')
141
- end
142
-
143
- describe 'dynamic jobs with payloads' do
144
- class DynJob1 < TestCaseJob
145
-
146
- def perform
147
- configure do
148
- run DynJob2, params: { a: 1 }
149
- run DynJob2, params: { a: 2 }
150
- run DynJob2, params: { a: 3 }
151
- end
152
- super
153
- end
154
-
155
- end
156
-
157
- class DynJob2 < TestCaseJob; end
158
- class DynJob3 < TestCaseJob; end
159
-
160
- class DynFlow1 < Burst::Workflow
161
-
162
- configure do |*_args|
163
- run DynJob1
164
- run DynJob3, after: DynJob1
165
- end
166
-
167
- end
168
-
169
- it 'create intemediate jobs' do
170
- w = DynFlow1.build
171
-
172
- perform_enqueued_jobs do
173
- w.start!
174
- end
175
-
176
- expect($job_handler.jobs.count).to eq 5
177
- expect($job_handler.jobs.last['payloads'].count).to eq 4
178
- end
179
- end
180
- end