burstflow 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +1 -3
- data/Gemfile.lock +119 -0
- data/burstflow.gemspec +10 -6
- data/config/database.yml +4 -3
- data/db/migrate/20180101000001_create_workflow.rb +1 -0
- data/db/schema.rb +13 -7
- data/lib/burstflow.rb +11 -0
- data/lib/burstflow/job.rb +102 -0
- data/lib/burstflow/job/callbacks.rb +55 -0
- data/lib/burstflow/job/exception.rb +8 -0
- data/lib/burstflow/job/initialization.rb +35 -0
- data/lib/{burst → burstflow/job}/model.rb +1 -3
- data/lib/burstflow/job/state.rb +125 -0
- data/lib/burstflow/manager.rb +123 -0
- data/lib/burstflow/railtie.rb +6 -0
- data/lib/burstflow/version.rb +3 -0
- data/lib/burstflow/worker.rb +59 -0
- data/lib/burstflow/workflow.rb +207 -0
- data/lib/burstflow/workflow/builder.rb +91 -0
- data/lib/burstflow/workflow/callbacks.rb +66 -0
- data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
- data/lib/burstflow/workflow/exception.rb +8 -0
- data/lib/generators/burstflow/install/install_generator.rb +22 -0
- data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
- data/spec/builder_spec.rb +63 -0
- data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
- data/spec/generators/install_generator_spec.rb +27 -0
- data/spec/job_spec.rb +18 -8
- data/spec/spec_helper.rb +4 -1
- data/spec/support/database_clean.rb +4 -1
- data/spec/workflow_spec.rb +397 -147
- metadata +45 -21
- data/db/migrate/20180101000001_create_workflow.rb +0 -13
- data/db/seeds.rb +0 -1
- data/lib/burst.rb +0 -37
- data/lib/burst/builder.rb +0 -48
- data/lib/burst/configuration.rb +0 -27
- data/lib/burst/job.rb +0 -187
- data/lib/burst/manager.rb +0 -79
- data/lib/burst/worker.rb +0 -42
- data/lib/burst/workflow.rb +0 -148
- data/spec/cases_spec.rb +0 -180
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burstflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samoilenko Yuri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-02-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|
@@ -80,7 +80,21 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
-
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: generator_spec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob.
|
84
98
|
It has dependency, result pipelining and suspend/resume ability
|
85
99
|
email:
|
86
100
|
- kinnalru@gmail.com
|
@@ -93,30 +107,40 @@ files:
|
|
93
107
|
- ".rubocop.yml"
|
94
108
|
- ".travis.yml"
|
95
109
|
- Gemfile
|
110
|
+
- Gemfile.lock
|
96
111
|
- README.md
|
97
112
|
- Rakefile
|
98
113
|
- burstflow.gemspec
|
99
114
|
- config/database.yml
|
100
115
|
- db/migrate/20180101000001_create_workflow.rb
|
101
116
|
- db/schema.rb
|
102
|
-
-
|
103
|
-
- lib/
|
104
|
-
- lib/
|
105
|
-
- lib/
|
106
|
-
- lib/
|
107
|
-
- lib/
|
108
|
-
- lib/
|
109
|
-
- lib/
|
110
|
-
- lib/
|
111
|
-
- lib/
|
112
|
-
-
|
113
|
-
-
|
117
|
+
- lib/burstflow.rb
|
118
|
+
- lib/burstflow/job.rb
|
119
|
+
- lib/burstflow/job/callbacks.rb
|
120
|
+
- lib/burstflow/job/exception.rb
|
121
|
+
- lib/burstflow/job/initialization.rb
|
122
|
+
- lib/burstflow/job/model.rb
|
123
|
+
- lib/burstflow/job/state.rb
|
124
|
+
- lib/burstflow/manager.rb
|
125
|
+
- lib/burstflow/railtie.rb
|
126
|
+
- lib/burstflow/version.rb
|
127
|
+
- lib/burstflow/worker.rb
|
128
|
+
- lib/burstflow/workflow.rb
|
129
|
+
- lib/burstflow/workflow/builder.rb
|
130
|
+
- lib/burstflow/workflow/callbacks.rb
|
131
|
+
- lib/burstflow/workflow/configuration.rb
|
132
|
+
- lib/burstflow/workflow/exception.rb
|
133
|
+
- lib/generators/burstflow/install/install_generator.rb
|
134
|
+
- lib/generators/burstflow/install/templates/create_workflow.rb
|
135
|
+
- spec/builder_spec.rb
|
136
|
+
- spec/burstflow_spec.rb
|
137
|
+
- spec/generators/install_generator_spec.rb
|
114
138
|
- spec/job_spec.rb
|
115
139
|
- spec/spec_helper.rb
|
116
140
|
- spec/support/database_clean.rb
|
117
141
|
- spec/support/runner.rb
|
118
142
|
- spec/workflow_spec.rb
|
119
|
-
homepage: https://github.com/RnD-Soft/
|
143
|
+
homepage: https://github.com/RnD-Soft/burstflow
|
120
144
|
licenses:
|
121
145
|
- MIT
|
122
146
|
metadata: {}
|
@@ -135,14 +159,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
135
159
|
- !ruby/object:Gem::Version
|
136
160
|
version: '0'
|
137
161
|
requirements: []
|
138
|
-
|
139
|
-
rubygems_version: 2.6.14
|
162
|
+
rubygems_version: 3.0.2
|
140
163
|
signing_key:
|
141
164
|
specification_version: 4
|
142
|
-
summary:
|
165
|
+
summary: Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob
|
143
166
|
test_files:
|
144
|
-
- spec/
|
145
|
-
- spec/
|
167
|
+
- spec/builder_spec.rb
|
168
|
+
- spec/burstflow_spec.rb
|
169
|
+
- spec/generators/install_generator_spec.rb
|
146
170
|
- spec/job_spec.rb
|
147
171
|
- spec/spec_helper.rb
|
148
172
|
- spec/support/database_clean.rb
|
data/db/seeds.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
# require_relative "../_import.rb"
|
data/lib/burst.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'bundler'
|
3
|
-
require 'bundler/setup'
|
4
|
-
Bundler.require(:default)
|
5
|
-
|
6
|
-
require 'active_support/all'
|
7
|
-
require 'active_support/dependencies'
|
8
|
-
require 'active_record'
|
9
|
-
require 'active_job'
|
10
|
-
|
11
|
-
require 'pathname'
|
12
|
-
require 'securerandom'
|
13
|
-
|
14
|
-
require 'burst/configuration'
|
15
|
-
require 'burst/model'
|
16
|
-
require 'burst/builder'
|
17
|
-
require 'burst/manager'
|
18
|
-
require 'burst/job'
|
19
|
-
require 'burst/workflow_helper'
|
20
|
-
require 'burst/workflow'
|
21
|
-
require 'burst/worker'
|
22
|
-
|
23
|
-
module Burst
|
24
|
-
|
25
|
-
def self.root
|
26
|
-
Pathname.new(__FILE__).parent.parent
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.configuration
|
30
|
-
@configuration ||= Configuration.new
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.configure
|
34
|
-
yield configuration
|
35
|
-
end
|
36
|
-
|
37
|
-
end
|
data/lib/burst/builder.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
module Burst::Builder
|
2
|
-
|
3
|
-
extend ActiveSupport::Concern
|
4
|
-
|
5
|
-
included do |_klass|
|
6
|
-
attr_accessor :build_deps
|
7
|
-
|
8
|
-
def initialize_builder
|
9
|
-
@build_deps = []
|
10
|
-
end
|
11
|
-
|
12
|
-
def run(klass, opts = {})
|
13
|
-
opts = opts.with_indifferent_access
|
14
|
-
|
15
|
-
before_deps = opts.delete(:before) || []
|
16
|
-
after_deps = opts.delete(:after) || []
|
17
|
-
|
18
|
-
job = klass.new(self, opts)
|
19
|
-
|
20
|
-
[*before_deps].each do |dep|
|
21
|
-
build_deps << { from: job.id, to: dep.to_s }
|
22
|
-
end
|
23
|
-
|
24
|
-
[*after_deps].each do |dep|
|
25
|
-
build_deps << { from: dep.to_s, to: job.id }
|
26
|
-
end
|
27
|
-
|
28
|
-
job_cache[job.id] = job
|
29
|
-
jobs[job.id] = job.model
|
30
|
-
|
31
|
-
job.id
|
32
|
-
end
|
33
|
-
|
34
|
-
def resolve_dependencies
|
35
|
-
build_deps.each do |dependency|
|
36
|
-
from = find_job(dependency[:from])
|
37
|
-
to = find_job(dependency[:to])
|
38
|
-
|
39
|
-
to.incoming << from.id
|
40
|
-
from.outgoing << to.id
|
41
|
-
|
42
|
-
to.incoming.uniq!
|
43
|
-
from.outgoing.uniq!
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
end
|
data/lib/burst/configuration.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
module Burst
|
2
|
-
|
3
|
-
class Configuration
|
4
|
-
|
5
|
-
attr_accessor :concurrency
|
6
|
-
|
7
|
-
def self.from_json(json)
|
8
|
-
new(Burst::JSON.decode(json, symbolize_keys: true))
|
9
|
-
end
|
10
|
-
|
11
|
-
def initialize(hash = {})
|
12
|
-
self.concurrency = hash.fetch(:concurrency, 5)
|
13
|
-
end
|
14
|
-
|
15
|
-
def to_hash
|
16
|
-
{
|
17
|
-
concurrency: concurrency
|
18
|
-
}
|
19
|
-
end
|
20
|
-
|
21
|
-
def to_json
|
22
|
-
Burst::JSON.encode(to_hash)
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
data/lib/burst/job.rb
DELETED
@@ -1,187 +0,0 @@
|
|
1
|
-
class Burst::Job
|
2
|
-
|
3
|
-
include Burst::Model
|
4
|
-
|
5
|
-
define_stored_attributes :id, :workflow_id, :klass, :params, :incoming, :outgoing, :payloads, :output
|
6
|
-
define_stored_attributes :enqueued_at, :started_at, :finished_at, :failed_at, :suspended_at, :resumed_at
|
7
|
-
|
8
|
-
SUSPEND = 'suspend'.freeze
|
9
|
-
|
10
|
-
class Error < ::RuntimeError; end
|
11
|
-
|
12
|
-
def initialize(workflow, hash_store = {})
|
13
|
-
@workflow = workflow
|
14
|
-
assign_default_values(hash_store)
|
15
|
-
end
|
16
|
-
|
17
|
-
def assign_default_values(hash_store)
|
18
|
-
set_model(hash_store.deep_dup)
|
19
|
-
|
20
|
-
self.id ||= SecureRandom.uuid
|
21
|
-
self.workflow_id ||= @workflow.id
|
22
|
-
self.klass ||= self.class.to_s
|
23
|
-
self.incoming ||= []
|
24
|
-
self.outgoing ||= []
|
25
|
-
end
|
26
|
-
|
27
|
-
def reload
|
28
|
-
assign_default_values(@workflow.get_job_hash(self.id))
|
29
|
-
end
|
30
|
-
|
31
|
-
def save!
|
32
|
-
@workflow.with_lock do
|
33
|
-
@workflow.set_job(self)
|
34
|
-
@workflow.save!
|
35
|
-
yield if block_given?
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def self.from_hash(workflow, hash_store)
|
40
|
-
hash_store[:klass].constantize.new(workflow, hash_store)
|
41
|
-
end
|
42
|
-
|
43
|
-
# execute this code by ActiveJob. You may return Burst::Job::SUSPEND to suspend job, or call suspend method
|
44
|
-
def perform; end
|
45
|
-
|
46
|
-
# execute this code when resumes after suspending
|
47
|
-
def resume(data)
|
48
|
-
set_output(data)
|
49
|
-
end
|
50
|
-
|
51
|
-
# store data to be available for next jobs
|
52
|
-
def set_output(data)
|
53
|
-
self.output = data
|
54
|
-
end
|
55
|
-
|
56
|
-
# mark execution as suspended
|
57
|
-
def suspend
|
58
|
-
set_output(SUSPEND)
|
59
|
-
end
|
60
|
-
|
61
|
-
def configure
|
62
|
-
@workflow.with_lock do
|
63
|
-
yield
|
64
|
-
@workflow.resolve_dependencies
|
65
|
-
@workflow.save!
|
66
|
-
@workflow.all_jobs.to_a.each(&:save!)
|
67
|
-
reload
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def run(klass, opts = {})
|
72
|
-
opts[:after] = [*opts[:after], self.id].uniq
|
73
|
-
opts[:before] = [*opts[:before], *self.outgoing].uniq
|
74
|
-
@workflow.run(klass, opts)
|
75
|
-
end
|
76
|
-
|
77
|
-
def attributes
|
78
|
-
{
|
79
|
-
workflow_id: self.workflow_id,
|
80
|
-
id: self.id,
|
81
|
-
klass: self.klass,
|
82
|
-
params: params,
|
83
|
-
incoming: self.incoming,
|
84
|
-
outgoing: self.outgoing,
|
85
|
-
output: output,
|
86
|
-
started_at: started_at,
|
87
|
-
enqueued_at: enqueued_at,
|
88
|
-
finished_at: finished_at,
|
89
|
-
failed_at: failed_at,
|
90
|
-
suspended_at: suspended_at,
|
91
|
-
resumed_at: resumed_at
|
92
|
-
}
|
93
|
-
end
|
94
|
-
|
95
|
-
# mark job as enqueued when it is scheduled to queue
|
96
|
-
def enqueue!
|
97
|
-
raise Error.new('Already enqueued') if enqueued?
|
98
|
-
self.enqueued_at = current_timestamp
|
99
|
-
self.started_at = nil
|
100
|
-
self.finished_at = nil
|
101
|
-
self.failed_at = nil
|
102
|
-
self.suspended_at = nil
|
103
|
-
self.resumed_at = nil
|
104
|
-
end
|
105
|
-
|
106
|
-
# mark job as started when it is start performing
|
107
|
-
def start!
|
108
|
-
raise Error.new('Already started') if started?
|
109
|
-
self.started_at = current_timestamp
|
110
|
-
end
|
111
|
-
|
112
|
-
# mark job as finished when it is finish performing
|
113
|
-
def finish!
|
114
|
-
raise Error.new('Already finished') if finished?
|
115
|
-
self.finished_at = current_timestamp
|
116
|
-
end
|
117
|
-
|
118
|
-
# mark job as failed when it is failed
|
119
|
-
def fail!
|
120
|
-
raise Error.new('Already failed') if failed?
|
121
|
-
self.finished_at = self.failed_at = current_timestamp
|
122
|
-
end
|
123
|
-
|
124
|
-
# mark job as suspended
|
125
|
-
def suspend!
|
126
|
-
self.suspended_at = current_timestamp
|
127
|
-
end
|
128
|
-
|
129
|
-
# mark job as resumed
|
130
|
-
def resume!
|
131
|
-
raise Error.new('Not suspended ') unless suspended?
|
132
|
-
raise Error.new('Already resumed ') if resumed?
|
133
|
-
self.resumed_at = current_timestamp
|
134
|
-
end
|
135
|
-
|
136
|
-
def enqueued?
|
137
|
-
!enqueued_at.nil?
|
138
|
-
end
|
139
|
-
|
140
|
-
def started?
|
141
|
-
!started_at.nil?
|
142
|
-
end
|
143
|
-
|
144
|
-
def finished?
|
145
|
-
!finished_at.nil?
|
146
|
-
end
|
147
|
-
|
148
|
-
def running?
|
149
|
-
started? && !finished? && !suspended?
|
150
|
-
end
|
151
|
-
|
152
|
-
def failed?
|
153
|
-
!failed_at.nil?
|
154
|
-
end
|
155
|
-
|
156
|
-
def suspended?
|
157
|
-
!suspended_at.nil? && !resumed?
|
158
|
-
end
|
159
|
-
|
160
|
-
def resumed?
|
161
|
-
!resumed_at.nil?
|
162
|
-
end
|
163
|
-
|
164
|
-
def succeeded?
|
165
|
-
finished? && !failed?
|
166
|
-
end
|
167
|
-
|
168
|
-
def ready_to_start?
|
169
|
-
!running? && !enqueued? && !finished? && !failed? && parents_succeeded?
|
170
|
-
end
|
171
|
-
|
172
|
-
def initial?
|
173
|
-
incoming.empty?
|
174
|
-
end
|
175
|
-
|
176
|
-
def current_timestamp
|
177
|
-
Time.now.to_i
|
178
|
-
end
|
179
|
-
|
180
|
-
def parents_succeeded?
|
181
|
-
incoming.all? do |id|
|
182
|
-
@workflow.get_job(id).succeeded?
|
183
|
-
end
|
184
|
-
end
|
185
|
-
|
186
|
-
|
187
|
-
end
|
data/lib/burst/manager.rb
DELETED
@@ -1,79 +0,0 @@
|
|
1
|
-
class Burst::Manager
|
2
|
-
|
3
|
-
attr_accessor :workflow
|
4
|
-
|
5
|
-
def initialize(workflow)
|
6
|
-
@workflow = workflow
|
7
|
-
end
|
8
|
-
|
9
|
-
def start
|
10
|
-
workflow.with_lock do
|
11
|
-
raise 'Already started' unless workflow.initial?
|
12
|
-
|
13
|
-
workflow.initial_jobs.each do |job|
|
14
|
-
enqueue_job!(job)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def enqueue_job!(job)
|
20
|
-
job.enqueue!
|
21
|
-
job.save! do
|
22
|
-
Burst::Worker.perform_later(workflow.id, job.id)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def resume!(job, data)
|
27
|
-
job.resume!
|
28
|
-
job.save! do
|
29
|
-
Burst::Worker.perform_later(workflow.id, job.id, data)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def start_job!(job)
|
34
|
-
job.start!
|
35
|
-
job.save!
|
36
|
-
|
37
|
-
job.perform
|
38
|
-
end
|
39
|
-
|
40
|
-
def resume_job!(job, data)
|
41
|
-
job.resume(data)
|
42
|
-
end
|
43
|
-
|
44
|
-
def suspend_job!(job)
|
45
|
-
job.suspend!
|
46
|
-
job.save!
|
47
|
-
end
|
48
|
-
|
49
|
-
def finish_job!(job)
|
50
|
-
job.finish!
|
51
|
-
job.save!
|
52
|
-
|
53
|
-
workflow.with_lock do
|
54
|
-
enqueue_outgoing_jobs(job)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def job_performed!(job, result)
|
59
|
-
if result == Burst::Job::SUSPEND || job.output == Burst::Job::SUSPEND
|
60
|
-
suspend_job!(job)
|
61
|
-
else
|
62
|
-
finish_job!(job)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def fail_job!(job)
|
67
|
-
job.fail!
|
68
|
-
job.save!
|
69
|
-
end
|
70
|
-
|
71
|
-
def enqueue_outgoing_jobs(job)
|
72
|
-
job.outgoing.each do |job_id|
|
73
|
-
out = workflow.get_job(job_id)
|
74
|
-
|
75
|
-
enqueue_job!(out) if out.ready_to_start?
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
end
|