burstflow 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -3
  3. data/Gemfile.lock +119 -0
  4. data/burstflow.gemspec +10 -6
  5. data/config/database.yml +4 -3
  6. data/db/migrate/20180101000001_create_workflow.rb +1 -0
  7. data/db/schema.rb +13 -7
  8. data/lib/burstflow.rb +11 -0
  9. data/lib/burstflow/job.rb +102 -0
  10. data/lib/burstflow/job/callbacks.rb +55 -0
  11. data/lib/burstflow/job/exception.rb +8 -0
  12. data/lib/burstflow/job/initialization.rb +35 -0
  13. data/lib/{burst → burstflow/job}/model.rb +1 -3
  14. data/lib/burstflow/job/state.rb +125 -0
  15. data/lib/burstflow/manager.rb +123 -0
  16. data/lib/burstflow/railtie.rb +6 -0
  17. data/lib/burstflow/version.rb +3 -0
  18. data/lib/burstflow/worker.rb +59 -0
  19. data/lib/burstflow/workflow.rb +207 -0
  20. data/lib/burstflow/workflow/builder.rb +91 -0
  21. data/lib/burstflow/workflow/callbacks.rb +66 -0
  22. data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
  23. data/lib/burstflow/workflow/exception.rb +8 -0
  24. data/lib/generators/burstflow/install/install_generator.rb +22 -0
  25. data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
  26. data/spec/builder_spec.rb +63 -0
  27. data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
  28. data/spec/generators/install_generator_spec.rb +27 -0
  29. data/spec/job_spec.rb +18 -8
  30. data/spec/spec_helper.rb +4 -1
  31. data/spec/support/database_clean.rb +4 -1
  32. data/spec/workflow_spec.rb +397 -147
  33. metadata +45 -21
  34. data/db/migrate/20180101000001_create_workflow.rb +0 -13
  35. data/db/seeds.rb +0 -1
  36. data/lib/burst.rb +0 -37
  37. data/lib/burst/builder.rb +0 -48
  38. data/lib/burst/configuration.rb +0 -27
  39. data/lib/burst/job.rb +0 -187
  40. data/lib/burst/manager.rb +0 -79
  41. data/lib/burst/worker.rb +0 -42
  42. data/lib/burst/workflow.rb +0 -148
  43. data/spec/cases_spec.rb +0 -180
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: burstflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Samoilenko Yuri
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-01-02 00:00:00.000000000 Z
11
+ date: 2019-02-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activejob
@@ -80,7 +80,21 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
- description: Burst is a parallel workflow runner using ActiveRecord and ActiveJob.
83
+ - !ruby/object:Gem::Dependency
84
+ name: generator_spec
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob.
84
98
  It has dependency, result pipelining and suspend/resume ability
85
99
  email:
86
100
  - kinnalru@gmail.com
@@ -93,30 +107,40 @@ files:
93
107
  - ".rubocop.yml"
94
108
  - ".travis.yml"
95
109
  - Gemfile
110
+ - Gemfile.lock
96
111
  - README.md
97
112
  - Rakefile
98
113
  - burstflow.gemspec
99
114
  - config/database.yml
100
115
  - db/migrate/20180101000001_create_workflow.rb
101
116
  - db/schema.rb
102
- - db/seeds.rb
103
- - lib/burst.rb
104
- - lib/burst/builder.rb
105
- - lib/burst/configuration.rb
106
- - lib/burst/job.rb
107
- - lib/burst/manager.rb
108
- - lib/burst/model.rb
109
- - lib/burst/worker.rb
110
- - lib/burst/workflow.rb
111
- - lib/burst/workflow_helper.rb
112
- - spec/burst_spec.rb
113
- - spec/cases_spec.rb
117
+ - lib/burstflow.rb
118
+ - lib/burstflow/job.rb
119
+ - lib/burstflow/job/callbacks.rb
120
+ - lib/burstflow/job/exception.rb
121
+ - lib/burstflow/job/initialization.rb
122
+ - lib/burstflow/job/model.rb
123
+ - lib/burstflow/job/state.rb
124
+ - lib/burstflow/manager.rb
125
+ - lib/burstflow/railtie.rb
126
+ - lib/burstflow/version.rb
127
+ - lib/burstflow/worker.rb
128
+ - lib/burstflow/workflow.rb
129
+ - lib/burstflow/workflow/builder.rb
130
+ - lib/burstflow/workflow/callbacks.rb
131
+ - lib/burstflow/workflow/configuration.rb
132
+ - lib/burstflow/workflow/exception.rb
133
+ - lib/generators/burstflow/install/install_generator.rb
134
+ - lib/generators/burstflow/install/templates/create_workflow.rb
135
+ - spec/builder_spec.rb
136
+ - spec/burstflow_spec.rb
137
+ - spec/generators/install_generator_spec.rb
114
138
  - spec/job_spec.rb
115
139
  - spec/spec_helper.rb
116
140
  - spec/support/database_clean.rb
117
141
  - spec/support/runner.rb
118
142
  - spec/workflow_spec.rb
119
- homepage: https://github.com/RnD-Soft/burst
143
+ homepage: https://github.com/RnD-Soft/burstflow
120
144
  licenses:
121
145
  - MIT
122
146
  metadata: {}
@@ -135,14 +159,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
135
159
  - !ruby/object:Gem::Version
136
160
  version: '0'
137
161
  requirements: []
138
- rubyforge_project:
139
- rubygems_version: 2.6.14
162
+ rubygems_version: 3.0.2
140
163
  signing_key:
141
164
  specification_version: 4
142
- summary: Burst is a parallel workflow runner using ActiveRecord and ActiveJob
165
+ summary: Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob
143
166
  test_files:
144
- - spec/burst_spec.rb
145
- - spec/cases_spec.rb
167
+ - spec/builder_spec.rb
168
+ - spec/burstflow_spec.rb
169
+ - spec/generators/install_generator_spec.rb
146
170
  - spec/job_spec.rb
147
171
  - spec/spec_helper.rb
148
172
  - spec/support/database_clean.rb
@@ -1,13 +0,0 @@
1
- class CreateWorkflow < ActiveRecord::Migration[5.1]
2
-
3
- def change
4
- enable_extension 'pgcrypto'
5
-
6
- create_table :burst_workflows, id: :uuid do |t|
7
- t.jsonb :flow, null: false, default: {}
8
-
9
- t.timestamps
10
- end
11
- end
12
-
13
- end
@@ -1 +0,0 @@
1
- # require_relative "../_import.rb"
@@ -1,37 +0,0 @@
1
- require 'rubygems'
2
- require 'bundler'
3
- require 'bundler/setup'
4
- Bundler.require(:default)
5
-
6
- require 'active_support/all'
7
- require 'active_support/dependencies'
8
- require 'active_record'
9
- require 'active_job'
10
-
11
- require 'pathname'
12
- require 'securerandom'
13
-
14
- require 'burst/configuration'
15
- require 'burst/model'
16
- require 'burst/builder'
17
- require 'burst/manager'
18
- require 'burst/job'
19
- require 'burst/workflow_helper'
20
- require 'burst/workflow'
21
- require 'burst/worker'
22
-
23
- module Burst
24
-
25
- def self.root
26
- Pathname.new(__FILE__).parent.parent
27
- end
28
-
29
- def self.configuration
30
- @configuration ||= Configuration.new
31
- end
32
-
33
- def self.configure
34
- yield configuration
35
- end
36
-
37
- end
@@ -1,48 +0,0 @@
1
- module Burst::Builder
2
-
3
- extend ActiveSupport::Concern
4
-
5
- included do |_klass|
6
- attr_accessor :build_deps
7
-
8
- def initialize_builder
9
- @build_deps = []
10
- end
11
-
12
- def run(klass, opts = {})
13
- opts = opts.with_indifferent_access
14
-
15
- before_deps = opts.delete(:before) || []
16
- after_deps = opts.delete(:after) || []
17
-
18
- job = klass.new(self, opts)
19
-
20
- [*before_deps].each do |dep|
21
- build_deps << { from: job.id, to: dep.to_s }
22
- end
23
-
24
- [*after_deps].each do |dep|
25
- build_deps << { from: dep.to_s, to: job.id }
26
- end
27
-
28
- job_cache[job.id] = job
29
- jobs[job.id] = job.model
30
-
31
- job.id
32
- end
33
-
34
- def resolve_dependencies
35
- build_deps.each do |dependency|
36
- from = find_job(dependency[:from])
37
- to = find_job(dependency[:to])
38
-
39
- to.incoming << from.id
40
- from.outgoing << to.id
41
-
42
- to.incoming.uniq!
43
- from.outgoing.uniq!
44
- end
45
- end
46
- end
47
-
48
- end
@@ -1,27 +0,0 @@
1
- module Burst
2
-
3
- class Configuration
4
-
5
- attr_accessor :concurrency
6
-
7
- def self.from_json(json)
8
- new(Burst::JSON.decode(json, symbolize_keys: true))
9
- end
10
-
11
- def initialize(hash = {})
12
- self.concurrency = hash.fetch(:concurrency, 5)
13
- end
14
-
15
- def to_hash
16
- {
17
- concurrency: concurrency
18
- }
19
- end
20
-
21
- def to_json
22
- Burst::JSON.encode(to_hash)
23
- end
24
-
25
- end
26
-
27
- end
@@ -1,187 +0,0 @@
1
- class Burst::Job
2
-
3
- include Burst::Model
4
-
5
- define_stored_attributes :id, :workflow_id, :klass, :params, :incoming, :outgoing, :payloads, :output
6
- define_stored_attributes :enqueued_at, :started_at, :finished_at, :failed_at, :suspended_at, :resumed_at
7
-
8
- SUSPEND = 'suspend'.freeze
9
-
10
- class Error < ::RuntimeError; end
11
-
12
- def initialize(workflow, hash_store = {})
13
- @workflow = workflow
14
- assign_default_values(hash_store)
15
- end
16
-
17
- def assign_default_values(hash_store)
18
- set_model(hash_store.deep_dup)
19
-
20
- self.id ||= SecureRandom.uuid
21
- self.workflow_id ||= @workflow.id
22
- self.klass ||= self.class.to_s
23
- self.incoming ||= []
24
- self.outgoing ||= []
25
- end
26
-
27
- def reload
28
- assign_default_values(@workflow.get_job_hash(self.id))
29
- end
30
-
31
- def save!
32
- @workflow.with_lock do
33
- @workflow.set_job(self)
34
- @workflow.save!
35
- yield if block_given?
36
- end
37
- end
38
-
39
- def self.from_hash(workflow, hash_store)
40
- hash_store[:klass].constantize.new(workflow, hash_store)
41
- end
42
-
43
- # execute this code by ActiveJob. You may return Burst::Job::SUSPEND to suspend job, or call suspend method
44
- def perform; end
45
-
46
- # execute this code when resumes after suspending
47
- def resume(data)
48
- set_output(data)
49
- end
50
-
51
- # store data to be available for next jobs
52
- def set_output(data)
53
- self.output = data
54
- end
55
-
56
- # mark execution as suspended
57
- def suspend
58
- set_output(SUSPEND)
59
- end
60
-
61
- def configure
62
- @workflow.with_lock do
63
- yield
64
- @workflow.resolve_dependencies
65
- @workflow.save!
66
- @workflow.all_jobs.to_a.each(&:save!)
67
- reload
68
- end
69
- end
70
-
71
- def run(klass, opts = {})
72
- opts[:after] = [*opts[:after], self.id].uniq
73
- opts[:before] = [*opts[:before], *self.outgoing].uniq
74
- @workflow.run(klass, opts)
75
- end
76
-
77
- def attributes
78
- {
79
- workflow_id: self.workflow_id,
80
- id: self.id,
81
- klass: self.klass,
82
- params: params,
83
- incoming: self.incoming,
84
- outgoing: self.outgoing,
85
- output: output,
86
- started_at: started_at,
87
- enqueued_at: enqueued_at,
88
- finished_at: finished_at,
89
- failed_at: failed_at,
90
- suspended_at: suspended_at,
91
- resumed_at: resumed_at
92
- }
93
- end
94
-
95
- # mark job as enqueued when it is scheduled to queue
96
- def enqueue!
97
- raise Error.new('Already enqueued') if enqueued?
98
- self.enqueued_at = current_timestamp
99
- self.started_at = nil
100
- self.finished_at = nil
101
- self.failed_at = nil
102
- self.suspended_at = nil
103
- self.resumed_at = nil
104
- end
105
-
106
- # mark job as started when it is start performing
107
- def start!
108
- raise Error.new('Already started') if started?
109
- self.started_at = current_timestamp
110
- end
111
-
112
- # mark job as finished when it is finish performing
113
- def finish!
114
- raise Error.new('Already finished') if finished?
115
- self.finished_at = current_timestamp
116
- end
117
-
118
- # mark job as failed when it is failed
119
- def fail!
120
- raise Error.new('Already failed') if failed?
121
- self.finished_at = self.failed_at = current_timestamp
122
- end
123
-
124
- # mark job as suspended
125
- def suspend!
126
- self.suspended_at = current_timestamp
127
- end
128
-
129
- # mark job as resumed
130
- def resume!
131
- raise Error.new('Not suspended ') unless suspended?
132
- raise Error.new('Already resumed ') if resumed?
133
- self.resumed_at = current_timestamp
134
- end
135
-
136
- def enqueued?
137
- !enqueued_at.nil?
138
- end
139
-
140
- def started?
141
- !started_at.nil?
142
- end
143
-
144
- def finished?
145
- !finished_at.nil?
146
- end
147
-
148
- def running?
149
- started? && !finished? && !suspended?
150
- end
151
-
152
- def failed?
153
- !failed_at.nil?
154
- end
155
-
156
- def suspended?
157
- !suspended_at.nil? && !resumed?
158
- end
159
-
160
- def resumed?
161
- !resumed_at.nil?
162
- end
163
-
164
- def succeeded?
165
- finished? && !failed?
166
- end
167
-
168
- def ready_to_start?
169
- !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
170
- end
171
-
172
- def initial?
173
- incoming.empty?
174
- end
175
-
176
- def current_timestamp
177
- Time.now.to_i
178
- end
179
-
180
- def parents_succeeded?
181
- incoming.all? do |id|
182
- @workflow.get_job(id).succeeded?
183
- end
184
- end
185
-
186
-
187
- end
@@ -1,79 +0,0 @@
1
- class Burst::Manager
2
-
3
- attr_accessor :workflow
4
-
5
- def initialize(workflow)
6
- @workflow = workflow
7
- end
8
-
9
- def start
10
- workflow.with_lock do
11
- raise 'Already started' unless workflow.initial?
12
-
13
- workflow.initial_jobs.each do |job|
14
- enqueue_job!(job)
15
- end
16
- end
17
- end
18
-
19
- def enqueue_job!(job)
20
- job.enqueue!
21
- job.save! do
22
- Burst::Worker.perform_later(workflow.id, job.id)
23
- end
24
- end
25
-
26
- def resume!(job, data)
27
- job.resume!
28
- job.save! do
29
- Burst::Worker.perform_later(workflow.id, job.id, data)
30
- end
31
- end
32
-
33
- def start_job!(job)
34
- job.start!
35
- job.save!
36
-
37
- job.perform
38
- end
39
-
40
- def resume_job!(job, data)
41
- job.resume(data)
42
- end
43
-
44
- def suspend_job!(job)
45
- job.suspend!
46
- job.save!
47
- end
48
-
49
- def finish_job!(job)
50
- job.finish!
51
- job.save!
52
-
53
- workflow.with_lock do
54
- enqueue_outgoing_jobs(job)
55
- end
56
- end
57
-
58
- def job_performed!(job, result)
59
- if result == Burst::Job::SUSPEND || job.output == Burst::Job::SUSPEND
60
- suspend_job!(job)
61
- else
62
- finish_job!(job)
63
- end
64
- end
65
-
66
- def fail_job!(job)
67
- job.fail!
68
- job.save!
69
- end
70
-
71
- def enqueue_outgoing_jobs(job)
72
- job.outgoing.each do |job_id|
73
- out = workflow.get_job(job_id)
74
-
75
- enqueue_job!(out) if out.ready_to_start?
76
- end
77
- end
78
-
79
- end