burstflow 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -3
  3. data/Gemfile.lock +119 -0
  4. data/burstflow.gemspec +10 -6
  5. data/config/database.yml +4 -3
  6. data/db/migrate/20180101000001_create_workflow.rb +1 -0
  7. data/db/schema.rb +13 -7
  8. data/lib/burstflow.rb +11 -0
  9. data/lib/burstflow/job.rb +102 -0
  10. data/lib/burstflow/job/callbacks.rb +55 -0
  11. data/lib/burstflow/job/exception.rb +8 -0
  12. data/lib/burstflow/job/initialization.rb +35 -0
  13. data/lib/{burst → burstflow/job}/model.rb +1 -3
  14. data/lib/burstflow/job/state.rb +125 -0
  15. data/lib/burstflow/manager.rb +123 -0
  16. data/lib/burstflow/railtie.rb +6 -0
  17. data/lib/burstflow/version.rb +3 -0
  18. data/lib/burstflow/worker.rb +59 -0
  19. data/lib/burstflow/workflow.rb +207 -0
  20. data/lib/burstflow/workflow/builder.rb +91 -0
  21. data/lib/burstflow/workflow/callbacks.rb +66 -0
  22. data/lib/{burst/workflow_helper.rb → burstflow/workflow/configuration.rb} +8 -39
  23. data/lib/burstflow/workflow/exception.rb +8 -0
  24. data/lib/generators/burstflow/install/install_generator.rb +22 -0
  25. data/lib/generators/burstflow/install/templates/create_workflow.rb +15 -0
  26. data/spec/builder_spec.rb +63 -0
  27. data/spec/{burst_spec.rb → burstflow_spec.rb} +1 -1
  28. data/spec/generators/install_generator_spec.rb +27 -0
  29. data/spec/job_spec.rb +18 -8
  30. data/spec/spec_helper.rb +4 -1
  31. data/spec/support/database_clean.rb +4 -1
  32. data/spec/workflow_spec.rb +397 -147
  33. metadata +45 -21
  34. data/db/migrate/20180101000001_create_workflow.rb +0 -13
  35. data/db/seeds.rb +0 -1
  36. data/lib/burst.rb +0 -37
  37. data/lib/burst/builder.rb +0 -48
  38. data/lib/burst/configuration.rb +0 -27
  39. data/lib/burst/job.rb +0 -187
  40. data/lib/burst/manager.rb +0 -79
  41. data/lib/burst/worker.rb +0 -42
  42. data/lib/burst/workflow.rb +0 -148
  43. data/spec/cases_spec.rb +0 -180
@@ -0,0 +1,91 @@
1
+ module Burstflow
2
+
3
+ class Workflow::Builder
4
+ def initialize workflow, *args, &block
5
+ @workflow = workflow
6
+ @deps = []
7
+ @jobs_by_class = {}
8
+ @jobs_by_id = {}
9
+
10
+ @workflow.jobs_config.each_pair do |id, job_hash|
11
+ job = Burstflow::Job.from_hash(@workflow, job_hash)
12
+
13
+ @jobs_by_class[job.klass.to_s] ||= []
14
+ @jobs_by_class[job.klass.to_s] << job
15
+ @jobs_by_id[id] = job
16
+ job.incoming.each do |from|
17
+ @deps << { from: from, to: id }
18
+ end
19
+
20
+ job.outgoing.each do |to|
21
+ @deps << { from: id, to: to }
22
+ end
23
+
24
+ @deps.uniq!
25
+ end
26
+
27
+ instance_exec *args, &block
28
+ resolve_dependencies
29
+ end
30
+
31
+ def run(klass, opts = {})
32
+ opts = opts.with_indifferent_access
33
+
34
+ before_deps = opts.delete(:before) || []
35
+ after_deps = opts.delete(:after) || []
36
+
37
+ job = klass.new(@workflow, opts)
38
+
39
+ [*before_deps].each do |dep|
40
+ @deps << { from: job.id, to: dep.to_s }
41
+ end
42
+
43
+ [*after_deps].each do |dep|
44
+ @deps << { from: dep.to_s, to: job.id }
45
+ end
46
+
47
+ @jobs_by_class[klass.to_s] ||= []
48
+ @jobs_by_class[klass.to_s] << job
49
+
50
+ raise "Job id duplication" if @jobs_by_id.key?(job.id)
51
+ @jobs_by_id[job.id] = job
52
+
53
+ job.id
54
+ end
55
+
56
+ def find_job id_or_klass
57
+ id = if @jobs_by_id.key?(id_or_klass)
58
+ id_or_klass
59
+ else
60
+ jobs = @jobs_by_class[id_or_klass.to_s]
61
+
62
+ raise "No job with #{id_or_klass.to_s} klass or id found" if jobs.count == 0
63
+ raise "Duplicated jobs with #{id_or_klass.to_s} klass or id detected" if jobs.count > 1
64
+
65
+ jobs.first.id
66
+ end
67
+
68
+ @jobs_by_id[id]
69
+ end
70
+
71
+ def resolve_dependencies
72
+ @deps.each do |dependency|
73
+ from = find_job(dependency[:from].to_s)
74
+ to = find_job(dependency[:to].to_s)
75
+
76
+ to.incoming << from.id
77
+ from.outgoing << to.id
78
+
79
+ to.incoming.uniq!
80
+ from.outgoing.uniq!
81
+ end
82
+ end
83
+
84
+ def as_json
85
+ @jobs_by_id.each_with_object({}) do |(id, job), json|
86
+ json[job.id] = job.as_json
87
+ end
88
+ end
89
+
90
+ end
91
+ end
@@ -0,0 +1,66 @@
1
+ module Burstflow::Workflow::Callbacks
2
+ extend ActiveSupport::Concern
3
+ include ActiveSupport::Callbacks
4
+
5
+ included do
6
+
7
+ define_callbacks :failure, :finish, :suspend, :resume
8
+
9
+ end
10
+
11
+ class_methods do
12
+
13
+ def before_failure(*filters, &blk)
14
+ set_callback(:failure, :before, *filters, &blk)
15
+ end
16
+
17
+ def after_failure(*filters, &blk)
18
+ set_callback(:failure, :after, *filters, &blk)
19
+ end
20
+
21
+ def around_failure(*filters, &blk)
22
+ set_callback(:failure, :around, *filters, &blk)
23
+ end
24
+
25
+
26
+ def before_finish(*filters, &blk)
27
+ set_callback(:finish, :before, *filters, &blk)
28
+ end
29
+
30
+ def after_finish(*filters, &blk)
31
+ set_callback(:finish, :after, *filters, &blk)
32
+ end
33
+
34
+ def around_finish(*filters, &blk)
35
+ set_callback(:finish, :around, *filters, &blk)
36
+ end
37
+
38
+
39
+ def before_suspend(*filters, &blk)
40
+ set_callback(:suspend, :before, *filters, &blk)
41
+ end
42
+
43
+ def after_suspend(*filters, &blk)
44
+ set_callback(:suspend, :after, *filters, &blk)
45
+ end
46
+
47
+ def around_suspend(*filters, &blk)
48
+ set_callback(:suspend, :around, *filters, &blk)
49
+ end
50
+
51
+
52
+ def before_resume(*filters, &blk)
53
+ set_callback(:resume, :before, *filters, &blk)
54
+ end
55
+
56
+ def after_resume(*filters, &blk)
57
+ set_callback(:resume, :after, *filters, &blk)
58
+ end
59
+
60
+ def around_resume(*filters, &blk)
61
+ set_callback(:resume, :around, *filters, &blk)
62
+ end
63
+
64
+ end
65
+
66
+ end
@@ -1,5 +1,6 @@
1
- module Burst::WorkflowHelper
1
+ module Burstflow
2
2
 
3
+ module Workflow::Configuration
3
4
  extend ActiveSupport::Concern
4
5
 
5
6
  class JSONBWithIndifferentAccess
@@ -18,25 +19,12 @@ module Burst::WorkflowHelper
18
19
 
19
20
  included do |_klass|
20
21
  serialize :flow, JSONBWithIndifferentAccess
21
-
22
- def first_job
23
- all_jobs.min_by{|n| n.started_at || Time.now.to_i }
24
- end
25
-
26
- def last_job
27
- all_jobs.max_by{|n| n.finished_at || 0 } if finished?
28
- end
29
-
30
- def started_at
31
- first_job&.started_at
32
- end
33
-
34
- def finished_at
35
- last_job&.finished_at
36
- end
37
-
22
+
38
23
  def configure(*args)
39
- instance_exec *args, &self.class.configuration
24
+ builder = Builder.new()
25
+ builder.instance_exec *args, &self.class.configuration
26
+ builder.resolve_dependencies
27
+ builder.as_json
40
28
  end
41
29
  end
42
30
 
@@ -64,23 +52,4 @@ module Burst::WorkflowHelper
64
52
 
65
53
  end
66
54
 
67
-
68
- # class Test
69
- # def self.configure &block
70
- # @c = block
71
- # end
72
-
73
- # def self.c
74
- # @c
75
- # end
76
-
77
- # def go
78
- # puts "111111"
79
- # end
80
-
81
- # def run
82
- # instance_eval &Test.c
83
- # end
84
-
85
-
86
- # end
55
+ end
@@ -0,0 +1,8 @@
1
+ class Burstflow::Workflow::InternalError < ::RuntimeError
2
+ attr_accessor :workflow
3
+
4
+ def initialize(workflow, message)
5
+ @workflow = workflow
6
+ super(message)
7
+ end
8
+ end
@@ -0,0 +1,22 @@
1
+ require 'rails/generators'
2
+ require 'rails/generators/migration'
3
+
4
+ module Burstflow
5
+ module Generators
6
+ class InstallGenerator < ::Rails::Generators::Base
7
+ include Rails::Generators::Migration
8
+ source_root File.expand_path('../templates', __FILE__)
9
+ desc "Add the migrations for Burstflow"
10
+
11
+ def self.next_migration_number(path)
12
+ next_migration_number = current_migration_number(path) + 1
13
+ ActiveRecord::Migration.next_migration_number(next_migration_number)
14
+ end
15
+
16
+ def copy_migrations
17
+ migration_template "create_workflow.rb",
18
+ "db/migrate/create_workflow.rb"
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,15 @@
1
+ class CreateWorkflow < ActiveRecord::Migration[5.1]
2
+
3
+ def change
4
+ enable_extension 'pgcrypto'
5
+
6
+ create_table :burstflow_workflows, id: :uuid do |t|
7
+ t.string :type, index: true
8
+ t.string :status, index: true
9
+ t.jsonb :flow, null: false, default: {}
10
+
11
+ t.timestamps
12
+ end
13
+ end
14
+
15
+ end
@@ -0,0 +1,63 @@
1
+ require 'spec_helper'
2
+
3
+ describe Burstflow::Workflow::Builder do
4
+ context 'initializing' do
5
+ BuilderJob1 = Class.new(Burstflow::Job)
6
+ BuilderJob2 = Class.new(Burstflow::Job)
7
+ BuilderJob3 = Class.new(Burstflow::Job)
8
+
9
+ let(:workflow){double(:workflow, id: 'id1', jobs_config: {})}
10
+
11
+ it 'without dependencies' do
12
+ builder = Burstflow::Workflow::Builder.new workflow do
13
+ $jobid1 = run BuilderJob1, params: { param1: true }
14
+ end
15
+
16
+ flow = builder.as_json
17
+ expect(flow.count).to eq 1
18
+
19
+ expect(flow[$jobid1]).to include(:id, :incoming, :outgoing, workflow_id: 'id1', params: {'param1' => true})
20
+ end
21
+
22
+ it 'with dependencies' do
23
+ builder = Burstflow::Workflow::Builder.new workflow, :arg1, :arg2 do |arg1, arg2|
24
+ $jobid1 = run BuilderJob1, params: { param1: true, arg: arg1}
25
+ $jobid2 = run BuilderJob2, params: { param2: true, arg: arg2}, after: BuilderJob1
26
+ $jobid3 = run BuilderJob3, before: BuilderJob2, after: $jobid1
27
+ $jobid4 = run BuilderJob3, after: $jobid3
28
+ end
29
+
30
+ flow = builder.as_json
31
+ expect(flow.count).to eq 4
32
+
33
+ expect(flow[$jobid1]).to include(:id,
34
+ klass: BuilderJob1.to_s,
35
+ incoming: [],
36
+ outgoing: [$jobid2, $jobid3],
37
+ workflow_id: 'id1',
38
+ params: {'param1' => true, 'arg' => :arg1})
39
+
40
+ expect(flow[$jobid2]).to include(:id,
41
+ klass: BuilderJob2.to_s,
42
+ incoming: [$jobid1, $jobid3],
43
+ outgoing: [],
44
+ workflow_id: 'id1',
45
+ params: {'param2' => true, 'arg' => :arg2})
46
+
47
+ expect(flow[$jobid3]).to include(:id,
48
+ klass: BuilderJob3.to_s,
49
+ incoming: [$jobid1],
50
+ outgoing: [$jobid2, $jobid4],
51
+ workflow_id: 'id1',
52
+ params: nil)
53
+
54
+ expect(flow[$jobid4]).to include(:id,
55
+ klass: BuilderJob3.to_s,
56
+ incoming: [$jobid3],
57
+ outgoing: [],
58
+ workflow_id: 'id1',
59
+ params: nil)
60
+ end
61
+
62
+ end
63
+ end
@@ -1,4 +1,4 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Burst do
3
+ describe Burstflow do
4
4
  end
@@ -0,0 +1,27 @@
1
+ require 'spec_helper'
2
+ require "generator_spec"
3
+ require 'tmpdir'
4
+
5
+ require 'generators/burstflow/install/install_generator'
6
+
7
+ module Burstflow
8
+ module Generators
9
+ describe InstallGenerator, type: :generator do
10
+ root_dir = File.expand_path(Dir.tmpdir(), __FILE__)
11
+ destination root_dir
12
+
13
+ before :all do
14
+ prepare_destination
15
+ run_generator
16
+ end
17
+
18
+ it "creates the installation db migration" do
19
+ migration_file =
20
+ Dir.glob("#{root_dir}/db/migrate/*create_workflow.rb")
21
+
22
+ assert_file migration_file[0],
23
+ /CreateWorkflow < ActiveRecord::Migration/
24
+ end
25
+ end
26
+ end
27
+ end
@@ -1,17 +1,17 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Burst::Job do
4
- let(:w) { Burst::Workflow.new }
3
+ describe Burstflow::Job do
4
+ let(:w) { Burstflow::Workflow.new }
5
5
 
6
6
  context 'initializing' do
7
- class TestJob1 < Burst::Job
7
+ class JobJob1 < Burstflow::Job
8
8
  end
9
9
 
10
- subject{ TestJob1.new(w, {}.with_indifferent_access) }
10
+ subject{ JobJob1.new(w, {}.with_indifferent_access) }
11
11
 
12
12
  it 'empty' do
13
13
  expect(subject.workflow_id).to eq w.id
14
- expect(subject.klass).to eq TestJob1.to_s
14
+ expect(subject.klass).to eq JobJob1.to_s
15
15
 
16
16
  expect(subject.incoming).to eq []
17
17
  expect(subject.outgoing).to eq []
@@ -36,6 +36,7 @@ describe Burst::Job do
36
36
  end
37
37
 
38
38
  it '#start!' do
39
+ subject.enqueue!
39
40
  subject.start!
40
41
 
41
42
  expect(subject.started?).to eq true
@@ -43,6 +44,8 @@ describe Burst::Job do
43
44
  end
44
45
 
45
46
  it '#finish!' do
47
+ subject.enqueue!
48
+ subject.start!
46
49
  subject.finish!
47
50
 
48
51
  expect(subject.finished?).to eq true
@@ -52,29 +55,36 @@ describe Burst::Job do
52
55
  end
53
56
 
54
57
  it '#fail!' do
55
- subject.fail!
58
+ subject.enqueue!
59
+ subject.start!
60
+ subject.fail! "error"
56
61
 
57
62
  expect(subject.finished?).to eq true
58
63
  expect(subject.succeeded?).to eq false
59
64
  expect(subject.failed?).to eq true
60
65
  expect(subject.ready_to_start?).to eq false
66
+ expect(subject.failure).to include(message: 'error')
61
67
  end
62
68
 
63
69
  it '#suspend!' do
70
+ subject.enqueue!
71
+ subject.start!
64
72
  subject.suspend!
65
73
 
66
74
  expect(subject.finished?).to eq false
67
75
  expect(subject.suspended?).to eq true
68
- expect(subject.ready_to_start?).to eq true
76
+ expect(subject.ready_to_start?).to eq false
69
77
  end
70
78
 
71
79
  it '#resume!' do
80
+ subject.enqueue!
81
+ subject.start!
72
82
  subject.suspend!
73
83
  subject.resume!
74
84
 
75
85
  expect(subject.finished?).to eq false
76
86
  expect(subject.resumed?).to eq true
77
- expect(subject.ready_to_start?).to eq true
87
+ expect(subject.ready_to_start?).to eq false
78
88
  end
79
89
  end
80
90
  end