gush 0.4.1 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
data/gush.gemspec CHANGED
@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "gush"
7
- spec.version = "0.4.1"
7
+ spec.version = "1.0.0"
8
8
  spec.authors = ["Piotrek Okoński"]
9
9
  spec.email = ["piotrek@okonski.org"]
10
- spec.summary = "Fast and distributed workflow runner using only Sidekiq and Redis"
11
- spec.description = "Gush is a parallel workflow runner using only Redis as its message broker and Sidekiq for workers."
12
- spec.homepage = "https://github.com/pokonski/gush"
10
+ spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
11
+ spec.description = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
12
+ spec.homepage = "https://github.com/chaps-io/gush"
13
13
  spec.license = "MIT"
14
14
 
15
15
  spec.files = `git ls-files -z`.split("\x0")
@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
- spec.add_dependency "sidekiq", "~> 4.0"
20
+ spec.add_dependency "activejob", "~> 5.0"
21
+ spec.add_dependency "connection_pool", "~> 2.2.1"
21
22
  spec.add_dependency "multi_json", "~> 1.11"
22
23
  spec.add_dependency "redis", "~> 3.2"
23
24
  spec.add_dependency "hiredis", "~> 0.6"
data/lib/gush.rb CHANGED
@@ -5,7 +5,6 @@ require "hiredis"
5
5
  require "pathname"
6
6
  require "redis"
7
7
  require "securerandom"
8
- require "sidekiq"
9
8
  require "multi_json"
10
9
 
11
10
  require "gush/json"
@@ -34,18 +33,5 @@ module Gush
34
33
 
35
34
  def self.configure
36
35
  yield configuration
37
- reconfigure_sidekiq
38
- end
39
-
40
- def self.reconfigure_sidekiq
41
- Sidekiq.configure_server do |config|
42
- config.redis = { url: configuration.redis_url, queue: configuration.namespace}
43
- end
44
-
45
- Sidekiq.configure_client do |config|
46
- config.redis = { url: configuration.redis_url, queue: configuration.namespace}
47
- end
48
36
  end
49
37
  end
50
-
51
- Gush.reconfigure_sidekiq
data/lib/gush/cli.rb CHANGED
@@ -2,16 +2,12 @@ require 'terminal-table'
2
2
  require 'colorize'
3
3
  require 'thor'
4
4
  require 'launchy'
5
- require 'sidekiq'
6
- require 'sidekiq/api'
7
5
 
8
6
  module Gush
9
7
  class CLI < Thor
10
8
  class_option :gushfile, desc: "configuration file to use", aliases: "-f"
11
- class_option :concurrency, desc: "concurrency setting for Sidekiq", aliases: "-c"
12
9
  class_option :redis, desc: "Redis URL to use", aliases: "-r"
13
10
  class_option :namespace, desc: "namespace to run jobs in", aliases: "-n"
14
- class_option :env, desc: "Sidekiq environment", aliases: "-e"
15
11
 
16
12
  def initialize(*)
17
13
  super
@@ -20,7 +16,6 @@ module Gush
20
16
  config.concurrency = options.fetch("concurrency", config.concurrency)
21
17
  config.redis_url = options.fetch("redis", config.redis_url)
22
18
  config.namespace = options.fetch("namespace", config.namespace)
23
- config.environment = options.fetch("environment", config.environment)
24
19
  end
25
20
  load_gushfile
26
21
  end
@@ -52,11 +47,6 @@ module Gush
52
47
  client.stop_workflow(id)
53
48
  end
54
49
 
55
- desc "clear", "Clears all jobs from Sidekiq queue"
56
- def clear
57
- Sidekiq::Queue.new(client.configuration.namespace).clear
58
- end
59
-
60
50
  desc "show [workflow_id]", "Shows details about workflow with given ID"
61
51
  option :skip_overview, type: :boolean
62
52
  option :skip_jobs, type: :boolean
@@ -79,22 +69,17 @@ module Gush
79
69
  def list
80
70
  workflows = client.all_workflows
81
71
  rows = workflows.map do |workflow|
82
- [workflow.id, workflow.class, {alignment: :center, value: status_for(workflow)}]
72
+ [workflow.id, (Time.at(workflow.started_at) if workflow.started_at), workflow.class, {alignment: :center, value: status_for(workflow)}]
83
73
  end
84
74
  headers = [
85
75
  {alignment: :center, value: 'id'},
76
+ {alignment: :center, value: 'started at'},
86
77
  {alignment: :center, value: 'name'},
87
78
  {alignment: :center, value: 'status'}
88
79
  ]
89
80
  puts Terminal::Table.new(headings: headers, rows: rows)
90
81
  end
91
82
 
92
- desc "workers", "Starts Sidekiq workers"
93
- def workers
94
- config = client.configuration
95
- Kernel.exec "bundle exec sidekiq -r #{config.gushfile} -c #{config.concurrency} -q #{config.namespace} -e #{config.environment} -v"
96
- end
97
-
98
83
  desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
99
84
  def viz(name)
100
85
  client
@@ -136,7 +121,7 @@ module Gush
136
121
  raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
137
122
  end
138
123
 
139
- require file
124
+ load file.to_s
140
125
  rescue LoadError
141
126
  raise Thor::Error, "failed to require #{file}".colorize(:red)
142
127
  end
@@ -65,7 +65,7 @@ module Gush
65
65
  end
66
66
 
67
67
  def started_at
68
- workflow.started_at.inspect
68
+ Time.at(workflow.started_at) if workflow.started_at
69
69
  end
70
70
 
71
71
  def failed_status
data/lib/gush/client.rb CHANGED
@@ -1,15 +1,15 @@
1
+ require 'connection_pool'
2
+
1
3
  module Gush
2
4
  class Client
3
- attr_reader :configuration, :sidekiq
5
+ attr_reader :configuration
4
6
 
5
7
  def initialize(config = Gush.configuration)
6
8
  @configuration = config
7
- @sidekiq = build_sidekiq
8
9
  end
9
10
 
10
11
  def configure
11
12
  yield configuration
12
- @sidekiq = build_sidekiq
13
13
  end
14
14
 
15
15
  def create_workflow(name)
@@ -111,8 +111,7 @@ module Gush
111
111
  end
112
112
  end
113
113
 
114
- def load_job(workflow_id, job_id)
115
- workflow = find_workflow(workflow_id)
114
+ def find_job(workflow_id, job_id)
116
115
  job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
117
116
  hypen = '-' if job_name_match.nil?
118
117
 
@@ -129,7 +128,7 @@ module Gush
129
128
  return nil if data.nil?
130
129
 
131
130
  data = Gush::JSON.decode(data, symbolize_keys: true)
132
- Gush::Job.from_hash(workflow, data)
131
+ Gush::Job.from_hash(data)
133
132
  end
134
133
 
135
134
  def destroy_workflow(workflow)
@@ -145,51 +144,28 @@ module Gush
145
144
  end
146
145
  end
147
146
 
148
- def worker_report(message)
149
- report("gush.workers.status", message)
150
- end
151
-
152
- def workflow_report(message)
153
- report("gush.workflows.status", message)
154
- end
155
-
156
147
  def enqueue_job(workflow_id, job)
157
148
  job.enqueue!
158
149
  persist_job(workflow_id, job)
159
150
 
160
- sidekiq.push(
161
- 'class' => Gush::Worker,
162
- 'queue' => configuration.namespace,
163
- 'args' => [workflow_id, job.name]
164
- )
151
+ Gush::Worker.set(queue: configuration.namespace).perform_later(*[workflow_id, job.name])
165
152
  end
166
153
 
167
154
  private
168
155
 
169
156
  def workflow_from_hash(hash, nodes = nil)
170
- flow = hash[:klass].constantize.new *hash[:arguments]
157
+ flow = hash[:klass].constantize.new(*hash[:arguments])
171
158
  flow.jobs = []
172
159
  flow.stopped = hash.fetch(:stopped, false)
173
160
  flow.id = hash[:id]
174
161
 
175
162
  (nodes || hash[:nodes]).each do |node|
176
- flow.jobs << Gush::Job.from_hash(flow, node)
163
+ flow.jobs << Gush::Job.from_hash(node)
177
164
  end
178
165
 
179
166
  flow
180
167
  end
181
168
 
182
- def report(key, message)
183
- connection_pool.with do |redis|
184
- redis.publish(key, Gush::JSON.encode(message))
185
- end
186
- end
187
-
188
-
189
- def build_sidekiq
190
- Sidekiq::Client.new(connection_pool)
191
- end
192
-
193
169
  def build_redis
194
170
  Redis.new(url: configuration.redis_url)
195
171
  end
@@ -1,6 +1,6 @@
1
1
  module Gush
2
2
  class Configuration
3
- attr_accessor :concurrency, :namespace, :redis_url, :environment
3
+ attr_accessor :concurrency, :namespace, :redis_url
4
4
 
5
5
  def self.from_json(json)
6
6
  new(Gush::JSON.decode(json, symbolize_keys: true))
@@ -10,8 +10,7 @@ module Gush
10
10
  self.concurrency = hash.fetch(:concurrency, 5)
11
11
  self.namespace = hash.fetch(:namespace, 'gush')
12
12
  self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
13
- self.gushfile = hash.fetch(:gushfile, 'Gushfile.rb')
14
- self.environment = hash.fetch(:environment, 'development')
13
+ self.gushfile = hash.fetch(:gushfile, 'Gushfile')
15
14
  end
16
15
 
17
16
  def gushfile=(path)
@@ -19,15 +18,14 @@ module Gush
19
18
  end
20
19
 
21
20
  def gushfile
22
- @gushfile.realpath
21
+ @gushfile.realpath if @gushfile.exist?
23
22
  end
24
23
 
25
24
  def to_hash
26
25
  {
27
26
  concurrency: concurrency,
28
27
  namespace: namespace,
29
- redis_url: redis_url,
30
- environment: environment
28
+ redis_url: redis_url
31
29
  }
32
30
  end
33
31
 
data/lib/gush/graph.rb CHANGED
@@ -41,7 +41,8 @@ module Gush
41
41
  graph.add_edges(job.name, end_node)
42
42
  else
43
43
  job.outgoing.each do |id|
44
- graph.add_edges(job.name, id)
44
+ outgoing_job = workflow.find_job(id)
45
+ graph.add_edges(job.name, outgoing_job.name)
45
46
  end
46
47
  end
47
48
  end
data/lib/gush/job.rb CHANGED
@@ -1,11 +1,10 @@
1
1
  module Gush
2
2
  class Job
3
3
  attr_accessor :workflow_id, :incoming, :outgoing, :params,
4
- :finished_at, :failed_at, :started_at, :enqueued_at, :payloads_hash, :klass
5
- attr_reader :name, :output_payload, :params, :payloads
4
+ :finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass
5
+ attr_reader :name, :output_payload, :params
6
6
 
7
- def initialize(workflow, opts = {})
8
- @workflow = workflow
7
+ def initialize(opts = {})
9
8
  options = opts.dup
10
9
  assign_variables(options)
11
10
  end
@@ -21,6 +20,7 @@ module Gush
21
20
  started_at: started_at,
22
21
  failed_at: failed_at,
23
22
  params: params,
23
+ workflow_id: workflow_id,
24
24
  output_payload: output_payload
25
25
  }
26
26
  end
@@ -29,21 +29,15 @@ module Gush
29
29
  Gush::JSON.encode(as_json)
30
30
  end
31
31
 
32
- def self.from_hash(flow, hash)
33
- hash[:klass].constantize.new(flow, hash)
32
+ def self.from_hash(hash)
33
+ hash[:klass].constantize.new(hash)
34
34
  end
35
35
 
36
36
  def output(data)
37
37
  @output_payload = data
38
38
  end
39
39
 
40
- def payloads
41
- payload_h = {}
42
- payloads_hash.each {|k,val| payload_h[k.to_s] = val.map {|h| h[:payload] }}
43
- payload_h
44
- end
45
-
46
- def work
40
+ def perform
47
41
  end
48
42
 
49
43
  def start!
@@ -95,7 +89,7 @@ module Gush
95
89
 
96
90
  def parents_succeeded?
97
91
  incoming.all? do |name|
98
- @workflow.find_job(name).succeeded?
92
+ client.find_job(workflow_id, name).succeeded?
99
93
  end
100
94
  end
101
95
 
@@ -104,8 +98,9 @@ module Gush
104
98
  end
105
99
 
106
100
  private
107
- def logger
108
- Sidekiq.logger
101
+
102
+ def client
103
+ @client ||= Client.new
109
104
  end
110
105
 
111
106
  def current_timestamp
@@ -123,6 +118,7 @@ module Gush
123
118
  @params = opts[:params] || {}
124
119
  @klass = opts[:klass]
125
120
  @output_payload = opts[:output_payload]
121
+ @workflow_id = opts[:workflow_id]
126
122
  end
127
123
  end
128
124
  end
data/lib/gush/worker.rb CHANGED
@@ -1,91 +1,63 @@
1
- require 'sidekiq'
1
+ require 'active_job'
2
2
 
3
3
  module Gush
4
- class Worker
5
- include ::Sidekiq::Worker
6
- sidekiq_options retry: false
7
-
4
+ class Worker < ::ActiveJob::Base
8
5
  def perform(workflow_id, job_id)
9
6
  setup_job(workflow_id, job_id)
10
7
 
11
- job.payloads_hash = incoming_payloads
12
-
13
- start = Time.now
14
- report(:started, start)
8
+ job.payloads = incoming_payloads
15
9
 
16
- failed = false
17
10
  error = nil
18
11
 
19
12
  mark_as_started
20
13
  begin
21
- job.work
22
- rescue Exception => error
14
+ job.perform
15
+ rescue StandardError => error
23
16
  mark_as_failed
24
- report(:failed, start, error.message)
25
17
  raise error
26
18
  else
27
19
  mark_as_finished
28
- report(:finished, start)
29
-
30
20
  enqueue_outgoing_jobs
31
21
  end
32
22
  end
33
23
 
34
24
  private
35
- attr_reader :client, :workflow, :job
25
+
26
+ attr_reader :client, :workflow_id, :job
36
27
 
37
28
  def client
38
29
  @client ||= Gush::Client.new(Gush.configuration)
39
30
  end
40
31
 
41
32
  def setup_job(workflow_id, job_id)
42
- @workflow ||= client.find_workflow(workflow_id)
43
- @job ||= workflow.find_job(job_id)
33
+ @workflow_id = workflow_id
34
+ @job ||= client.find_job(workflow_id, job_id)
44
35
  end
45
36
 
46
37
  def incoming_payloads
47
- payloads = {}
48
- job.incoming.each do |job_name|
49
- job = client.load_job(workflow.id, job_name)
50
- payloads[job.klass.to_s] ||= []
51
- payloads[job.klass.to_s] << {:id => job.name, :payload => job.output_payload}
38
+ job.incoming.map do |job_name|
39
+ job = client.find_job(workflow_id, job_name)
40
+ {
41
+ id: job.name,
42
+ class: job.klass.to_s,
43
+ output: job.output_payload
44
+ }
52
45
  end
53
- payloads
54
46
  end
55
47
 
56
48
  def mark_as_finished
57
49
  job.finish!
58
- client.persist_job(workflow.id, job)
50
+ client.persist_job(workflow_id, job)
59
51
  end
60
52
 
61
53
  def mark_as_failed
62
54
  job.fail!
63
- client.persist_job(workflow.id, job)
55
+ client.persist_job(workflow_id, job)
64
56
  end
65
57
 
66
58
  def mark_as_started
67
59
  job.start!
68
- client.persist_job(workflow.id, job)
69
- end
70
-
71
- def report_workflow_status
72
- client.workflow_report({
73
- workflow_id: workflow.id,
74
- status: workflow.status,
75
- started_at: workflow.started_at,
76
- finished_at: workflow.finished_at
77
- })
78
- end
79
-
80
- def report(status, start, error = nil)
81
- message = {
82
- status: status,
83
- workflow_id: workflow.id,
84
- job: job.name,
85
- duration: elapsed(start)
86
- }
87
- message[:error] = error if error
88
- client.worker_report(message)
60
+ client.persist_job(workflow_id, job)
89
61
  end
90
62
 
91
63
  def elapsed(start)
@@ -94,9 +66,9 @@ module Gush
94
66
 
95
67
  def enqueue_outgoing_jobs
96
68
  job.outgoing.each do |job_name|
97
- out = client.load_job(workflow.id, job_name)
69
+ out = client.find_job(workflow_id, job_name)
98
70
  if out.ready_to_start?
99
- client.enqueue_job(workflow.id, out)
71
+ client.enqueue_job(workflow_id, out)
100
72
  end
101
73
  end
102
74
  end