gush 0.4.1 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/.travis.yml +2 -2
- data/CHANGELOG.md +33 -1
- data/README.md +195 -80
- data/gush.gemspec +6 -5
- data/lib/gush.rb +0 -14
- data/lib/gush/cli.rb +3 -18
- data/lib/gush/cli/overview.rb +1 -1
- data/lib/gush/client.rb +8 -32
- data/lib/gush/configuration.rb +4 -6
- data/lib/gush/graph.rb +2 -1
- data/lib/gush/job.rb +12 -16
- data/lib/gush/worker.rb +21 -49
- data/lib/gush/workflow.rb +9 -5
- data/spec/{Gushfile.rb → Gushfile} +0 -0
- data/spec/features/integration_spec.rb +62 -23
- data/spec/gush/client_spec.rb +1 -1
- data/spec/gush/configuration_spec.rb +0 -3
- data/spec/gush/job_spec.rb +3 -3
- data/spec/gush/worker_spec.rb +33 -41
- data/spec/gush/workflow_spec.rb +4 -2
- data/spec/gush_spec.rb +4 -4
- data/spec/spec_helper.rb +23 -9
- metadata +26 -12
data/gush.gemspec
CHANGED
@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "gush"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "1.0.0"
|
8
8
|
spec.authors = ["Piotrek Okoński"]
|
9
9
|
spec.email = ["piotrek@okonski.org"]
|
10
|
-
spec.summary = "Fast and distributed workflow runner
|
11
|
-
spec.description = "Gush is a parallel workflow runner using
|
12
|
-
spec.homepage = "https://github.com/
|
10
|
+
spec.summary = "Fast and distributed workflow runner based on ActiveJob and Redis"
|
11
|
+
spec.description = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
|
12
|
+
spec.homepage = "https://github.com/chaps-io/gush"
|
13
13
|
spec.license = "MIT"
|
14
14
|
|
15
15
|
spec.files = `git ls-files -z`.split("\x0")
|
@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_dependency "
|
20
|
+
spec.add_dependency "activejob", "~> 5.0"
|
21
|
+
spec.add_dependency "connection_pool", "~> 2.2.1"
|
21
22
|
spec.add_dependency "multi_json", "~> 1.11"
|
22
23
|
spec.add_dependency "redis", "~> 3.2"
|
23
24
|
spec.add_dependency "hiredis", "~> 0.6"
|
data/lib/gush.rb
CHANGED
@@ -5,7 +5,6 @@ require "hiredis"
|
|
5
5
|
require "pathname"
|
6
6
|
require "redis"
|
7
7
|
require "securerandom"
|
8
|
-
require "sidekiq"
|
9
8
|
require "multi_json"
|
10
9
|
|
11
10
|
require "gush/json"
|
@@ -34,18 +33,5 @@ module Gush
|
|
34
33
|
|
35
34
|
def self.configure
|
36
35
|
yield configuration
|
37
|
-
reconfigure_sidekiq
|
38
|
-
end
|
39
|
-
|
40
|
-
def self.reconfigure_sidekiq
|
41
|
-
Sidekiq.configure_server do |config|
|
42
|
-
config.redis = { url: configuration.redis_url, queue: configuration.namespace}
|
43
|
-
end
|
44
|
-
|
45
|
-
Sidekiq.configure_client do |config|
|
46
|
-
config.redis = { url: configuration.redis_url, queue: configuration.namespace}
|
47
|
-
end
|
48
36
|
end
|
49
37
|
end
|
50
|
-
|
51
|
-
Gush.reconfigure_sidekiq
|
data/lib/gush/cli.rb
CHANGED
@@ -2,16 +2,12 @@ require 'terminal-table'
|
|
2
2
|
require 'colorize'
|
3
3
|
require 'thor'
|
4
4
|
require 'launchy'
|
5
|
-
require 'sidekiq'
|
6
|
-
require 'sidekiq/api'
|
7
5
|
|
8
6
|
module Gush
|
9
7
|
class CLI < Thor
|
10
8
|
class_option :gushfile, desc: "configuration file to use", aliases: "-f"
|
11
|
-
class_option :concurrency, desc: "concurrency setting for Sidekiq", aliases: "-c"
|
12
9
|
class_option :redis, desc: "Redis URL to use", aliases: "-r"
|
13
10
|
class_option :namespace, desc: "namespace to run jobs in", aliases: "-n"
|
14
|
-
class_option :env, desc: "Sidekiq environment", aliases: "-e"
|
15
11
|
|
16
12
|
def initialize(*)
|
17
13
|
super
|
@@ -20,7 +16,6 @@ module Gush
|
|
20
16
|
config.concurrency = options.fetch("concurrency", config.concurrency)
|
21
17
|
config.redis_url = options.fetch("redis", config.redis_url)
|
22
18
|
config.namespace = options.fetch("namespace", config.namespace)
|
23
|
-
config.environment = options.fetch("environment", config.environment)
|
24
19
|
end
|
25
20
|
load_gushfile
|
26
21
|
end
|
@@ -52,11 +47,6 @@ module Gush
|
|
52
47
|
client.stop_workflow(id)
|
53
48
|
end
|
54
49
|
|
55
|
-
desc "clear", "Clears all jobs from Sidekiq queue"
|
56
|
-
def clear
|
57
|
-
Sidekiq::Queue.new(client.configuration.namespace).clear
|
58
|
-
end
|
59
|
-
|
60
50
|
desc "show [workflow_id]", "Shows details about workflow with given ID"
|
61
51
|
option :skip_overview, type: :boolean
|
62
52
|
option :skip_jobs, type: :boolean
|
@@ -79,22 +69,17 @@ module Gush
|
|
79
69
|
def list
|
80
70
|
workflows = client.all_workflows
|
81
71
|
rows = workflows.map do |workflow|
|
82
|
-
[workflow.id, workflow.class, {alignment: :center, value: status_for(workflow)}]
|
72
|
+
[workflow.id, (Time.at(workflow.started_at) if workflow.started_at), workflow.class, {alignment: :center, value: status_for(workflow)}]
|
83
73
|
end
|
84
74
|
headers = [
|
85
75
|
{alignment: :center, value: 'id'},
|
76
|
+
{alignment: :center, value: 'started at'},
|
86
77
|
{alignment: :center, value: 'name'},
|
87
78
|
{alignment: :center, value: 'status'}
|
88
79
|
]
|
89
80
|
puts Terminal::Table.new(headings: headers, rows: rows)
|
90
81
|
end
|
91
82
|
|
92
|
-
desc "workers", "Starts Sidekiq workers"
|
93
|
-
def workers
|
94
|
-
config = client.configuration
|
95
|
-
Kernel.exec "bundle exec sidekiq -r #{config.gushfile} -c #{config.concurrency} -q #{config.namespace} -e #{config.environment} -v"
|
96
|
-
end
|
97
|
-
|
98
83
|
desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
|
99
84
|
def viz(name)
|
100
85
|
client
|
@@ -136,7 +121,7 @@ module Gush
|
|
136
121
|
raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
|
137
122
|
end
|
138
123
|
|
139
|
-
|
124
|
+
load file.to_s
|
140
125
|
rescue LoadError
|
141
126
|
raise Thor::Error, "failed to require #{file}".colorize(:red)
|
142
127
|
end
|
data/lib/gush/cli/overview.rb
CHANGED
data/lib/gush/client.rb
CHANGED
@@ -1,15 +1,15 @@
|
|
1
|
+
require 'connection_pool'
|
2
|
+
|
1
3
|
module Gush
|
2
4
|
class Client
|
3
|
-
attr_reader :configuration
|
5
|
+
attr_reader :configuration
|
4
6
|
|
5
7
|
def initialize(config = Gush.configuration)
|
6
8
|
@configuration = config
|
7
|
-
@sidekiq = build_sidekiq
|
8
9
|
end
|
9
10
|
|
10
11
|
def configure
|
11
12
|
yield configuration
|
12
|
-
@sidekiq = build_sidekiq
|
13
13
|
end
|
14
14
|
|
15
15
|
def create_workflow(name)
|
@@ -111,8 +111,7 @@ module Gush
|
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
114
|
-
def
|
115
|
-
workflow = find_workflow(workflow_id)
|
114
|
+
def find_job(workflow_id, job_id)
|
116
115
|
job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
|
117
116
|
hypen = '-' if job_name_match.nil?
|
118
117
|
|
@@ -129,7 +128,7 @@ module Gush
|
|
129
128
|
return nil if data.nil?
|
130
129
|
|
131
130
|
data = Gush::JSON.decode(data, symbolize_keys: true)
|
132
|
-
Gush::Job.from_hash(
|
131
|
+
Gush::Job.from_hash(data)
|
133
132
|
end
|
134
133
|
|
135
134
|
def destroy_workflow(workflow)
|
@@ -145,51 +144,28 @@ module Gush
|
|
145
144
|
end
|
146
145
|
end
|
147
146
|
|
148
|
-
def worker_report(message)
|
149
|
-
report("gush.workers.status", message)
|
150
|
-
end
|
151
|
-
|
152
|
-
def workflow_report(message)
|
153
|
-
report("gush.workflows.status", message)
|
154
|
-
end
|
155
|
-
|
156
147
|
def enqueue_job(workflow_id, job)
|
157
148
|
job.enqueue!
|
158
149
|
persist_job(workflow_id, job)
|
159
150
|
|
160
|
-
|
161
|
-
'class' => Gush::Worker,
|
162
|
-
'queue' => configuration.namespace,
|
163
|
-
'args' => [workflow_id, job.name]
|
164
|
-
)
|
151
|
+
Gush::Worker.set(queue: configuration.namespace).perform_later(*[workflow_id, job.name])
|
165
152
|
end
|
166
153
|
|
167
154
|
private
|
168
155
|
|
169
156
|
def workflow_from_hash(hash, nodes = nil)
|
170
|
-
flow = hash[:klass].constantize.new
|
157
|
+
flow = hash[:klass].constantize.new(*hash[:arguments])
|
171
158
|
flow.jobs = []
|
172
159
|
flow.stopped = hash.fetch(:stopped, false)
|
173
160
|
flow.id = hash[:id]
|
174
161
|
|
175
162
|
(nodes || hash[:nodes]).each do |node|
|
176
|
-
flow.jobs << Gush::Job.from_hash(
|
163
|
+
flow.jobs << Gush::Job.from_hash(node)
|
177
164
|
end
|
178
165
|
|
179
166
|
flow
|
180
167
|
end
|
181
168
|
|
182
|
-
def report(key, message)
|
183
|
-
connection_pool.with do |redis|
|
184
|
-
redis.publish(key, Gush::JSON.encode(message))
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
|
189
|
-
def build_sidekiq
|
190
|
-
Sidekiq::Client.new(connection_pool)
|
191
|
-
end
|
192
|
-
|
193
169
|
def build_redis
|
194
170
|
Redis.new(url: configuration.redis_url)
|
195
171
|
end
|
data/lib/gush/configuration.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Gush
|
2
2
|
class Configuration
|
3
|
-
attr_accessor :concurrency, :namespace, :redis_url
|
3
|
+
attr_accessor :concurrency, :namespace, :redis_url
|
4
4
|
|
5
5
|
def self.from_json(json)
|
6
6
|
new(Gush::JSON.decode(json, symbolize_keys: true))
|
@@ -10,8 +10,7 @@ module Gush
|
|
10
10
|
self.concurrency = hash.fetch(:concurrency, 5)
|
11
11
|
self.namespace = hash.fetch(:namespace, 'gush')
|
12
12
|
self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
|
13
|
-
self.gushfile = hash.fetch(:gushfile, 'Gushfile
|
14
|
-
self.environment = hash.fetch(:environment, 'development')
|
13
|
+
self.gushfile = hash.fetch(:gushfile, 'Gushfile')
|
15
14
|
end
|
16
15
|
|
17
16
|
def gushfile=(path)
|
@@ -19,15 +18,14 @@ module Gush
|
|
19
18
|
end
|
20
19
|
|
21
20
|
def gushfile
|
22
|
-
@gushfile.realpath
|
21
|
+
@gushfile.realpath if @gushfile.exist?
|
23
22
|
end
|
24
23
|
|
25
24
|
def to_hash
|
26
25
|
{
|
27
26
|
concurrency: concurrency,
|
28
27
|
namespace: namespace,
|
29
|
-
redis_url: redis_url
|
30
|
-
environment: environment
|
28
|
+
redis_url: redis_url
|
31
29
|
}
|
32
30
|
end
|
33
31
|
|
data/lib/gush/graph.rb
CHANGED
data/lib/gush/job.rb
CHANGED
@@ -1,11 +1,10 @@
|
|
1
1
|
module Gush
|
2
2
|
class Job
|
3
3
|
attr_accessor :workflow_id, :incoming, :outgoing, :params,
|
4
|
-
:finished_at, :failed_at, :started_at, :enqueued_at, :
|
5
|
-
attr_reader :name, :output_payload, :params
|
4
|
+
:finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass
|
5
|
+
attr_reader :name, :output_payload, :params
|
6
6
|
|
7
|
-
def initialize(
|
8
|
-
@workflow = workflow
|
7
|
+
def initialize(opts = {})
|
9
8
|
options = opts.dup
|
10
9
|
assign_variables(options)
|
11
10
|
end
|
@@ -21,6 +20,7 @@ module Gush
|
|
21
20
|
started_at: started_at,
|
22
21
|
failed_at: failed_at,
|
23
22
|
params: params,
|
23
|
+
workflow_id: workflow_id,
|
24
24
|
output_payload: output_payload
|
25
25
|
}
|
26
26
|
end
|
@@ -29,21 +29,15 @@ module Gush
|
|
29
29
|
Gush::JSON.encode(as_json)
|
30
30
|
end
|
31
31
|
|
32
|
-
def self.from_hash(
|
33
|
-
hash[:klass].constantize.new(
|
32
|
+
def self.from_hash(hash)
|
33
|
+
hash[:klass].constantize.new(hash)
|
34
34
|
end
|
35
35
|
|
36
36
|
def output(data)
|
37
37
|
@output_payload = data
|
38
38
|
end
|
39
39
|
|
40
|
-
def
|
41
|
-
payload_h = {}
|
42
|
-
payloads_hash.each {|k,val| payload_h[k.to_s] = val.map {|h| h[:payload] }}
|
43
|
-
payload_h
|
44
|
-
end
|
45
|
-
|
46
|
-
def work
|
40
|
+
def perform
|
47
41
|
end
|
48
42
|
|
49
43
|
def start!
|
@@ -95,7 +89,7 @@ module Gush
|
|
95
89
|
|
96
90
|
def parents_succeeded?
|
97
91
|
incoming.all? do |name|
|
98
|
-
|
92
|
+
client.find_job(workflow_id, name).succeeded?
|
99
93
|
end
|
100
94
|
end
|
101
95
|
|
@@ -104,8 +98,9 @@ module Gush
|
|
104
98
|
end
|
105
99
|
|
106
100
|
private
|
107
|
-
|
108
|
-
|
101
|
+
|
102
|
+
def client
|
103
|
+
@client ||= Client.new
|
109
104
|
end
|
110
105
|
|
111
106
|
def current_timestamp
|
@@ -123,6 +118,7 @@ module Gush
|
|
123
118
|
@params = opts[:params] || {}
|
124
119
|
@klass = opts[:klass]
|
125
120
|
@output_payload = opts[:output_payload]
|
121
|
+
@workflow_id = opts[:workflow_id]
|
126
122
|
end
|
127
123
|
end
|
128
124
|
end
|
data/lib/gush/worker.rb
CHANGED
@@ -1,91 +1,63 @@
|
|
1
|
-
require '
|
1
|
+
require 'active_job'
|
2
2
|
|
3
3
|
module Gush
|
4
|
-
class Worker
|
5
|
-
include ::Sidekiq::Worker
|
6
|
-
sidekiq_options retry: false
|
7
|
-
|
4
|
+
class Worker < ::ActiveJob::Base
|
8
5
|
def perform(workflow_id, job_id)
|
9
6
|
setup_job(workflow_id, job_id)
|
10
7
|
|
11
|
-
job.
|
12
|
-
|
13
|
-
start = Time.now
|
14
|
-
report(:started, start)
|
8
|
+
job.payloads = incoming_payloads
|
15
9
|
|
16
|
-
failed = false
|
17
10
|
error = nil
|
18
11
|
|
19
12
|
mark_as_started
|
20
13
|
begin
|
21
|
-
job.
|
22
|
-
rescue
|
14
|
+
job.perform
|
15
|
+
rescue StandardError => error
|
23
16
|
mark_as_failed
|
24
|
-
report(:failed, start, error.message)
|
25
17
|
raise error
|
26
18
|
else
|
27
19
|
mark_as_finished
|
28
|
-
report(:finished, start)
|
29
|
-
|
30
20
|
enqueue_outgoing_jobs
|
31
21
|
end
|
32
22
|
end
|
33
23
|
|
34
24
|
private
|
35
|
-
|
25
|
+
|
26
|
+
attr_reader :client, :workflow_id, :job
|
36
27
|
|
37
28
|
def client
|
38
29
|
@client ||= Gush::Client.new(Gush.configuration)
|
39
30
|
end
|
40
31
|
|
41
32
|
def setup_job(workflow_id, job_id)
|
42
|
-
@
|
43
|
-
@job ||=
|
33
|
+
@workflow_id = workflow_id
|
34
|
+
@job ||= client.find_job(workflow_id, job_id)
|
44
35
|
end
|
45
36
|
|
46
37
|
def incoming_payloads
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
38
|
+
job.incoming.map do |job_name|
|
39
|
+
job = client.find_job(workflow_id, job_name)
|
40
|
+
{
|
41
|
+
id: job.name,
|
42
|
+
class: job.klass.to_s,
|
43
|
+
output: job.output_payload
|
44
|
+
}
|
52
45
|
end
|
53
|
-
payloads
|
54
46
|
end
|
55
47
|
|
56
48
|
def mark_as_finished
|
57
49
|
job.finish!
|
58
|
-
client.persist_job(
|
50
|
+
client.persist_job(workflow_id, job)
|
59
51
|
end
|
60
52
|
|
61
53
|
def mark_as_failed
|
62
54
|
job.fail!
|
63
|
-
client.persist_job(
|
55
|
+
client.persist_job(workflow_id, job)
|
64
56
|
end
|
65
57
|
|
66
58
|
def mark_as_started
|
67
59
|
job.start!
|
68
|
-
client.persist_job(
|
69
|
-
end
|
70
|
-
|
71
|
-
def report_workflow_status
|
72
|
-
client.workflow_report({
|
73
|
-
workflow_id: workflow.id,
|
74
|
-
status: workflow.status,
|
75
|
-
started_at: workflow.started_at,
|
76
|
-
finished_at: workflow.finished_at
|
77
|
-
})
|
78
|
-
end
|
79
|
-
|
80
|
-
def report(status, start, error = nil)
|
81
|
-
message = {
|
82
|
-
status: status,
|
83
|
-
workflow_id: workflow.id,
|
84
|
-
job: job.name,
|
85
|
-
duration: elapsed(start)
|
86
|
-
}
|
87
|
-
message[:error] = error if error
|
88
|
-
client.worker_report(message)
|
60
|
+
client.persist_job(workflow_id, job)
|
89
61
|
end
|
90
62
|
|
91
63
|
def elapsed(start)
|
@@ -94,9 +66,9 @@ module Gush
|
|
94
66
|
|
95
67
|
def enqueue_outgoing_jobs
|
96
68
|
job.outgoing.each do |job_name|
|
97
|
-
out = client.
|
69
|
+
out = client.find_job(workflow_id, job_name)
|
98
70
|
if out.ready_to_start?
|
99
|
-
client.enqueue_job(
|
71
|
+
client.enqueue_job(workflow_id, out)
|
100
72
|
end
|
101
73
|
end
|
102
74
|
end
|