gush 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/.rspec +1 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +128 -0
- data/Rakefile +1 -0
- data/bin/gush +12 -0
- data/gush.gemspec +32 -0
- data/lib/gush.rb +47 -0
- data/lib/gush/cli.rb +245 -0
- data/lib/gush/client.rb +146 -0
- data/lib/gush/configuration.rb +42 -0
- data/lib/gush/errors.rb +3 -0
- data/lib/gush/job.rb +161 -0
- data/lib/gush/logger_builder.rb +15 -0
- data/lib/gush/metadata.rb +24 -0
- data/lib/gush/null_logger.rb +6 -0
- data/lib/gush/version.rb +3 -0
- data/lib/gush/worker.rb +100 -0
- data/lib/gush/workflow.rb +154 -0
- data/spec/Gushfile.rb +0 -0
- data/spec/lib/gush/client_spec.rb +125 -0
- data/spec/lib/gush/configuration_spec.rb +27 -0
- data/spec/lib/gush/job_spec.rb +114 -0
- data/spec/lib/gush/logger_builder_spec.rb +25 -0
- data/spec/lib/gush/null_logger_spec.rb +15 -0
- data/spec/lib/gush/worker_spec.rb +96 -0
- data/spec/lib/gush/workflow_spec.rb +246 -0
- data/spec/lib/gush_spec.rb +39 -0
- data/spec/redis.conf +2 -0
- data/spec/spec_helper.rb +79 -0
- metadata +256 -0
data/lib/gush/client.rb
ADDED
@@ -0,0 +1,146 @@
|
|
1
|
+
module Gush
|
2
|
+
class Client
|
3
|
+
attr_reader :configuration
|
4
|
+
|
5
|
+
def initialize(config = Gush.configuration)
|
6
|
+
@configuration = config
|
7
|
+
@sidekiq = build_sidekiq
|
8
|
+
@redis = build_redis
|
9
|
+
load_gushfile
|
10
|
+
end
|
11
|
+
|
12
|
+
def configure
|
13
|
+
yield configuration
|
14
|
+
@sidekiq = build_sidekiq
|
15
|
+
@redis = build_redis
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_workflow(name)
|
19
|
+
id = SecureRandom.uuid.split("-").first
|
20
|
+
|
21
|
+
begin
|
22
|
+
workflow = name.constantize.new(id)
|
23
|
+
rescue NameError
|
24
|
+
raise WorkflowNotFound.new("Workflow with given name doesn't exist")
|
25
|
+
end
|
26
|
+
|
27
|
+
persist_workflow(workflow)
|
28
|
+
workflow
|
29
|
+
end
|
30
|
+
|
31
|
+
def start_workflow(id, jobs = [])
|
32
|
+
workflow = find_workflow(id)
|
33
|
+
workflow.start!
|
34
|
+
persist_workflow(workflow)
|
35
|
+
|
36
|
+
jobs = if jobs.empty?
|
37
|
+
workflow.next_jobs
|
38
|
+
else
|
39
|
+
jobs.map {|name| workflow.find_job(name) }
|
40
|
+
end
|
41
|
+
|
42
|
+
jobs.each do |job|
|
43
|
+
job.enqueue!
|
44
|
+
persist_job(workflow.id, job)
|
45
|
+
enqueue_job(workflow.id, job)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def stop_workflow(id)
|
50
|
+
workflow = find_workflow(id)
|
51
|
+
workflow.stop!
|
52
|
+
persist_workflow(workflow)
|
53
|
+
end
|
54
|
+
|
55
|
+
def all_workflows
|
56
|
+
redis.keys("gush.workflows.*").map do |key|
|
57
|
+
id = key.sub("gush.workflows.", "")
|
58
|
+
find_workflow(id)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def find_workflow(id)
|
63
|
+
data = redis.get("gush.workflows.#{id}")
|
64
|
+
unless data.nil?
|
65
|
+
hash = Yajl::Parser.parse(data, symbolize_keys: true)
|
66
|
+
keys = redis.keys("gush.jobs.#{id}.*")
|
67
|
+
nodes = redis.mget(*keys).map { |json| Yajl::Parser.parse(json, symbolize_keys: true) }
|
68
|
+
workflow_from_hash(hash, nodes)
|
69
|
+
else
|
70
|
+
raise WorkflowNotFound.new("Workflow with given id doesn't exist")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def persist_workflow(workflow)
|
75
|
+
redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
|
76
|
+
workflow.nodes.each {|job| persist_job(workflow.id, job) }
|
77
|
+
end
|
78
|
+
|
79
|
+
def persist_job(workflow_id, job)
|
80
|
+
redis.set("gush.jobs.#{workflow_id}.#{job.class.to_s}", job.to_json)
|
81
|
+
end
|
82
|
+
|
83
|
+
def destroy_workflow(workflow)
|
84
|
+
redis.del("gush.workflows.#{workflow.id}")
|
85
|
+
workflow.nodes.each {|job| destroy_job(workflow.id, job) }
|
86
|
+
end
|
87
|
+
|
88
|
+
def destroy_job(workflow_id, job)
|
89
|
+
redis.del("gush.jobs.#{workflow_id}.#{job.class.to_s}")
|
90
|
+
end
|
91
|
+
|
92
|
+
def worker_report(message)
|
93
|
+
report("gush.workers.status", message)
|
94
|
+
end
|
95
|
+
|
96
|
+
def workflow_report(message)
|
97
|
+
report("gush.workflows.status", message)
|
98
|
+
end
|
99
|
+
|
100
|
+
private
|
101
|
+
|
102
|
+
attr_reader :sidekiq, :redis
|
103
|
+
|
104
|
+
def workflow_from_hash(hash, nodes = nil)
|
105
|
+
flow = hash[:klass].constantize.new(hash[:id], configure: false)
|
106
|
+
flow.logger_builder(hash.fetch(:logger_builder, 'Gush::LoggerBuilder').constantize)
|
107
|
+
flow.stopped = hash.fetch(:stopped, false)
|
108
|
+
|
109
|
+
(nodes || hash[:nodes]).each do |node|
|
110
|
+
flow.nodes << Gush::Job.from_hash(node)
|
111
|
+
end
|
112
|
+
|
113
|
+
flow
|
114
|
+
end
|
115
|
+
|
116
|
+
def report(key, message)
|
117
|
+
redis.publish(key, Yajl::Encoder.new.encode(message))
|
118
|
+
end
|
119
|
+
|
120
|
+
def enqueue_job(workflow_id, job)
|
121
|
+
sidekiq.push(
|
122
|
+
'class' => Gush::Worker,
|
123
|
+
'queue' => configuration.namespace,
|
124
|
+
'args' => [workflow_id, job.class.to_s, configuration.to_json]
|
125
|
+
)
|
126
|
+
end
|
127
|
+
|
128
|
+
def build_sidekiq
|
129
|
+
Sidekiq::Client.new(connection_pool)
|
130
|
+
end
|
131
|
+
|
132
|
+
def build_redis
|
133
|
+
Redis.new(url: configuration.redis_url)
|
134
|
+
end
|
135
|
+
|
136
|
+
def connection_pool
|
137
|
+
ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
|
138
|
+
end
|
139
|
+
|
140
|
+
def load_gushfile
|
141
|
+
require configuration.gushfile
|
142
|
+
rescue LoadError
|
143
|
+
raise Thor::Error, "failed to load #{configuration.gushfile.basename}".colorize(:red)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'yajl'
|
2
|
+
|
3
|
+
module Gush
|
4
|
+
class Configuration
|
5
|
+
attr_accessor :concurrency, :namespace, :redis_url, :environment
|
6
|
+
|
7
|
+
def self.from_json(json)
|
8
|
+
new(Yajl::Parser.parse(json, symbolize_keys: true))
|
9
|
+
end
|
10
|
+
|
11
|
+
def initialize(hash = {})
|
12
|
+
self.concurrency = hash.fetch(:concurrency, 5)
|
13
|
+
self.namespace = hash.fetch(:namespace, 'gush')
|
14
|
+
self.redis_url = hash.fetch(:redis_url, 'redis://localhost:6379')
|
15
|
+
self.gushfile = hash.fetch(:gushfile, 'Gushfile.rb')
|
16
|
+
self.environment = hash.fetch(:environment, 'development')
|
17
|
+
end
|
18
|
+
|
19
|
+
def gushfile=(path)
|
20
|
+
@gushfile = Pathname(path)
|
21
|
+
end
|
22
|
+
|
23
|
+
def gushfile
|
24
|
+
raise Thor::Error, "#{@gushfile} not found, please add it to your project".colorize(:red) unless @gushfile.exist?
|
25
|
+
@gushfile.realpath
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_hash
|
29
|
+
{
|
30
|
+
concurrency: concurrency,
|
31
|
+
namespace: namespace,
|
32
|
+
redis_url: redis_url,
|
33
|
+
environment: environment,
|
34
|
+
gushfile: gushfile.to_path
|
35
|
+
}
|
36
|
+
end
|
37
|
+
|
38
|
+
def to_json
|
39
|
+
Yajl::Encoder.new.encode(to_hash)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/gush/errors.rb
ADDED
data/lib/gush/job.rb
ADDED
@@ -0,0 +1,161 @@
|
|
1
|
+
require 'gush/metadata'
|
2
|
+
|
3
|
+
module Gush
|
4
|
+
class Job
|
5
|
+
include Gush::Metadata
|
6
|
+
|
7
|
+
RECURSION_LIMIT = 1000
|
8
|
+
|
9
|
+
DEFAULTS = {
|
10
|
+
finished: false,
|
11
|
+
enqueued: false,
|
12
|
+
failed: false,
|
13
|
+
running: false
|
14
|
+
}
|
15
|
+
|
16
|
+
attr_accessor :finished, :enqueued, :failed, :workflow_id, :incoming, :outgoing,
|
17
|
+
:finished_at, :failed_at, :started_at, :jid, :running
|
18
|
+
|
19
|
+
attr_reader :name
|
20
|
+
|
21
|
+
attr_writer :logger
|
22
|
+
|
23
|
+
def initialize(opts = {})
|
24
|
+
options = DEFAULTS.dup.merge(opts)
|
25
|
+
assign_variables(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def as_json
|
29
|
+
{
|
30
|
+
name: @name,
|
31
|
+
klass: self.class.to_s,
|
32
|
+
finished: @finished,
|
33
|
+
enqueued: @enqueued,
|
34
|
+
failed: @failed,
|
35
|
+
incoming: @incoming,
|
36
|
+
outgoing: @outgoing,
|
37
|
+
finished_at: @finished_at,
|
38
|
+
started_at: @started_at,
|
39
|
+
failed_at: @failed_at,
|
40
|
+
running: @running
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_json(options = {})
|
45
|
+
Yajl::Encoder.new.encode(as_json)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.from_hash(hash)
|
49
|
+
hash[:klass].constantize.new(
|
50
|
+
name: hash[:name],
|
51
|
+
finished: hash[:finished],
|
52
|
+
enqueued: hash[:enqueued],
|
53
|
+
failed: hash[:failed],
|
54
|
+
incoming: hash[:incoming],
|
55
|
+
outgoing: hash[:outgoing],
|
56
|
+
failed_at: hash[:failed_at],
|
57
|
+
finished_at: hash[:finished_at],
|
58
|
+
started_at: hash[:started_at],
|
59
|
+
running: hash[:running]
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
def before_work
|
64
|
+
end
|
65
|
+
|
66
|
+
def work
|
67
|
+
end
|
68
|
+
|
69
|
+
def after_work
|
70
|
+
end
|
71
|
+
|
72
|
+
def start!
|
73
|
+
@enqueued = false
|
74
|
+
@running = true
|
75
|
+
@started_at = Time.now.to_i
|
76
|
+
end
|
77
|
+
|
78
|
+
def enqueue!
|
79
|
+
@enqueued = true
|
80
|
+
@running = false
|
81
|
+
@failed = false
|
82
|
+
@started_at = nil
|
83
|
+
@finished_at = nil
|
84
|
+
@failed_at = nil
|
85
|
+
end
|
86
|
+
|
87
|
+
def finish!
|
88
|
+
@running = false
|
89
|
+
@finished = true
|
90
|
+
@enqueued = false
|
91
|
+
@failed = false
|
92
|
+
@finished_at = Time.now.to_i
|
93
|
+
end
|
94
|
+
|
95
|
+
def fail!
|
96
|
+
@finished = true
|
97
|
+
@running = false
|
98
|
+
@failed = true
|
99
|
+
@enqueued = false
|
100
|
+
@finished_at = Time.now.to_i
|
101
|
+
@failed_at = Time.now.to_i
|
102
|
+
end
|
103
|
+
|
104
|
+
def enqueued?
|
105
|
+
!!enqueued
|
106
|
+
end
|
107
|
+
|
108
|
+
def finished?
|
109
|
+
!!finished
|
110
|
+
end
|
111
|
+
|
112
|
+
def failed?
|
113
|
+
!!failed
|
114
|
+
end
|
115
|
+
|
116
|
+
def succeeded?
|
117
|
+
finished? && !failed?
|
118
|
+
end
|
119
|
+
|
120
|
+
def running?
|
121
|
+
!!running
|
122
|
+
end
|
123
|
+
|
124
|
+
def can_be_started?(flow)
|
125
|
+
!running? &&
|
126
|
+
!enqueued? &&
|
127
|
+
!finished? &&
|
128
|
+
!failed? &&
|
129
|
+
dependencies_satisfied?(flow)
|
130
|
+
end
|
131
|
+
|
132
|
+
def dependencies(flow, level = 0)
|
133
|
+
fail DependencyLevelTooDeep if level > RECURSION_LIMIT
|
134
|
+
(incoming.map {|name| flow.find_job(name) } + incoming.flat_map{ |name| flow.find_job(name).dependencies(flow, level + 1) }).uniq
|
135
|
+
end
|
136
|
+
|
137
|
+
def logger
|
138
|
+
fail "You cannot log when the job is not running" unless running?
|
139
|
+
@logger
|
140
|
+
end
|
141
|
+
|
142
|
+
private
|
143
|
+
|
144
|
+
def assign_variables(options)
|
145
|
+
@name = options[:name]
|
146
|
+
@finished = options[:finished]
|
147
|
+
@enqueued = options[:enqueued]
|
148
|
+
@failed = options[:failed]
|
149
|
+
@incoming = options[:incoming] || []
|
150
|
+
@outgoing = options[:outgoing] || []
|
151
|
+
@failed_at = options[:failed_at]
|
152
|
+
@finished_at = options[:finished_at]
|
153
|
+
@started_at = options[:started_at]
|
154
|
+
@running = options[:running]
|
155
|
+
end
|
156
|
+
|
157
|
+
def dependencies_satisfied?(flow)
|
158
|
+
dependencies(flow).all? { |dep| !dep.enqueued? && dep.finished? && !dep.failed? }
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Gush
|
2
|
+
module Metadata
|
3
|
+
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(ClassMethods)
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def metadata(params = {})
|
10
|
+
@metadata = (@metadata || {}).merge(params)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def name
|
15
|
+
metadata[:name] || @name
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def metadata
|
21
|
+
self.class.metadata
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/gush/version.rb
ADDED
data/lib/gush/worker.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
require 'yajl'
|
3
|
+
|
4
|
+
module Gush
|
5
|
+
class Worker
|
6
|
+
include ::Sidekiq::Worker
|
7
|
+
sidekiq_options retry: false
|
8
|
+
|
9
|
+
def perform(workflow_id, job_id, configuration_json)
|
10
|
+
configure_client(configuration_json)
|
11
|
+
|
12
|
+
workflow = client.find_workflow(workflow_id)
|
13
|
+
job = workflow.find_job(job_id)
|
14
|
+
|
15
|
+
start = Time.now
|
16
|
+
report(workflow, job, :started, start)
|
17
|
+
|
18
|
+
job.logger = workflow.build_logger_for_job(job, job_id)
|
19
|
+
job.jid = jid
|
20
|
+
|
21
|
+
failed = false
|
22
|
+
error = nil
|
23
|
+
|
24
|
+
mark_as_started(workflow, job)
|
25
|
+
begin
|
26
|
+
job.before_work
|
27
|
+
job.work
|
28
|
+
job.after_work
|
29
|
+
rescue Exception => e
|
30
|
+
failed = true
|
31
|
+
error = e
|
32
|
+
end
|
33
|
+
|
34
|
+
unless failed
|
35
|
+
report(workflow, job, :finished, start)
|
36
|
+
mark_as_finished(workflow, job)
|
37
|
+
|
38
|
+
continue_workflow(workflow)
|
39
|
+
else
|
40
|
+
log_exception(job.logger, error)
|
41
|
+
mark_as_failed(workflow, job)
|
42
|
+
report(workflow, job, :failed, start, error.message)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
attr_reader :client
|
49
|
+
|
50
|
+
def configure_client(config_json)
|
51
|
+
@client = Client.new(Configuration.from_json(config_json))
|
52
|
+
end
|
53
|
+
|
54
|
+
def mark_as_finished(workflow, job)
|
55
|
+
job.finish!
|
56
|
+
client.persist_job(workflow.id, job)
|
57
|
+
end
|
58
|
+
|
59
|
+
def mark_as_failed(workflow, job)
|
60
|
+
job.fail!
|
61
|
+
client.persist_job(workflow.id, job)
|
62
|
+
end
|
63
|
+
|
64
|
+
def mark_as_started(workflow, job)
|
65
|
+
job.start!
|
66
|
+
client.persist_job(workflow.id, job)
|
67
|
+
end
|
68
|
+
|
69
|
+
def report_workflow_status(workflow, job)
|
70
|
+
message = {workflow_id: workflow.id, status: workflow.status, started_at: workflow.started_at, finished_at: workflow.finished_at }
|
71
|
+
client.workflow_report(message)
|
72
|
+
end
|
73
|
+
|
74
|
+
def report(workflow, job, status, start, error = nil)
|
75
|
+
message = {status: status, workflow_id: workflow.id, job: job.name, duration: elapsed(start)}
|
76
|
+
message[:error] = error if error
|
77
|
+
client.worker_report(message)
|
78
|
+
end
|
79
|
+
|
80
|
+
def elapsed(start)
|
81
|
+
(Time.now - start).to_f.round(3)
|
82
|
+
end
|
83
|
+
|
84
|
+
def continue_workflow(workflow)
|
85
|
+
# refetch is important to get correct workflow status
|
86
|
+
unless client.find_workflow(workflow.id).stopped?
|
87
|
+
client.start_workflow(workflow.id)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_exception(logger, exception)
|
92
|
+
first, *rest = exception.backtrace
|
93
|
+
|
94
|
+
logger << "#{first}: #{exception.message} (#{exception.class})\n"
|
95
|
+
rest.each do |line|
|
96
|
+
logger << " from #{line}\n"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|