jflow 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/jflow_worker +7 -53
- data/lib/jflow/activity/task.rb +10 -0
- data/lib/jflow/activity/worker.rb +4 -4
- data/lib/jflow/cli.rb +94 -0
- data/lib/jflow/version.rb +1 -1
- data/lib/jflow/worker_thread.rb +25 -0
- data/lib/jflow.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7563030ceebcc9e516193c78494b51b36a90994d
|
4
|
+
data.tar.gz: 3d1e018a39774fd9678e14f69dcf320d4dfea1a5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ff699996a0efe0652c3bed720093d34cc3c3d0f5b1b0739706fde67ce292102dc00fcc1f43918554210679ee2c13de6da77b23cbb4d8bd8a0f673fb0f33fd419
|
7
|
+
data.tar.gz: 8e524de0665600be667b2834edd02a89dd3a1d611d867b004fbff233b16dc7ef14070fd67dc37b47696d7d5f8364323a37de024633abfb2bf6805056940be9da
|
data/bin/jflow_worker
CHANGED
@@ -1,6 +1,4 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require "bundler/setup"
|
4
2
|
require "jflow"
|
5
3
|
require "slop"
|
6
4
|
|
@@ -8,61 +6,17 @@ opts = Slop.parse do |o|
|
|
8
6
|
o.string '-f', '--file', 'worker configuration file'
|
9
7
|
end
|
10
8
|
|
11
|
-
|
12
|
-
|
13
|
-
configuration_validator = {
|
14
|
-
"number_of_workers" => "integer",
|
15
|
-
"domain" => "string",
|
16
|
-
"tasklist" => "string",
|
17
|
-
"activities_path" => "array"
|
18
|
-
}
|
19
|
-
|
20
|
-
validator = HashValidator.validate(configuration, configuration_validator)
|
21
|
-
raise "configuration is invalid! #{validator.errors}" unless validator.valid?
|
22
|
-
|
23
|
-
JFlow.configure do |c|
|
24
|
-
c.load_paths = configuration["activities_path"]
|
25
|
-
c.swf_client = Aws::SWF::Client.new
|
26
|
-
end
|
27
|
-
|
28
|
-
JFlow.load_activities
|
9
|
+
raise "You need to specify a file!" unless opts[:file]
|
29
10
|
|
30
|
-
|
11
|
+
configuration = JSON.parse(File.read(opts[:file]))
|
31
12
|
|
32
|
-
|
33
|
-
threads << Thread.new do
|
34
|
-
domain = configuration["domain"]
|
35
|
-
tasklist = configuration["tasklist"]
|
36
|
-
JFlow::Activity::Worker.new(domain, tasklist)
|
37
|
-
.start!
|
38
|
-
end
|
39
|
-
end
|
13
|
+
cli = JFlow::Cli.new(configuration)
|
40
14
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
threads.each do |thread|
|
45
|
-
thread["do_exit"] = true
|
46
|
-
if thread["state"] == :working
|
47
|
-
kill_threads << Thread.new do
|
48
|
-
sleep 60
|
49
|
-
if thread.alive?
|
50
|
-
thread.raise("Workers are going down!")
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
15
|
+
["INT", "TERM"].each do |signal|
|
16
|
+
Signal.trap(signal) do
|
17
|
+
cli.shutdown_workers
|
54
18
|
end
|
55
|
-
kill_threads.each { |thr| thr.join }
|
56
19
|
end
|
57
20
|
|
58
|
-
|
59
|
-
shutdown(threads)
|
60
|
-
}
|
61
|
-
|
62
|
-
# Trap `Kill `
|
63
|
-
Signal.trap("TERM") {
|
64
|
-
shutdown(threads)
|
65
|
-
}
|
66
|
-
|
67
|
-
threads.each { |thr| thr.join }
|
21
|
+
cli.start_workers
|
68
22
|
|
data/lib/jflow/activity/task.rb
CHANGED
@@ -24,6 +24,14 @@ module JFlow
|
|
24
24
|
task.task_token
|
25
25
|
end
|
26
26
|
|
27
|
+
def run_id
|
28
|
+
task.workflow_execution.run_id
|
29
|
+
end
|
30
|
+
|
31
|
+
def workflow_id
|
32
|
+
task.workflow_execution.workflow_id
|
33
|
+
end
|
34
|
+
|
27
35
|
def klass
|
28
36
|
@klass_value ||= JFlow.configuration.activity_map.klass_for(name,version)
|
29
37
|
raise "Could not find code to run for given activity" unless @klass_value
|
@@ -47,6 +55,7 @@ module JFlow
|
|
47
55
|
end
|
48
56
|
|
49
57
|
def completed!(result)
|
58
|
+
log "Task Completed"
|
50
59
|
swf_client.respond_activity_task_completed({
|
51
60
|
task_token: token,
|
52
61
|
result: result,
|
@@ -55,6 +64,7 @@ module JFlow
|
|
55
64
|
|
56
65
|
|
57
66
|
def failed!(exception)
|
67
|
+
log "Task Failed #{exception.message}"
|
58
68
|
swf_client.respond_activity_task_failed({
|
59
69
|
task_token: token,
|
60
70
|
reason: exception.message,
|
@@ -19,11 +19,11 @@ module JFlow
|
|
19
19
|
|
20
20
|
|
21
21
|
def poll
|
22
|
-
Thread.current
|
22
|
+
Thread.current.set_state(:polling)
|
23
23
|
response = JFlow.configuration.swf_client.poll_for_activity_task(poll_params)
|
24
24
|
if response.task_token
|
25
|
-
log "Got task #{response.task_token}"
|
26
25
|
task = JFlow::Activity::Task.new(response)
|
26
|
+
log "Got task #{task.workflow_id}-#{task.run_id}"
|
27
27
|
if should_be_working?
|
28
28
|
process(task)
|
29
29
|
else
|
@@ -38,7 +38,7 @@ module JFlow
|
|
38
38
|
|
39
39
|
def process(task)
|
40
40
|
begin
|
41
|
-
Thread.current
|
41
|
+
Thread.current.set_state(:working)
|
42
42
|
task.run!
|
43
43
|
rescue => exception
|
44
44
|
task.failed!(exception)
|
@@ -66,7 +66,7 @@ module JFlow
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def should_be_working?
|
69
|
-
Thread.current
|
69
|
+
!Thread.current.marked_for_shutdown?
|
70
70
|
end
|
71
71
|
|
72
72
|
end
|
data/lib/jflow/cli.rb
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
module JFlow
|
2
|
+
class Cli
|
3
|
+
|
4
|
+
VALIDATION = {
|
5
|
+
"number_of_workers" => "integer",
|
6
|
+
"domain" => "string",
|
7
|
+
"tasklist" => "string",
|
8
|
+
"activities_path" => "array"
|
9
|
+
}
|
10
|
+
|
11
|
+
attr_reader :number_of_workers, :domain, :tasklist, :worker_threads, :activities_path
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
validate_options(options)
|
15
|
+
@number_of_workers = options["number_of_workers"]
|
16
|
+
@domain = options["domain"]
|
17
|
+
@tasklist = options["tasklist"]
|
18
|
+
@activities_path = options["activities_path"]
|
19
|
+
@worker_threads = []
|
20
|
+
setup
|
21
|
+
end
|
22
|
+
|
23
|
+
def start_workers
|
24
|
+
number_of_workers.times do
|
25
|
+
worker_threads << worker_thread
|
26
|
+
end
|
27
|
+
worker_threads.each(&:join)
|
28
|
+
end
|
29
|
+
|
30
|
+
# here we want to handle all cases for clean kill of the workers.
|
31
|
+
# there is two state on which the workers can be
|
32
|
+
# - Polling : they are not working on anything and are just waiting for a task
|
33
|
+
# - Working : they are processing an activity right now
|
34
|
+
#
|
35
|
+
# for Polling, we cannot stop it in flight, so we send a signal to stop
|
36
|
+
# polling after the current long poll finishes, IF it picks up anything in the mean time
|
37
|
+
# the activity will be force failed
|
38
|
+
#
|
39
|
+
# for Working, we give a grace period of 60 seconds to finish otherwise we just send
|
40
|
+
# an exception to the thread to force the failure.
|
41
|
+
#
|
42
|
+
# Shutting down workers will take a exactly 60 secs in all cases.
|
43
|
+
def shutdown_workers
|
44
|
+
log "Sending kill signal to running threads. Please wait for current polling to finish"
|
45
|
+
kill_threads = []
|
46
|
+
worker_threads.each do |thread|
|
47
|
+
thread.mark_for_shutdown
|
48
|
+
if thread.currently_working?
|
49
|
+
kill_threads << kill_thread(thread)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
kill_threads.each(&:join)
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def setup
|
58
|
+
JFlow.configure do |c|
|
59
|
+
c.load_paths = activities_path
|
60
|
+
c.swf_client = Aws::SWF::Client.new
|
61
|
+
end
|
62
|
+
JFlow.load_activities
|
63
|
+
end
|
64
|
+
|
65
|
+
def kill_thread(thread)
|
66
|
+
Thread.new do
|
67
|
+
sleep 60
|
68
|
+
if thread.alive?
|
69
|
+
thread.raise("Workers are going down!")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def log(str)
|
75
|
+
JFlow.configuration.logger.info str
|
76
|
+
end
|
77
|
+
|
78
|
+
def worker_thread
|
79
|
+
JFlow::WorkerThread.new do
|
80
|
+
worker.start!
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def worker
|
85
|
+
JFlow::Activity::Worker.new(domain, tasklist)
|
86
|
+
end
|
87
|
+
|
88
|
+
def validate_options(options)
|
89
|
+
validator = HashValidator.validate(options, VALIDATION)
|
90
|
+
raise "configuration is invalid! #{validator.errors}" unless validator.valid?
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
end
|
data/lib/jflow/version.rb
CHANGED
@@ -0,0 +1,25 @@
|
|
1
|
+
module JFlow
|
2
|
+
class WorkerThread < Thread
|
3
|
+
|
4
|
+
def mark_for_shutdown
|
5
|
+
self["marked_for_shutdown"] = true
|
6
|
+
end
|
7
|
+
|
8
|
+
def marked_for_shutdown?
|
9
|
+
self["marked_for_shutdown"] == true
|
10
|
+
end
|
11
|
+
|
12
|
+
def currently_working?
|
13
|
+
self["state"] == :working
|
14
|
+
end
|
15
|
+
|
16
|
+
def currently_polling?
|
17
|
+
self["state"] == :polling
|
18
|
+
end
|
19
|
+
|
20
|
+
def set_state(state)
|
21
|
+
self["state"] = state
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
data/lib/jflow.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christophe Verbinnen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: slop
|
@@ -134,8 +134,10 @@ files:
|
|
134
134
|
- lib/jflow/activity/mixin.rb
|
135
135
|
- lib/jflow/activity/task.rb
|
136
136
|
- lib/jflow/activity/worker.rb
|
137
|
+
- lib/jflow/cli.rb
|
137
138
|
- lib/jflow/configuration.rb
|
138
139
|
- lib/jflow/version.rb
|
140
|
+
- lib/jflow/worker_thread.rb
|
139
141
|
homepage: https://github.com/djpate/jflow
|
140
142
|
licenses: []
|
141
143
|
metadata:
|