jflow 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 308f589a87738e9ab38d7a714ade494978ae753e
4
- data.tar.gz: 7b4769dcd0ffe32e4faa9983f23cb8a0efe693ff
3
+ metadata.gz: 7563030ceebcc9e516193c78494b51b36a90994d
4
+ data.tar.gz: 3d1e018a39774fd9678e14f69dcf320d4dfea1a5
5
5
  SHA512:
6
- metadata.gz: e0223780c90d2a666ab88fa98663ee704031d09ffe8c55e9f1a56206f6763c5d583e984b8d5da1384b7e04fdcef3a37dcaaad823402a928d16aab3889b444392
7
- data.tar.gz: 83afde68c71aca64663dd9dd5d2e769b50465a44cdd55ea099f3579d705a3959d0ba9b0cdc9a98af653c2c1d117328d55583de0131b92f3f842152d7534740b4
6
+ metadata.gz: ff699996a0efe0652c3bed720093d34cc3c3d0f5b1b0739706fde67ce292102dc00fcc1f43918554210679ee2c13de6da77b23cbb4d8bd8a0f673fb0f33fd419
7
+ data.tar.gz: 8e524de0665600be667b2834edd02a89dd3a1d611d867b004fbff233b16dc7ef14070fd67dc37b47696d7d5f8364323a37de024633abfb2bf6805056940be9da
data/bin/jflow_worker CHANGED
@@ -1,6 +1,4 @@
1
1
  #!/usr/bin/env ruby
2
-
3
- require "bundler/setup"
4
2
  require "jflow"
5
3
  require "slop"
6
4
 
@@ -8,61 +6,17 @@ opts = Slop.parse do |o|
8
6
  o.string '-f', '--file', 'worker configuration file'
9
7
  end
10
8
 
11
- configuration = JSON.parse(File.read(opts[:file]))
12
-
13
- configuration_validator = {
14
- "number_of_workers" => "integer",
15
- "domain" => "string",
16
- "tasklist" => "string",
17
- "activities_path" => "array"
18
- }
19
-
20
- validator = HashValidator.validate(configuration, configuration_validator)
21
- raise "configuration is invalid! #{validator.errors}" unless validator.valid?
22
-
23
- JFlow.configure do |c|
24
- c.load_paths = configuration["activities_path"]
25
- c.swf_client = Aws::SWF::Client.new
26
- end
27
-
28
- JFlow.load_activities
9
+ raise "You need to specify a file!" unless opts[:file]
29
10
 
30
- threads = []
11
+ configuration = JSON.parse(File.read(opts[:file]))
31
12
 
32
- configuration["number_of_workers"].times do |i|
33
- threads << Thread.new do
34
- domain = configuration["domain"]
35
- tasklist = configuration["tasklist"]
36
- JFlow::Activity::Worker.new(domain, tasklist)
37
- .start!
38
- end
39
- end
13
+ cli = JFlow::Cli.new(configuration)
40
14
 
41
- def shutdown(threads)
42
- JFlow.configuration.logger.info "Sending kill signal to running threads. Please wait for current polling to finish"
43
- kill_threads = []
44
- threads.each do |thread|
45
- thread["do_exit"] = true
46
- if thread["state"] == :working
47
- kill_threads << Thread.new do
48
- sleep 60
49
- if thread.alive?
50
- thread.raise("Workers are going down!")
51
- end
52
- end
53
- end
15
+ ["INT", "TERM"].each do |signal|
16
+ Signal.trap(signal) do
17
+ cli.shutdown_workers
54
18
  end
55
- kill_threads.each { |thr| thr.join }
56
19
  end
57
20
 
58
- Signal.trap("INT") {
59
- shutdown(threads)
60
- }
61
-
62
- # Trap `Kill `
63
- Signal.trap("TERM") {
64
- shutdown(threads)
65
- }
66
-
67
- threads.each { |thr| thr.join }
21
+ cli.start_workers
68
22
 
@@ -24,6 +24,14 @@ module JFlow
24
24
  task.task_token
25
25
  end
26
26
 
27
+ def run_id
28
+ task.workflow_execution.run_id
29
+ end
30
+
31
+ def workflow_id
32
+ task.workflow_execution.workflow_id
33
+ end
34
+
27
35
  def klass
28
36
  @klass_value ||= JFlow.configuration.activity_map.klass_for(name,version)
29
37
  raise "Could not find code to run for given activity" unless @klass_value
@@ -47,6 +55,7 @@ module JFlow
47
55
  end
48
56
 
49
57
  def completed!(result)
58
+ log "Task Completed"
50
59
  swf_client.respond_activity_task_completed({
51
60
  task_token: token,
52
61
  result: result,
@@ -55,6 +64,7 @@ module JFlow
55
64
 
56
65
 
57
66
  def failed!(exception)
67
+ log "Task Failed #{exception.message}"
58
68
  swf_client.respond_activity_task_failed({
59
69
  task_token: token,
60
70
  reason: exception.message,
@@ -19,11 +19,11 @@ module JFlow
19
19
 
20
20
 
21
21
  def poll
22
- Thread.current["state"] = :polling
22
+ Thread.current.set_state(:polling)
23
23
  response = JFlow.configuration.swf_client.poll_for_activity_task(poll_params)
24
24
  if response.task_token
25
- log "Got task #{response.task_token}"
26
25
  task = JFlow::Activity::Task.new(response)
26
+ log "Got task #{task.workflow_id}-#{task.run_id}"
27
27
  if should_be_working?
28
28
  process(task)
29
29
  else
@@ -38,7 +38,7 @@ module JFlow
38
38
 
39
39
  def process(task)
40
40
  begin
41
- Thread.current["state"] = :working
41
+ Thread.current.set_state(:working)
42
42
  task.run!
43
43
  rescue => exception
44
44
  task.failed!(exception)
@@ -66,7 +66,7 @@ module JFlow
66
66
  end
67
67
 
68
68
  def should_be_working?
69
- Thread.current["do_exit"] != true
69
+ !Thread.current.marked_for_shutdown?
70
70
  end
71
71
 
72
72
  end
data/lib/jflow/cli.rb ADDED
@@ -0,0 +1,94 @@
1
+ module JFlow
2
+ class Cli
3
+
4
+ VALIDATION = {
5
+ "number_of_workers" => "integer",
6
+ "domain" => "string",
7
+ "tasklist" => "string",
8
+ "activities_path" => "array"
9
+ }
10
+
11
+ attr_reader :number_of_workers, :domain, :tasklist, :worker_threads, :activities_path
12
+
13
+ def initialize(options)
14
+ validate_options(options)
15
+ @number_of_workers = options["number_of_workers"]
16
+ @domain = options["domain"]
17
+ @tasklist = options["tasklist"]
18
+ @activities_path = options["activities_path"]
19
+ @worker_threads = []
20
+ setup
21
+ end
22
+
23
+ def start_workers
24
+ number_of_workers.times do
25
+ worker_threads << worker_thread
26
+ end
27
+ worker_threads.each(&:join)
28
+ end
29
+
30
+ # here we want to handle all cases for clean kill of the workers.
31
+ # there is two state on which the workers can be
32
+ # - Polling : they are not working on anything and are just waiting for a task
33
+ # - Working : they are processing an activity right now
34
+ #
35
+ # for Polling, we cannot stop it in flight, so we send a signal to stop
36
+ # polling after the current long poll finishes, IF it picks up anything in the mean time
37
+ # the activity will be force failed
38
+ #
39
+ # for Working, we give a grace period of 60 seconds to finish otherwise we just send
40
+ # an exception to the thread to force the failure.
41
+ #
42
+ # Shutting down workers will take a exactly 60 secs in all cases.
43
+ def shutdown_workers
44
+ log "Sending kill signal to running threads. Please wait for current polling to finish"
45
+ kill_threads = []
46
+ worker_threads.each do |thread|
47
+ thread.mark_for_shutdown
48
+ if thread.currently_working?
49
+ kill_threads << kill_thread(thread)
50
+ end
51
+ end
52
+ kill_threads.each(&:join)
53
+ end
54
+
55
+ private
56
+
57
+ def setup
58
+ JFlow.configure do |c|
59
+ c.load_paths = activities_path
60
+ c.swf_client = Aws::SWF::Client.new
61
+ end
62
+ JFlow.load_activities
63
+ end
64
+
65
+ def kill_thread(thread)
66
+ Thread.new do
67
+ sleep 60
68
+ if thread.alive?
69
+ thread.raise("Workers are going down!")
70
+ end
71
+ end
72
+ end
73
+
74
+ def log(str)
75
+ JFlow.configuration.logger.info str
76
+ end
77
+
78
+ def worker_thread
79
+ JFlow::WorkerThread.new do
80
+ worker.start!
81
+ end
82
+ end
83
+
84
+ def worker
85
+ JFlow::Activity::Worker.new(domain, tasklist)
86
+ end
87
+
88
+ def validate_options(options)
89
+ validator = HashValidator.validate(options, VALIDATION)
90
+ raise "configuration is invalid! #{validator.errors}" unless validator.valid?
91
+ end
92
+
93
+ end
94
+ end
data/lib/jflow/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module JFlow
2
- VERSION = "0.2.6"
2
+ VERSION = "0.2.8"
3
3
  end
@@ -0,0 +1,25 @@
1
+ module JFlow
2
+ class WorkerThread < Thread
3
+
4
+ def mark_for_shutdown
5
+ self["marked_for_shutdown"] = true
6
+ end
7
+
8
+ def marked_for_shutdown?
9
+ self["marked_for_shutdown"] == true
10
+ end
11
+
12
+ def currently_working?
13
+ self["state"] == :working
14
+ end
15
+
16
+ def currently_polling?
17
+ self["state"] == :polling
18
+ end
19
+
20
+ def set_state(state)
21
+ self["state"] = state
22
+ end
23
+
24
+ end
25
+ end
data/lib/jflow.rb CHANGED
@@ -10,6 +10,8 @@ require "jflow/activity/mixin.rb"
10
10
  require "jflow/activity/task.rb"
11
11
  require "jflow/activity/map.rb"
12
12
  require "jflow/activity/worker.rb"
13
+ require "jflow/worker_thread.rb"
14
+ require "jflow/cli.rb"
13
15
 
14
16
  module JFlow
15
17
  class << self
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.6
4
+ version: 0.2.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christophe Verbinnen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-18 00:00:00.000000000 Z
11
+ date: 2016-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: slop
@@ -134,8 +134,10 @@ files:
134
134
  - lib/jflow/activity/mixin.rb
135
135
  - lib/jflow/activity/task.rb
136
136
  - lib/jflow/activity/worker.rb
137
+ - lib/jflow/cli.rb
137
138
  - lib/jflow/configuration.rb
138
139
  - lib/jflow/version.rb
140
+ - lib/jflow/worker_thread.rb
139
141
  homepage: https://github.com/djpate/jflow
140
142
  licenses: []
141
143
  metadata: