jflow 0.2.6 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 308f589a87738e9ab38d7a714ade494978ae753e
-  data.tar.gz: 7b4769dcd0ffe32e4faa9983f23cb8a0efe693ff
+  metadata.gz: 7563030ceebcc9e516193c78494b51b36a90994d
+  data.tar.gz: 3d1e018a39774fd9678e14f69dcf320d4dfea1a5
 SHA512:
-  metadata.gz: e0223780c90d2a666ab88fa98663ee704031d09ffe8c55e9f1a56206f6763c5d583e984b8d5da1384b7e04fdcef3a37dcaaad823402a928d16aab3889b444392
-  data.tar.gz: 83afde68c71aca64663dd9dd5d2e769b50465a44cdd55ea099f3579d705a3959d0ba9b0cdc9a98af653c2c1d117328d55583de0131b92f3f842152d7534740b4
+  metadata.gz: ff699996a0efe0652c3bed720093d34cc3c3d0f5b1b0739706fde67ce292102dc00fcc1f43918554210679ee2c13de6da77b23cbb4d8bd8a0f673fb0f33fd419
+  data.tar.gz: 8e524de0665600be667b2834edd02a89dd3a1d611d867b004fbff233b16dc7ef14070fd67dc37b47696d7d5f8364323a37de024633abfb2bf6805056940be9da

data/bin/jflow_worker

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-
-require "bundler/setup"
 require "jflow"
 require "slop"
 
@@ -8,61 +6,17 @@ opts = Slop.parse do |o|
   o.string '-f', '--file', 'worker configuration file'
 end
 
-configuration = JSON.parse(File.read(opts[:file]))
-
-configuration_validator = {
-  "number_of_workers" => "integer",
-  "domain"            => "string",
-  "tasklist"          => "string",
-  "activities_path"   => "array"
-}
-
-validator = HashValidator.validate(configuration, configuration_validator)
-raise "configuration is invalid! #{validator.errors}" unless validator.valid?
-
-JFlow.configure do |c|
-  c.load_paths = configuration["activities_path"]
-  c.swf_client = Aws::SWF::Client.new
-end
-
-JFlow.load_activities
+raise "You need to specify a file!" unless opts[:file]
 
-threads = []
+configuration = JSON.parse(File.read(opts[:file]))
 
-configuration["number_of_workers"].times do |i|
-  threads << Thread.new do
-    domain   = configuration["domain"]
-    tasklist = configuration["tasklist"]
-    JFlow::Activity::Worker.new(domain, tasklist)
-                         .start!
-  end
-end
+cli = JFlow::Cli.new(configuration)
 
-def shutdown(threads)
-  JFlow.configuration.logger.info "Sending kill signal to running threads. Please wait for current polling to finish"
-  kill_threads = []
-  threads.each do |thread|
-    thread["do_exit"] = true
-    if thread["state"] == :working
-      kill_threads << Thread.new do
-        sleep 60
-        if thread.alive?
-          thread.raise("Workers are going down!")
-        end
-      end
-    end
+["INT", "TERM"].each do |signal|
+  Signal.trap(signal) do
+    cli.shutdown_workers
   end
-  kill_threads.each { |thr| thr.join }
 end
 
-Signal.trap("INT") {
-  shutdown(threads)
-}
-
-# Trap `Kill `
-Signal.trap("TERM") {
-  shutdown(threads)
-}
-
-threads.each { |thr| thr.join }
+cli.start_workers
 

data/lib/jflow/activity/task.rb

@@ -24,6 +24,14 @@ module JFlow
         task.task_token
       end
 
+      def run_id
+        task.workflow_execution.run_id
+      end
+
+      def workflow_id
+        task.workflow_execution.workflow_id
+      end
+
       def klass
         @klass_value ||= JFlow.configuration.activity_map.klass_for(name,version)
         raise "Could not find code to run for given activity" unless @klass_value
@@ -47,6 +55,7 @@ module JFlow
       end
 
       def completed!(result)
+        log "Task Completed"
         swf_client.respond_activity_task_completed({
           task_token: token,
           result: result,
@@ -55,6 +64,7 @@ module JFlow
 
 
       def failed!(exception)
+        log "Task Failed #{exception.message}"
         swf_client.respond_activity_task_failed({
           task_token: token,
           reason: exception.message,

data/lib/jflow/activity/worker.rb

@@ -19,11 +19,11 @@ module JFlow
 
 
       def poll
-        Thread.current["state"] = :polling
+        Thread.current.set_state(:polling)
         response = JFlow.configuration.swf_client.poll_for_activity_task(poll_params)
         if response.task_token
-          log "Got task #{response.task_token}"
           task = JFlow::Activity::Task.new(response)
+          log "Got task #{task.workflow_id}-#{task.run_id}"
           if should_be_working?
             process(task)
           else
@@ -38,7 +38,7 @@ module JFlow
 
       def process(task)
         begin
-          Thread.current["state"] = :working
+          Thread.current.set_state(:working)
           task.run!
         rescue => exception
           task.failed!(exception)
@@ -66,7 +66,7 @@ module JFlow
       end
 
       def should_be_working?
-        Thread.current["do_exit"] != true
+        !Thread.current.marked_for_shutdown?
       end
 
     end

data/lib/jflow/cli.rb

@@ -0,0 +1,94 @@
+module JFlow
+  class Cli
+
+    VALIDATION = {
+      "number_of_workers" => "integer",
+      "domain"            => "string",
+      "tasklist"          => "string",
+      "activities_path"   => "array"
+    }
+
+    attr_reader :number_of_workers, :domain, :tasklist, :worker_threads, :activities_path
+
+    def initialize(options)
+      validate_options(options)
+      @number_of_workers  = options["number_of_workers"]
+      @domain             = options["domain"]
+      @tasklist           = options["tasklist"]
+      @activities_path    = options["activities_path"]
+      @worker_threads     = []
+      setup
+    end
+
+    def start_workers
+      number_of_workers.times do
+        worker_threads << worker_thread
+      end
+      worker_threads.each(&:join)
+    end
+
+    # here we want to handle all cases for clean kill of the workers.
+    # there is two state on which the workers can be
+    # - Polling : they are not working on anything and are just waiting for a task
+    # - Working : they are processing an activity right now
+    #
+    # for Polling, we cannot stop it in flight, so we send a signal to stop
+    # polling after the current long poll finishes, IF it picks up anything in the mean time
+    # the activity will be force failed
+    #
+    # for Working, we give a grace period of 60 seconds to finish otherwise we just send
+    # an exception to the thread to force the failure.
+    #
+    # Shutting down workers will take a exactly 60 secs in all cases.
+    def shutdown_workers
+      log "Sending kill signal to running threads. Please wait for current polling to finish"
+      kill_threads = []
+      worker_threads.each do |thread|
+        thread.mark_for_shutdown
+        if thread.currently_working?
+            kill_threads << kill_thread(thread)
+        end
+      end
+      kill_threads.each(&:join)
+    end
+
+    private
+
+    def setup
+      JFlow.configure do |c|
+        c.load_paths = activities_path
+        c.swf_client = Aws::SWF::Client.new
+      end
+      JFlow.load_activities
+    end
+
+    def kill_thread(thread)
+      Thread.new do
+        sleep 60
+        if thread.alive?
+          thread.raise("Workers are going down!")
+        end
+      end
+    end
+
+    def log(str)
+      JFlow.configuration.logger.info str
+    end
+
+    def worker_thread
+      JFlow::WorkerThread.new do
+        worker.start!
+      end
+    end
+
+    def worker
+      JFlow::Activity::Worker.new(domain, tasklist)
+    end
+
+    def validate_options(options)
+      validator = HashValidator.validate(options, VALIDATION)
+      raise "configuration is invalid! #{validator.errors}" unless validator.valid?
+    end
+
+  end
+end

data/lib/jflow/version.rb

@@ -1,3 +1,3 @@
 module JFlow
-  VERSION = "0.2.6"
+  VERSION = "0.2.8"
 end

data/lib/jflow/worker_thread.rb

@@ -0,0 +1,25 @@
+module JFlow
+  class WorkerThread < Thread
+
+    def mark_for_shutdown
+      self["marked_for_shutdown"] = true
+    end
+
+    def marked_for_shutdown?
+      self["marked_for_shutdown"] == true
+    end
+
+    def currently_working?
+      self["state"] == :working
+    end
+
+    def currently_polling?
+      self["state"] == :polling
+    end
+
+    def set_state(state)
+      self["state"] = state
+    end
+
+  end
+end

data/lib/jflow.rb

@@ -10,6 +10,8 @@ require "jflow/activity/mixin.rb"
 require "jflow/activity/task.rb"
 require "jflow/activity/map.rb"
 require "jflow/activity/worker.rb"
+require "jflow/worker_thread.rb"
+require "jflow/cli.rb"
 
 module JFlow
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jflow
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.8
 platform: ruby
 authors:
 - Christophe Verbinnen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-18 00:00:00.000000000 Z
+date: 2016-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -134,8 +134,10 @@ files:
 - lib/jflow/activity/mixin.rb
 - lib/jflow/activity/task.rb
 - lib/jflow/activity/worker.rb
+- lib/jflow/cli.rb
 - lib/jflow/configuration.rb
 - lib/jflow/version.rb
+- lib/jflow/worker_thread.rb
 homepage: https://github.com/djpate/jflow
 licenses: []
 metadata: