aws-flow 1.2.0 → 1.3.0

data/aws-flow.gemspec

@@ -4,9 +4,11 @@ Gem::Specification.new do |s|
   s.name        = 'aws-flow'
   s.version     = AWS::Flow::version
   s.date        = Time.now
-  s.summary     = "AWS Flow Decider package decider"
+  s.summary     = "AWS Flow Framework for Ruby"
   s.description = "Library to provide the AWS Flow Framework for Ruby"
-  s.authors     = "Michael Steger"
+  s.authors     = "Michael Steger, Paritosh Mohan, Jacques Thomas"
+  s.executables = ["aws-flow-ruby"]
+  s.homepage 	= "https://aws.amazon.com/swf/details/flow/"
   s.email       = ''
   s.files       = `git ls-files`.split("\n").reject {|file| file =~ /aws-flow-core/}
   s.require_paths << "lib/aws/"

data/bin/aws-flow-ruby

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'runner'
+
+AWS::Flow::Runner.main()

data/lib/aws/decider.rb

@@ -20,7 +20,7 @@ require 'aws-sdk'
 require 'securerandom'
 
 # Setting the user-agent as ruby-flow for all calls to the service
-AWS.config(:user_agent_prefix => "ruby-flow")
+AWS.config(:user_agent_prefix => "ruby-flow") unless AWS.config.user_agent_prefix
 
 require "aws/decider/utilities"
 require "aws/decider/worker"

data/lib/aws/decider/version.rb

@@ -16,7 +16,7 @@
 module AWS
   module Flow
     def self.version
-      "1.2.0"
+      "1.3.0"
     end
   end
 end

data/lib/aws/runner.rb

@@ -0,0 +1,300 @@
+module AWS
+  module Flow
+    module Runner
+
+      # import the necessary gems to run Ruby Flow code
+      require 'aws/decider'
+      include AWS::Flow
+      require 'json'
+      require 'optparse'
+      require 'socket'
+
+
+      ##
+      ## Helper to start workflow and activity workers according to a predefined
+      ## JSON file format that decribes where to find the required elements
+      ## 
+
+      # Example of the format:
+      # {
+      #     "domains": [
+      #        {
+      #            "name": <name_of_the_domain>,
+      #            "retention_in_days": <days>
+      #        }
+      #         //, ... can add more
+      #     ],
+      #     "activity_workers": [
+      #         
+      #        {
+      #             "domain": <name_of_the_domain>,
+      #             "task_list": <name_of_the_task_list>,
+      #             "activity_classes": [ <name_of_class_containing_the_activities_to_be_worked_on> ],
+      #             "number_of_workers": <number_of_activity_workers_to_spawn>,
+      #             "number_of_forks_per_worker": <number_of_forked_workers>
+      #         }
+      #         //, ... can add more
+      #     ],
+      #     "workflow_workers": [
+      #         {
+      #             "domain": <name_of_the_domain>,
+      #             "task_list": <name_of_the_task_list>,
+      #             "workflow_classes": [ <name_of_class_containing_the_workflows_to_be_worked_on> ],
+      #             "number_of_workers": <number_of_workflow_workers_to_spawn>
+      #         }
+      #         //, ... can add more
+      #     ],
+      #     // Configure which files are 'require'd in order to load the classes
+      #     "workflow_paths": [
+      #         "lib/workflow.rb"
+      #     ],
+      #     "activity_paths": [
+      #         "lib/activity.rb"
+      #     ],
+      #     // This is used by the opsworks recipe
+      #     "user_agent_prefix" : "ruby-flow-opsworks"
+      # }
+
+
+      # registers the domains if they are not
+      def self.setup_domains(json_config)
+
+        swf = create_service_client(json_config)
+
+        json_config['domains'].each do |d|
+          begin
+            swf.client.describe_domain :name => d['name']
+          rescue
+            swf.client.register_domain( { :name => d['name'], 
+                                          :workflow_execution_retention_period_in_days => d['retention_in_days'].to_s })
+          end
+        end
+      end
+
+      def self.set_process_name(name)
+        $0 = name
+      end
+
+      # searches the object space for all subclasses of clazz
+      def self.all_subclasses(clazz)
+        ObjectSpace.each_object(Class).select { |klass| klass.is_a? clazz }
+      end
+
+      # used to extract and validate the 'activity_classes'
+      # and 'workflow_classes' fields from the config, or autodiscover 
+      # subclasses in the ObjectSpace
+      def self.get_classes(json_fragment, what)
+        classes = json_fragment[what[:config_key]]
+        if classes.nil? || classes.empty? then
+          # discover the classes
+          classes = all_subclasses( what[:clazz] )
+        else
+          # constantize the class names we just read from the config
+          classes.map! { |c| Object.const_get(c) }
+        end
+        if classes.nil? || classes.empty? then
+          raise ArgumentError.new "need at least one implementation class"
+        end
+        classes
+      end
+
+      # used to add implementations to workers; see get_classes
+      def self.add_implementations(worker, json_fragment, what)
+        classes = get_classes(json_fragment, what)
+        classes.each { |c| worker.add_implementation(c) }
+      end
+
+      def self.spawn_and_start_workers(json_fragment, process_name, worker)
+        workers = []
+        json_fragment['number_of_workers'].times do
+          workers << fork do
+            set_process_name(process_name)
+            worker.start()
+          end
+        end
+        workers
+      end
+
+      # used to support host-specific task lists
+      # when the string "|hostname|" is found in the task list
+      # it is replaced by the host name
+      def self.expand_task_list(value)
+        raise ArgumentError.new unless value
+        ret = value
+        ret.gsub!("|hostname|", Socket.gethostname)
+        ret
+      end
+
+      def self.is_empty_field?(json_fragment, field_name)
+        field = json_fragment[field_name]
+        field.nil? || field.empty?
+      end
+
+      # This is used to issue the necessary "require" commands to 
+      # load the code needed to run a module
+      #
+      # config_path: the path where the config file is, to be able to 
+      #     resolve relative references
+      # json_config: the content of the config
+      # what: what should loaded. This is a hash expected to contain two keys:
+      #     - :default_file : the file to load unless a specific list is provided
+      #     - :config_key : the key of the config element which can contain a
+      #             specific list of files to load
+      def self.load_files(config_path, json_config, what)
+        if is_empty_field?(json_config, what[:config_key]) then 
+          file = File.join(File.dirname(config_path), what[:default_file])
+          require file if File.exists? file
+        else
+          json_config[what[:config_key]].each { |file| require file if File.exists? file }
+        end
+      end
+
+      def self.start_activity_workers(swf, config_path, json_config)
+        workers = []
+        # load all classes for the activities
+        load_files(config_path, json_config, {:config_key => 'activity_paths',
+                     :default_file => File.join('flow', 'activities.rb')})
+
+        # TODO: logger
+        # start the workers for each spec
+        json_config['activity_workers'].each do |w|
+          fork_count = w['number_of_forks_per_worker'] || 1
+          domain = AWS::SimpleWorkflow::Domain.new( w['domain'] )
+          task_list = expand_task_list(w['task_list'])
+
+          # create a worker
+          worker = ActivityWorker.new(swf.client, domain, task_list, *w['activities']) {{ :max_workers => fork_count }}
+          add_implementations(worker, w, {:config_key => 'activity_classes',
+                     :clazz => AWS::Flow::Activities})
+
+          # start as many workers as desired in child processes
+          workers << spawn_and_start_workers(w, "activity-worker", worker)
+        end
+
+        return workers
+      end
+
+      def self.start_workflow_workers(swf, config_path, json_config)
+        workers = []
+        # load all the classes for the workflows
+        load_files(config_path, json_config, {:config_key => 'workflow_paths',
+                     :default_file => File.join('flow', 'workflows.rb')})
+
+        # TODO: logger
+        # start the workers for each spec
+        json_config['workflow_workers'].each do |w|
+          domain = AWS::SimpleWorkflow::Domain.new( w['domain'] )
+          task_list = expand_task_list(w['task_list'])
+
+          # create a worker
+          worker = WorkflowWorker.new(swf.client, domain, task_list, *w['workflows'])
+          add_implementations(worker, w, {:config_key => 'workflow_classes',
+                     :clazz => AWS::Flow::Workflows})
+
+          # start as many workers as desired in child processes
+          workers << spawn_and_start_workers(w, "workflow-worker", worker)
+        end
+
+        return workers
+      end
+
+      def self.create_service_client(json_config)
+        # set the UserAgent prefix for all clients
+        if json_config['user_agent_prefix'] then
+          AWS.config(:user_agent_prefix => json_config['user_agent_prefix'])
+        end
+        
+        swf = AWS::SimpleWorkflow.new
+      end
+
+      #
+      # this will start all the workers and return an array of pids for the worker
+      # processes
+      # 
+      def self.start_workers(config_path, json_config)
+        
+        workers = []
+        
+        swf = create_service_client(json_config)
+
+        workers << start_activity_workers(swf, config_path, json_config)
+        workers << start_workflow_workers(swf, config_path, json_config)
+
+        # needed to avoid returning nested arrays based on the calls above
+        workers.flatten!
+
+      end
+
+      # setup forwarding of signals to child processes, to facilitate and support
+      # orderly shutdown
+      def self.setup_signal_handling(workers)
+        Signal.trap("INT") { workers.each { |w| Process.kill("INT", w) }  }
+      end
+
+      # TODO: use a logger
+      # this will wait until all the child workers have died
+      def self.wait_for_child_processes(workers)
+        until workers.empty?
+          puts "waiting on workers " + workers.to_s + " to complete"
+          dead_guys = Process.waitall
+          dead_guys.each { |pid, status| workers.delete(pid); puts pid.to_s + " exited" }
+        end
+      end
+
+      # this is used to extend the load path so that the 'require' 
+      # of workflow and activity implementation files can succeed
+      # before adding the implementation classes to the workers
+      def self.add_dir_to_load_path(path)
+        raise ArgumentError.new("Invalid directory path: \"" + path.to_s + "\"") if not FileTest.directory? path
+        $LOAD_PATH.unshift path.to_s
+      end
+
+      #
+      # loads the configuration from a JSON file
+      #
+      def self.load_config_json(path)
+        raise ArgumentError.new("Invalid file path: \"" + path.to_s + "\"") if not File.file? path
+        config = JSON.parse(File.open(path) { |f| f.read })
+      end
+
+
+      def self.parse_command_line(argv = ARGV)
+        options = {}
+        optparse = OptionParser.new do |opts|
+          opts.on('-f', '--file JSON_CONFIG_FILE', "Mandatory JSON config file") do |f|
+            options[:file] = f
+          end
+        end
+
+        optparse.parse!(argv)
+
+        # file parameter is not optional
+        raise OptionParser::MissingArgument.new("file") if options[:file].nil?
+
+        return options
+      end
+
+      def self.main
+        options = parse_command_line
+        config_path =  options[:file]
+        config = load_config_json( config_path )
+        add_dir_to_load_path( Pathname.new(config_path).dirname )
+        setup_domains(config)
+        workers = start_workers(config_path, config)
+        setup_signal_handling(workers)
+
+        # hang there until killed: this process is used to relay signals to children
+        # to support and facilitate an orderly shutdown
+        wait_for_child_processes(workers)
+
+      end
+
+    end
+  end
+end
+
+if __FILE__ == $0
+  AWS::Flow::Runner.main()
+end
+
+

data/spec/aws/integration/runner_integration_spec.rb

@@ -0,0 +1,181 @@
+require 'runner'
+require 'bundler/setup'
+require 'aws/decider'
+require 'logger'
+require 'socket'
+
+describe "Runner" do
+
+  # Copied from the utilities for the samples and recipes
+  module SharedUtils
+
+    def setup_domain(domain_name)
+      swf = AWS::SimpleWorkflow.new
+      domain = swf.domains[domain_name]
+      unless domain.exists?
+        swf.domains.create(domain_name, 10)
+      end
+      domain
+    end
+
+    def build_workflow_worker(domain, klass, task_list)
+      AWS::Flow::WorkflowWorker.new(domain.client, domain, task_list, klass)
+    end
+
+    def build_generic_activity_worker(domain, task_list)
+      AWS::Flow::ActivityWorker.new(domain.client, domain, task_list)
+    end
+
+    def build_activity_worker(domain, klass, task_list)
+      AWS::Flow::ActivityWorker.new(domain.client, domain, task_list, klass)
+    end
+
+    def build_workflow_client(domain, options_hash)
+      AWS::Flow::workflow_client(domain.client, domain) { options_hash }
+    end
+  end
+
+  class PingUtils
+    include SharedUtils
+
+    WF_VERSION = "1.0"
+    ACTIVITY_VERSION = "1.0"
+    WF_TASKLIST = "workflow_tasklist"
+    ACTIVITY_TASKLIST = "activity_tasklist"
+    DOMAIN = "PingTest"
+
+    def initialize
+      @domain = setup_domain(DOMAIN)
+    end
+
+    def activity_worker
+      build_activity_worker(@domain, PingActivity, ACTIVITY_TASKLIST)
+    end
+
+    def workflow_worker
+      build_workflow_worker(@domain, PingWorkflow, WF_TASKLIST)
+    end
+
+    def workflow_client
+      build_workflow_client(@domain, from_class: "PingWorkflow")
+    end
+  end  
+
+  # PingActivity class defines a set of activities for the Ping sample.
+  class PingActivity
+    extend AWS::Flow::Activities
+
+    # The activity method is used to define activities. It accepts a list of names
+    # of activities and a block specifying registration options for those
+    # activities
+    activity :ping do
+      {
+        version: PingUtils::ACTIVITY_VERSION,
+        default_task_list: PingUtils::ACTIVITY_TASKLIST,
+        default_task_schedule_to_start_timeout: 30,
+        default_task_start_to_close_timeout: 30
+      }
+    end
+
+    # This activity will say hello when invoked by the workflow
+    def ping() 
+      puts "Pong from #{Socket.gethostbyname(Socket.gethostname).first}"
+      "Pong from #{Socket.gethostbyname(Socket.gethostname).first}"
+    end
+  end
+
+  # PingWorkflow class defines the workflows for the Ping sample
+  class PingWorkflow
+    extend AWS::Flow::Workflows
+
+    workflow :ping do
+      {
+        version: PingUtils::WF_VERSION,
+        task_list: PingUtils::WF_TASKLIST,
+        execution_start_to_close_timeout: 30,
+      }
+    end
+
+    # Create an activity client using the activity_client method to schedule
+    # activities
+    activity_client(:client) { { from_class: "PingActivity" } }
+
+    # This is the entry point for the workflow
+    def ping()
+      # Use the activity client 'client' to invoke the say_hello activity
+      pong=client.ping()
+      "Got #{pong}"
+    end
+  end 
+
+  describe "Sanity Check" do
+
+    it "makes sure credentials and region are in the execution environment" do
+      # note: this could be refactored with a map, but errors are easier to figure out this way
+      begin
+        ENV['AWS_ACCESS_KEY_ID'].should_not be_nil
+        ENV['AWS_SECRET_ACCESS_KEY'].should_not be_nil
+        ENV['AWS_REGION'].should_not be_nil
+      rescue RSpec::Expectations::ExpectationNotMetError
+        # FIXME: there ought to be a better way to pass a useful message to the user
+        puts "\tPlease see the getting started to set up the environment"
+        puts "\thttp://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/installing.html#installing-credentials"
+        raise RSpec::Expectations::ExpectationNotMetError
+      end
+    end
+
+    it "makes sure the credentials and region in the environment can be used to talk to SWF" do
+      swf = AWS::SimpleWorkflow.new
+      domains = swf.client.list_domains "registration_status" => "REGISTERED"
+    end
+
+  end
+
+  describe "Hello World" do
+    
+    it "runs" do
+
+      runner_config = JSON.parse('{
+        "workflow_paths": [],
+        "workflow_workers": [
+          {
+            "domain": ' + "\"#{PingUtils::DOMAIN}\"" + ',
+            "task_list": ' + "\"#{PingUtils::WF_TASKLIST}\"" + ',
+            "workflow_classes": [ ' + "\"PingWorkflow\""  + ' ],
+            "number_of_workers": 1
+          }
+        ],
+        "activity_paths": [],
+        "activity_workers": [
+          {
+            "domain": ' + "\"#{PingUtils::DOMAIN}\"" + ',
+            "task_list": ' + "\"#{PingUtils::ACTIVITY_TASKLIST}\"" + ',
+            "activity_classes": [ ' + "\"PingActivity\""  + ' ],
+            "number_of_forks_per_worker": 1,
+            "number_of_workers": 1
+          }
+        ]
+      }')
+
+      # mock the load_files method to avoid having to create default files
+      AWS::Flow::Runner.stub(:load_files)
+
+      workers = AWS::Flow::Runner.start_workers("", runner_config)
+
+      utils  = PingUtils.new
+      wf_client = utils.workflow_client
+      
+      workflow_execution = wf_client.ping()
+
+      sleep 3 until [
+                     "WorkflowExecutionCompleted",
+                     "WorkflowExecutionTimedOut",
+                     "WorkflowExecutionFailed"
+                    ].include? workflow_execution.events.to_a.last.event_type
+
+      # kill the workers
+      workers.each { |w| Process.kill("KILL", w) }
+    end
+  end
+  
+end

data/spec/aws/unit/runner_unit_spec.rb

@@ -0,0 +1,536 @@
+require 'runner'
+require 'tempfile'
+require 'socket'
+require 'fileutils'
+require_relative '../../spec_helper.rb'
+
+describe "Runner" do
+  
+  describe "Command line" do
+    it "makes sure that the JSON file must be provided on the command line" do
+      expect { AWS::Flow::Runner.parse_command_line([]) }.to raise_error( OptionParser::MissingArgument )
+    end
+
+    it "makes sure that the JSON file must be provided on the command line (switch must be followed by argument)" do
+      expect { AWS::Flow::Runner.parse_command_line(["-f"]) }.to raise_error( OptionParser::MissingArgument )
+    end
+
+    it "makes sure that the JSON file must be provided on the command line (switch must be followed by argument which is valid file; valid case)" do
+      file = Tempfile.new('foo')
+      begin
+        expect { AWS::Flow::Runner.parse_command_line(["-f", file.path]) }.not_to raise_error
+      ensure
+        file.unlink
+      end
+    end
+
+  end
+
+  describe "JSON loading" do
+
+    it "makes sure that the JSON file exists" do
+      file = Tempfile.new('foo')
+      path = file.path
+      file.unlink
+      expect { AWS::Flow::Runner.load_config_json(path) }.to raise_error(ArgumentError)
+    end
+
+    it "makes sure that the JSON file has valid content" do
+      file = Tempfile.new('foo')
+      begin
+        File.write(file, "garbage{")
+        expect { AWS::Flow::Runner.load_config_json(file.path) }.to raise_error(JSON::ParserError)
+      ensure
+        file.unlink
+      end
+    end
+
+  end
+
+
+  describe "JSON validation" do
+
+    it "makes sure activity classes are provided (empty list)" do
+      document = '{
+        "activity_paths": [],
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "activity_classes": [],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      js = JSON.parse(document)
+
+      # just in case so we don't start chid processes
+      AWS::Flow::Runner.stub(:fork)
+
+      # make sure the error is thrown
+      expect {
+        AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, js)
+      }.to raise_error(ArgumentError)
+
+    end
+
+    it "makes sure activity classes are provided (no list)" do
+      document = '{
+        "activity_paths": [],
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      js = JSON.parse(document)
+
+      # just in case so we don't start chid processes
+      allow(AWS::Flow::Runner).to receive(:fork).and_return(42)
+
+      # make sure the error is thrown
+      expect {
+        AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, js)
+      }.to raise_error(ArgumentError)
+
+    end
+
+    it "makes sure workflow classes are provided (empty list)" do
+      document = '{
+        "workflow_paths": [],
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "workflow_classes": [],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      js = JSON.parse(document)
+
+      # just in case so we don't start chid processes
+      AWS::Flow::Runner.stub(:fork)
+
+      # make sure the error is thrown
+      expect {
+        AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, js)
+      }.to raise_error(ArgumentError)
+
+    end
+
+    it "makes sure workflow classes are provided (no list)" do
+      document = '{
+        "workflow_paths": [],
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      js = JSON.parse(document)
+
+      # just in case so we don't start chid processes
+      allow(AWS::Flow::Runner).to receive(:fork).and_return(42)
+
+      # make sure the error is thrown
+      expect {
+        AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, js)
+      }.to raise_error(ArgumentError)
+
+    end
+
+  end
+
+  describe "Starting workers" do
+
+    def workflow_js
+      document = '{
+        "workflow_paths": [],
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "workflow_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      JSON.parse(document)
+    end
+
+    def activity_js
+      document = '{
+        "activity_paths": [],
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "activity_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+      JSON.parse(document)
+    end
+
+    it "makes sure the number of workflow workers is correct" do
+      # mock out a few methods to focus on the fact that the workers were created
+      allow_any_instance_of(AWS::Flow::WorkflowWorker).to receive(:add_implementation).and_return(nil)
+      allow_any_instance_of(AWS::Flow::WorkflowWorker).to receive(:start).and_return(nil)
+      AWS::Flow::Runner.stub(:load_files)
+
+      # what we are testing:
+      expect(AWS::Flow::Runner).to receive(:fork).exactly(3).times
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, "", workflow_js)
+    end
+
+
+
+    it "makes sure the number of activity workers is correct" do
+      # mock out a few methods to focus on the fact that the workers were created
+      allow_any_instance_of(AWS::Flow::ActivityWorker).to receive(:add_implementation).and_return(nil)
+      allow_any_instance_of(AWS::Flow::ActivityWorker).to receive(:start).and_return(nil)
+      AWS::Flow::Runner.stub(:load_files)
+
+      # what we are testing:
+      expect(AWS::Flow::Runner).to receive(:fork).exactly(3).times
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, "",activity_js)
+    end
+    
+    it "makes sure the workflow implementation classes are added" do
+      # mock out a few methods to focus on the implementations being added
+      allow_any_instance_of(AWS::Flow::WorkflowWorker).to receive(:start).and_return(nil)
+      AWS::Flow::Runner.stub(:fork)
+      AWS::Flow::Runner.stub(:load_files)
+      
+      # stub that we can query later
+      implems = []
+      AWS::Flow::WorkflowWorker.any_instance.stub(:add_implementation) do |arg|
+        implems << arg
+      end
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, "",workflow_js)
+
+      # validate
+      expect(implems).to include(Object.const_get("Object"), Object.const_get("String"))
+    end
+
+    it "makes sure the activity implementation classes are added" do
+      # mock out a few methods to focus on the implementations being added
+      allow_any_instance_of(AWS::Flow::ActivityWorker).to receive(:start).and_return(nil)
+      AWS::Flow::Runner.stub(:fork)
+      AWS::Flow::Runner.stub(:load_files)
+      
+      # stub that we can query later
+      implems = []
+      AWS::Flow::ActivityWorker.any_instance.stub(:add_implementation) do |arg|
+        implems << arg
+      end
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, "",activity_js)
+
+      # validate
+      expect(implems).to include(Object.const_get("Object"), Object.const_get("String"))
+    end
+
+    it "makes sure the workflow worker is started" do
+      # mock out a few methods to focus on the worker getting started
+      allow_any_instance_of(AWS::Flow::WorkflowWorker).to receive(:add_implementation).and_return(nil)
+      AWS::Flow::Runner.stub(:fork).and_yield
+      AWS::Flow::Runner.stub(:load_files)
+      
+      # stub that we can query later
+      starts = 0
+      AWS::Flow::WorkflowWorker.any_instance.stub(:start) do |arg|
+        starts += 1
+      end
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, "",workflow_js)
+
+      # validate
+      expect(starts).to equal(3)
+    end
+
+    it "makes sure the activity worker is started" do
+      # mock out a few methods to focus on the worker getting started
+      allow_any_instance_of(AWS::Flow::ActivityWorker).to receive(:add_implementation).and_return(nil)
+      AWS::Flow::Runner.stub(:fork).and_yield
+      AWS::Flow::Runner.stub(:load_files)
+      
+      # stub that we can query later
+      starts = 0
+      AWS::Flow::ActivityWorker.any_instance.stub(:start) do |arg|
+        starts += 1
+      end
+
+      # start the workers 
+      workers = AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, "",activity_js)
+
+      # validate
+      expect(starts).to equal(3)
+    end
+
+  end
+
+
+
+  describe "Loading files" do
+
+    before(:each) do
+      # let's pretend the files exist, so that loading proceeds
+      allow(File).to receive(:exists?).and_return(true)
+      # stubs to avoid running code that should not be run/covered in these tests
+      AWS::Flow::Runner.stub(:add_implementations)
+      AWS::Flow::Runner.stub(:spawn_and_start_workers)
+    end
+
+    it "looks in the directory where the config is and loads the specified default" do
+      base = "/tmp/blahdir"
+      relative = File.join('flow', 'activities.rb')
+      
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join(base, relative))
+
+      AWS::Flow::Runner.load_files( File.join(base, "blahconfig"), "", 
+                                   {:config_key => "any_key_name",
+                                    :default_file => relative})
+    end
+
+    it "loads the default only if needed" do
+      base = "/tmp/blahdir"
+      relative = File.join('flow', 'activities.rb')
+
+      expect(AWS::Flow::Runner).to_not receive(:require).with(File.join(base, relative))
+      expect(AWS::Flow::Runner).to receive(:require).with("foo")
+      expect(AWS::Flow::Runner).to receive(:require).with("bar")
+
+      AWS::Flow::Runner.load_files( File.join(base, "blahconfig"), 
+                                    JSON.parse('{ "activity_paths": [ "foo", "bar"] }'), 
+                                   {:config_key => "activity_paths",
+                                    :default_file => relative})
+    end
+
+    it "loads the \"flow/activities.rb\" by default for activity worker" do
+      def activity_js
+        document = '{
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "activity_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join(".", "flow", "activities.rb"))
+
+      AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, ".", activity_js)
+    end
+
+    it "loads the \"flow/workflows.rb\" by default for workflow worker" do
+      def workflow_js
+        document = '{
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "workflow_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join(".", "flow", "workflows.rb"))
+
+      AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, ".", workflow_js)
+    end
+
+    it "takes activity_paths as override to \"flow/activities.rb\"" do
+      def activity_js
+        document = '{
+        "activity_paths": [ "foo", "bar"],
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "activity_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      expect(AWS::Flow::Runner).to_not receive(:require).with(File.join(".", "flow", "activities.rb"))
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join("foo"))
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join("bar"))
+
+      AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, ".", activity_js)
+    end
+
+    it "takes workflow_paths as override to \"flow/workflows.rb\"" do
+      def workflow_js
+        document = '{
+        "workflow_paths": [ "foo", "bar"],
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "workflow_classes": [ "Object", "String" ],
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      expect(AWS::Flow::Runner).to_not receive(:require).with(File.join(".", "flow", "workflows.rb"))
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join("foo"))
+      expect(AWS::Flow::Runner).to receive(:require).with(File.join("bar"))
+
+      AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, ".", workflow_js)
+    end
+
+  end
+
+
+
+
+  describe "Implementation classes discovery" do
+
+    # because the object space is not reset between test runs, these
+    # classes are declared here for all the tests in this section to use
+    class MyActivity1
+      extend AWS::Flow::Activities
+    end
+    class MyActivity2
+      extend AWS::Flow::Activities
+    end
+
+    class MyWorkflow1
+      extend AWS::Flow::Workflows
+    end
+    class MyWorkflow2
+      extend AWS::Flow::Workflows
+    end
+
+    before(:each) do
+      # stubs to avoid running code that should not be run/covered in these tests
+      AWS::Flow::Runner.stub(:spawn_and_start_workers)
+    end
+
+    it "finds all the subclasses properly" do
+      module Clown 
+      end
+      class Whiteface 
+        extend Clown 
+      end
+      class Auguste 
+        extend Clown 
+      end
+
+      sub = AWS::Flow::Runner.all_subclasses(Clown)
+      expect(sub).to include(Whiteface)
+      expect(sub).to include(Auguste)
+    end
+
+    it "finds all the subclasses of AWS::Flow::Activities properly" do
+      sub = AWS::Flow::Runner.all_subclasses(AWS::Flow::Activities)
+      expect(sub).to include(MyActivity1)
+      expect(sub).to include(MyActivity2)
+    end
+
+    it "finds all the subclasses of AWS::Flow::Workflows properly" do
+      sub = AWS::Flow::Runner.all_subclasses(AWS::Flow::Workflows)
+      expect(sub).to include(MyWorkflow1)
+      expect(sub).to include(MyWorkflow2)
+    end
+
+    it "finds the activity implementations when they are in the environment" do
+      def activity_js
+        document = '{
+        "activity_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      impls = []
+      AWS::Flow::ActivityWorker.any_instance.stub(:add_implementation) do |impl|
+        impls << impl
+      end
+
+      AWS::Flow::Runner.start_activity_workers(AWS::SimpleWorkflow.new, ".", activity_js)
+
+      expect(impls).to include(MyActivity2)
+      expect(impls).to include(MyActivity1)
+    end
+
+    it "finds the workflow implementations when they are in the environment" do
+      def workflow_js
+        document = '{
+        "workflow_workers": [
+          {
+            "domain": "foo",
+            "task_list": "bar",
+            "number_of_workers": 3
+          }
+        ]
+      }'
+        JSON.parse(document)
+      end
+
+      impls = []
+      AWS::Flow::WorkflowWorker.any_instance.stub(:add_implementation) do |impl|
+        impls << impl
+      end
+
+      AWS::Flow::Runner.start_workflow_workers(AWS::SimpleWorkflow.new, ".", workflow_js)
+
+      expect(impls).to include(MyWorkflow2)
+      expect(impls).to include(MyWorkflow1)
+    end
+
+  end
+
+  describe "Host-specific tasklists" do
+    
+    it "expand to the local host name" do
+      # note how we test for value equality; not object equality
+      expect(AWS::Flow::Runner.expand_task_list("|hostname|")).to eq(Socket.gethostname)
+    end
+
+    it "expand to the local host name even in multiple places" do
+      # note how we test for value equality; not object equality
+      expect(AWS::Flow::Runner.expand_task_list("xxx|hostname|yy|hostname|zz")).to eq("xxx#{Socket.gethostname}yy#{Socket.gethostname}zz")
+    end
+
+    it "preserves the task list value if no expanded pattern found" do
+      # note how we test for value equality; not object equality
+      expect(AWS::Flow::Runner.expand_task_list("xxxzz")).to eq("xxxzz")
+    end
+
+  end
+
+end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: aws-flow
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
+  prerelease: 
 platform: ruby
 authors:
-- Michael Steger
+- Michael Steger, Paritosh Mohan, Jacques Thomas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-06 00:00:00.000000000 Z
+date: 2014-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -23,6 +25,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -32,7 +35,8 @@ dependencies:
         version: 1.39.0
 description: Library to provide the AWS Flow Framework for Ruby
 email: ''
-executables: []
+executables:
+- aws-flow-ruby
 extensions: []
 extra_rdoc_files: []
 files:
@@ -41,6 +45,7 @@ files:
 - NOTICE.TXT
 - Rakefile
 - aws-flow.gemspec
+- bin/aws-flow-ruby
 - lib/aws/decider.rb
 - lib/aws/decider/activity.rb
 - lib/aws/decider/activity_definition.rb
@@ -77,7 +82,9 @@ files:
 - lib/aws/flow/implementation.rb
 - lib/aws/flow/simple_dfa.rb
 - lib/aws/flow/tasks.rb
+- lib/aws/runner.rb
 - spec/aws/integration/integration_spec.rb
+- spec/aws/integration/runner_integration_spec.rb
 - spec/aws/unit/async_backtrace_spec.rb
 - spec/aws/unit/async_scope_spec.rb
 - spec/aws/unit/begin_rescue_ensure_spec.rb
@@ -92,31 +99,32 @@ files:
 - spec/aws/unit/options_spec.rb
 - spec/aws/unit/preinclude_tests.rb
 - spec/aws/unit/rubyflow.rb
+- spec/aws/unit/runner_unit_spec.rb
 - spec/aws/unit/simple_dfa_spec.rb
 - spec/spec_helper.rb
-homepage: 
+homepage: https://aws.amazon.com/swf/details/flow/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 - lib/aws/
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
-summary: AWS Flow Decider package decider
+specification_version: 3
+summary: AWS Flow Framework for Ruby
 test_files: []
-has_rdoc: 

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODBhMDgxOTcxNDAyNGUzMzZmMDllMDdjMWRiZmE3ZjZmM2NmOTA3MQ==
-  data.tar.gz: !binary |-
-    ZDhmNDA5OTBhNThkNDhmMGQ4MjM2ZDcyMGJhM2U1YTNlMGZiYzkyNg==
-SHA512:
-  metadata.gz: !binary |-
-    NTRlNDUxZDIyZjQyNGRhNzAyOTZhZmQ0NjkwNjI0NThmZGIwOTJmYWVlN2Y3
-    ODY2MGZiZjFmOWE4MGJjN2VhNzJkNzBkMjhiYmNmYmNlMTExNjg3NTlkZjRm
-    YzlmZGQ3MjE5MWEyM2I2YjEyYmZiN2NmOGY2YTllMDY0MzE4YTg=
-  data.tar.gz: !binary |-
-    YTczMTcwODZmMWZkNTgxODQ2NTZjY2FjNmE4ZGRiY2I5NzA0OWVhNTcxY2Ey
-    OThhNGI3ZDY2MDM2ZmM1ZmQ3M2QyZTcyN2JkZGYzNDlkYjM4NjVlMjA1NTMw
-    N2VhMjllYTRhYTE5M2E4M2FhNGZlMGY2ZjEzYjdjMzI4NjFkZmE=