aws-flow 2.3.1 → 2.4.0

data/lib/aws/decider/task_poller.rb

@@ -136,6 +136,7 @@ module AWS
       #
       def initialize(service, domain, task_list, activity_definition_map, executor, options=nil)
         @service = service
+        @service_opts = @service.config.to_h
         @domain = domain
         @task_list = task_list
         @activity_definition_map = activity_definition_map
@@ -319,10 +320,9 @@ module AWS
         # Since we can't change the pool of an already existing NetHttpHandler,
         # we also create a new NetHttpHandler in order to use the new pool.
 
-        options = @service.config.to_h
-        options[:connection_pool] = AWS::Core::Http::ConnectionPool.build(options[:http_handler].pool.options)
-        options[:http_handler] = AWS::Core::Http::NetHttpHandler.new(options)
-        @service = AWS::SimpleWorkflow.new(options).client
+        @service_opts[:connection_pool] = AWS::Core::Http::ConnectionPool.build(@service_opts[:http_handler].pool.options)
+        @service_opts[:http_handler] = AWS::Core::Http::NetHttpHandler.new(@service_opts)
+        @service = @service.with_options(@service_opts)
 
         begin
           begin

data/lib/aws/decider/utilities.rb

@@ -92,9 +92,28 @@ module AWS
         client_options
       end
 
+      # This method is used to register a domain with the Simple Workflow
+      # Service.
+      # @param name Name of the domain
+      # @param retention Workflow execution retention period in days
       # @api private
+      def self.register_domain(name, retention=nil)
+        swf = AWS::SimpleWorkflow.new
+        retention ||= FlowConstants::RETENTION_DEFAULT
+        begin
+          swf.client.register_domain({
+            name: name.to_s,
+            workflow_execution_retention_period_in_days: retention.to_s
+          })
+        rescue AWS::SimpleWorkflow::Errors::DomainAlreadyExistsFault => e
+          # possible log an INFO/WARN if the domain already exists.
+        end
+        return AWS::SimpleWorkflow::Domain.new(name.to_s)
+      end
+
       # This method is used to truncate Activity and Workflow exceptions to
       # fit them into responses to the SWF service.
+      # @api private
       def self.check_and_truncate_exception error, converter
 
         # serialize the exception so that we can check the actual size of the
@@ -228,6 +247,25 @@ module AWS
         return true
       end
 
+      # @api private
+      def self.add_workflow_worker_to_spec(spec, klass, tasklist)
+        add_worker_to_spec(spec, 'workflow', klass, tasklist)
+      end
+
+      # @api private
+      def self.add_activity_worker_to_spec(spec, klass, tasklist)
+        add_worker_to_spec(spec, 'activity', klass, tasklist)
+      end
+
+      # @api private
+      def self.add_worker_to_spec(spec, type, klass, tasklist)
+        spec["#{type}_workers"] << {
+          "#{type}_classes" => ["#{klass}"],
+          "task_list" => "#{tasklist}"
+        }
+        spec
+      end
+
       # @api private
       module SelfMethods
         # @api private

data/lib/aws/decider/version.rb

@@ -16,7 +16,7 @@
 module AWS
   module Flow
     def self.version
-      "2.3.1"
+      "2.4.0"
     end
   end
 end

data/lib/aws/decider/worker.rb

@@ -287,12 +287,13 @@ module AWS
         @activity_type_options = []
         @options = Utilities::interpret_block_for_options(WorkerOptions, block)
 
-        @logger = @options.logger if @options
-        @logger ||= Utilities::LogFactory.make_logger(self)
-        @options.logger ||= @logger if @options
-
-        max_workers = @options.execution_workers if @options
-        max_workers = 20 if (max_workers.nil? || max_workers.zero?)
+        if @options
+          @logger = @options.logger || Utilities::LogFactory.make_logger(self)
+          @options.logger ||= @logger
+          max_workers = @options.execution_workers
+          @options.use_forking = false if (max_workers && max_workers.zero?)
+        end
+        max_workers = 20 if (max_workers.nil?)
         @executor = ForkingExecutor.new(
           :max_workers => max_workers,
           :logger => @logger
@@ -338,7 +339,7 @@ module AWS
             registration_difference =  default_options.sort.to_a - previous_registration.sort.to_a
 
             unless registration_difference.empty?
-              raise "There is a difference between the types you have registered previously and the types you are currently registering, but you haven't changed the version. These new changes will not be picked up. In particular, these options are different #{Hash[registration_difference]}"
+              raise "Activity [#{activity_type_options[:name]}]: There is a difference between the types you have registered previously and the types you are currently registering, but you haven't changed the version. These new changes will not be picked up. In particular, these options are different #{Hash[registration_difference]}"
             end
             # Purposefully eaten up, the alternative is to check first, and who
             # wants to do two trips when one will do?

data/lib/aws/decider/workflow_definition_factory.rb

@@ -61,7 +61,7 @@ module AWS
         if ! implementation_options.nil?
           @converter = implementation_options.data_converter
         end
-        @converter ||= FlowConstants.default_data_converter
+        @converter ||= FlowConstants.data_converter
 
       end
 

data/lib/aws/runner.rb

@@ -41,6 +41,7 @@ module AWS
 
       # Import the necessary gems to run Ruby Flow code.
       require 'aws/decider'
+      require 'aws/templates'
       include AWS::Flow
       require 'json'
       require 'optparse'
@@ -51,21 +52,16 @@ module AWS
       # @api private
       def self.setup_domain(json_config)
 
-        swf = create_service_client(json_config)
+        set_user_agent(json_config)
+
+        # If domain is not provided, use the default ruby flow domain
+        domain = json_config['domain'] || { 'name' => FlowConstants.defaults[:domain] }
 
-        domain = json_config['domain']
         # If retention period is not provided, default it to 7 days
         retention = domain['retention_in_days'] || FlowConstants::RETENTION_DEFAULT
 
-        begin
-          swf.client.register_domain({
-            name: domain['name'],
-            workflow_execution_retention_period_in_days: retention.to_s
-          })
-        rescue AWS::SimpleWorkflow::Errors::DomainAlreadyExistsFault => e
-          # possible log an INFO/WARN if the domain already exists.
-        end
-        return AWS::SimpleWorkflow::Domain.new( domain['name'] )
+        AWS::Flow::Utilities.register_domain(domain['name'], retention.to_s)
+
       end
 
       # @api private
@@ -104,26 +100,19 @@ module AWS
         classes
       end
 
-      # Used to add implementations to workers; see [get_classes] for more
-      # information.
-      #
-      # @api private
-      def self.add_implementations(worker, json_fragment, what)
-        classes = get_classes(json_fragment, what)
-        classes.each { |c| worker.add_implementation(c) }
-      end
-
       # Spawns the workers.
       #
       # @api private
       def self.spawn_and_start_workers(json_fragment, process_name, worker)
         workers = []
         num_of_workers = json_fragment['number_of_workers'] || FlowConstants::NUM_OF_WORKERS_DEFAULT
+        should_register = true
         num_of_workers.times do
           workers << fork do
             set_process_name(process_name)
-            worker.start()
+            worker.start(should_register)
           end
+          should_register = false
         end
         workers
       end
@@ -153,7 +142,7 @@ module AWS
       #
       # json_config: the content of the config
       #
-      # what: what should loaded. This is a hash expected to contain two keys:
+      # what: what should be loaded. This is a hash expected to contain two keys:
       #
       #     - :default_file : the file to load unless a specific list is provided
       #
@@ -177,34 +166,92 @@ module AWS
       # section of [the runner specification file][], or that are loaded from
       # `require` statements in the `workflows.rb` file.
       #
+      # If the 'activity' classes are regular ruby classes, this method will
+      # create a proxy AWS::Flow::Activities class for each regular ruby class
+      # loaded and add the proxy implementation to the ActivityWorker.
+      #
       # @api private
-      def self.start_activity_workers(swf, domain = nil, config_path, json_config)
+      def self.start_activity_workers(swf, domain = nil, json_config)
         workers = []
-        # load all classes for the activities
-        load_files(config_path, json_config, {config_key: 'activity_paths',
-                     default_file: File.join('flow', 'activities.rb')})
         domain = setup_domain(json_config) if domain.nil?
 
+        # This will be used later to start default workflow workers. If the
+        # 'activity_workers' and 'default_workers' keys are not specified in the
+        # json spec, then we don't start any default workers. Hence this value
+        # is defaulted to 0.
+        number_of_default_workers = 0
+
         # TODO: logger
         # start the workers for each spec
-        json_config['activity_workers'].each do |w|
-          # If number of forks is not provided, it will automatically default to 20
-          # within the ActivityWorker
-          fork_count = w['number_of_forks_per_worker']
-          task_list = expand_task_list(w['task_list'])
-
-          # create a worker
-          worker = ActivityWorker.new(swf.client, domain, task_list, *w['activities']) {{ execution_workers: fork_count }}
-          add_implementations(worker, w, {config_key: 'activity_classes',
-                     clazz: AWS::Flow::Activities})
-
-          # start as many workers as desired in child processes
-          workers << spawn_and_start_workers(w, "activity-worker", worker)
+        if json_config['activity_workers']
+          json_config['activity_workers'].each do |w|
+            # If number of forks is not provided, it will automatically default to 20
+            # within the ActivityWorker
+            fork_count = w['number_of_forks_per_worker']
+            task_list = expand_task_list(w['task_list']) if w['task_list']
+
+            # Get activity classes
+            classes = get_classes(w, {config_key: 'activity_classes',
+                                      clazz: AWS::Flow::Activities})
+
+            # If task_list is not provided, use the name of the first class as the
+            # task_list for this worker
+            task_list ||= "#{classes.first}"
+
+            # Create a worker
+            worker = ActivityWorker.new(swf.client, domain, task_list) {{ execution_workers: fork_count }}
+
+            classes.each do |c|
+              c = AWS::Flow::Templates.make_activity_class(c) unless c.is_a?(AWS::Flow::Activities)
+              worker.add_implementation(c)
+            end
+
+            # We add 1 default worker for each activity worker.
+            number_of_default_workers += w['number_of_workers'] || FlowConstants::NUM_OF_WORKERS_DEFAULT
+
+            # start as many workers as desired in child processes
+            workers << spawn_and_start_workers(w, "activity-worker", worker)
+          end
+
+          # Create the config for default workers if it's not passed in the
+          # json_config
+          if json_config['default_workers'].nil? || json_config['default_workers']['number_of_workers'].nil?
+            json_config['default_workers'] = {
+              'number_of_workers' => number_of_default_workers
+            }
+          end
         end
 
+        # Start the default workflow workers
+        workers << start_default_workers(swf, domain, json_config)
+
         return workers
       end
 
+      # Starts workflow workers for the default workflow type 'FlowDefaultWorkflowRuby'.
+      # If 'default_workers' key is not set in the json spec, we set the number
+      # of workers equal to the number of activity workers
+      # Default workers are used to for processing workflow templates
+      #
+      # @api private
+      def self.start_default_workers(swf, domain = nil, json_config)
+        workers = []
+        domain = setup_domain(json_config) if domain.nil?
+
+        if json_config['default_workers']
+          # Also register the default result activity type in the given domain
+          AWS::Flow::Templates.register_default_result_activity(domain)
+
+          klass = AWS::Flow::Templates.default_workflow
+          task_list = FlowConstants.defaults[:task_list]
+          # Create a worker
+          worker = WorkflowWorker.new(swf.client, domain, task_list, klass)
+          # This will take care of both registering and starting the default workers
+          workers << spawn_and_start_workers(json_config['default_workers'], "default-worker", worker)
+        end
+        workers
+      end
+
       # Starts the workflow workers.
       #
       # The workflows run by the workers consist of each class that extends
@@ -213,37 +260,42 @@ module AWS
       # `require` statements in the `workflows.rb` file.
       #
       # @api private
-      def self.start_workflow_workers(swf, domain = nil, config_path, json_config)
+      def self.start_workflow_workers(swf, domain = nil, json_config)
         workers = []
-        # load all the classes for the workflows
-        load_files(config_path, json_config, {config_key: 'workflow_paths',
-                     default_file: File.join('flow', 'workflows.rb')})
         domain = setup_domain(json_config) if domain.nil?
 
         # TODO: logger
         # start the workers for each spec
-        json_config['workflow_workers'].each do |w|
-          task_list = expand_task_list(w['task_list'])
+        if json_config['workflow_workers']
+          json_config['workflow_workers'].each do |w|
+            task_list = expand_task_list(w['task_list'])
+
+            # Get workflow classes
+            classes = get_classes(w, {config_key: 'workflow_classes',
+                                      clazz: AWS::Flow::Workflows})
 
-          # create a worker
-          worker = WorkflowWorker.new(swf.client, domain, task_list, *w['workflows'])
-          add_implementations(worker, w, {config_key: 'workflow_classes',
-                     clazz: AWS::Flow::Workflows})
+            # Create a worker
+            worker = WorkflowWorker.new(swf.client, domain, task_list, *classes)
 
-          # start as many workers as desired in child processes
-          workers << spawn_and_start_workers(w, "workflow-worker", worker)
+            # Start as many workers as desired in child processes
+            workers << spawn_and_start_workers(w, "workflow-worker", worker)
+          end
         end
 
         return workers
       end
 
       # @api private
-      def self.create_service_client(json_config)
+      def self.set_user_agent(json_config)
         # set the UserAgent prefix for all clients
         if json_config['user_agent_prefix'] then
           AWS.config(user_agent_prefix: json_config['user_agent_prefix'])
         end
+      end
 
+      # @api private
+      def self.create_service_client(json_config)
+        set_user_agent(json_config)
         swf = AWS::SimpleWorkflow.new
       end
 
@@ -251,18 +303,36 @@ module AWS
       # worker processes.
       #
       # @api private
-      def self.start_workers(domain = nil, config_path, json_config)
+      def self.start_workers(domain = nil, json_config)
+
         workers = []
 
         swf = create_service_client(json_config)
 
-        workers << start_activity_workers(swf, domain, config_path, json_config)
-        workers << start_workflow_workers(swf, domain, config_path, json_config)
+        workers << start_activity_workers(swf, domain, json_config)
+        workers << start_workflow_workers(swf, domain, json_config)
 
         # needed to avoid returning nested arrays based on the calls above
         workers.flatten!
       end
 
+      # Loads activity and workflow classes
+      #
+      # config_path: the path where the config file is, to be able to
+      #     resolve relative references
+      #
+      # json_config: the content of the config
+      #
+      # @api private
+      def self.load_classes(config_path, json_config)
+        # load all classes for the activities
+        load_files(config_path, json_config, {config_key: 'activity_paths',
+                                              default_file: File.join('flow', 'activities.rb')})
+        # load all the classes for the workflows
+        load_files(config_path, json_config, {config_key: 'workflow_paths',
+                                              default_file: File.join('flow', 'workflows.rb')})
+      end
+
       # Sets up forwarding of signals to child processes to facilitate and
       # support orderly shutdown.
       #
@@ -325,23 +395,35 @@ module AWS
         return options
       end
 
+
       #
-      # Invoked from the shell.
+      # Invoked from code. This is a helper method that can be used to start the
+      # runner from code. This is especially helpful for debugging purposes.
       #
-      # @api private
-      def self.main
-        options = parse_command_line
-        config_path =  options[:file]
-        config = load_config_json( config_path )
-        add_dir_to_load_path( Pathname.new(config_path).dirname )
-        domain = setup_domain(config)
-        workers = start_workers(domain, config_path, config)
+      # worker_spec: Hash representation of the json worker spec
+      #
+      def self.run(worker_spec)
+        workers = start_workers(worker_spec)
         setup_signal_handling(workers)
 
         # Hang there until killed: this process is used to relay signals to
         # children to support and facilitate an orderly shutdown.
         wait_for_child_processes(workers)
       end
+
+      #
+      # Invoked from the shell.
+      #
+      # @api private
+      def self.main
+        options = parse_command_line
+        config_path =  options[:file]
+        worker_spec = load_config_json(config_path)
+        add_dir_to_load_path(Pathname.new(config_path).dirname)
+        load_classes(config_path, worker_spec)
+        run(worker_spec)
+      end
+
     end
   end
 end
@@ -350,4 +432,3 @@ if __FILE__ == $0
   AWS::Flow::Runner.main()
 end
 
-