bolt 0.16.1 → 0.16.2

data/lib/bolt/inventory/group.rb

@@ -1,7 +1,7 @@
 module Bolt
   class Inventory
     # Group is a specific implementation of Inventory based on nested
-    # structred data.
+    # structured data.
     class Group
       attr_accessor :name, :nodes, :groups, :config, :rest
 
@@ -30,29 +30,45 @@ module Bolt
         @rest = data.reject { |k, _| %w[name nodes config groups].include? k }
       end
 
-      def validate(used_names = [], depth = 0)
-        raise ValidationError.new("Group does not have a name", nil)  unless @name
+      def validate(used_names = Set.new, node_names = Set.new, depth = 0)
+        raise ValidationError.new("Group does not have a name", nil) unless @name
         if used_names.include?(@name)
           raise ValidationError.new("Tried to redefine group #{@name}", @name)
         end
         raise ValidationError.new("Invalid Group name #{@name}", @name) unless @name =~ /\A[a-z0-9_]+\Z/
 
+        if node_names.include?(@name)
+          raise ValidationError.new("Group #{@name} conflicts with node of the same name", @name)
+        end
         raise ValidationError.new("Group #{@name} is too deeply nested", @name) if depth > 1
 
         used_names << @name
 
         @nodes.each do |n|
-          raise ValidationError.new("node #{n['name']} does not have a name", @name) unless n['name']
+          # Require nodes to be referenced only by their host name
+          host = Addressable::URI.parse('//' + n['name']).host
+          ipv6host = Addressable::URI.parse('//[' + n['name'] + ']').host
+          if n['name'] != host && n['name'] != ipv6host
+            raise ValidationError.new("Invalid node name #{n['name']}", n['name'])
+          end
+
+          raise ValidationError.new("Node #{n['name']} does not have a name", n['name']) unless n['name']
+          if used_names.include?(n['name'])
+            raise ValidationError.new("Group #{n['name']} conflicts with node of the same name", n['name'])
+          end
+
+          node_names << n['name']
         end
 
         @groups.each do |g|
           begin
-            g.validate(used_names, depth + 1)
+            g.validate(used_names, node_names, depth + 1)
           rescue ValidationError => e
             e.add_parent(@name)
             raise e
           end
         end
+
         nil
       end
 
@@ -92,9 +108,24 @@ module Bolt
         }
       end
 
+      # Returns all nodes contained within the group, which includes nodes from subgroups.
       def node_names
+        @groups.inject(local_node_names) do |acc, g|
+          acc.merge(g.node_names)
+        end
+      end
+
+      # Return a mapping of group names to group.
+      def collect_groups
+        @groups.inject(name => self) do |acc, g|
+          acc.merge(g.collect_groups)
+        end
+      end
+
+      def local_node_names
         @_node_names ||= Set.new(nodes.map { |n| n['name'] })
       end
+      private :local_node_names
 
       def node_collect(node_name)
         data = @groups.inject(nil) do |acc, g|
@@ -118,7 +149,7 @@ module Bolt
 
         if data
           data_merge(group_data, data)
-        elsif node_names.include?(node_name)
+        elsif local_node_names.include?(node_name)
           group_data
         end
       end

data/lib/bolt/pal.rb

@@ -37,6 +37,22 @@ module Bolt
 
       # Now that puppet is loaded we can include puppet mixins in data types
       Bolt::ResultSet.include_iterable
+
+      # TODO: This is a hack for PUP-8441 remove it once that is fixed
+      require_relative '../../vendored/puppet/lib/puppet/datatypes/impl/error.rb'
+      Puppet::DataTypes::Error.class_eval do
+        def to_json(opts = nil)
+          _pcore_init_hash.to_json(opts)
+        end
+      end
+    end
+
+    # Create a top-level alias for TargetSpec so that users don't have to
+    # namespace it with Boltlib, which is just an implementation detail. This
+    # allows TargetSpec to feel like a built-in type in bolt, rather than
+    # something has been, no pun intended, "bolted on".
+    def add_target_spec(compiler)
+      compiler.evaluate_string('type TargetSpec = Boltlib::TargetSpec')
     end
 
     # Runs a block in a PAL script compiler configured for Bolt.  Catches
@@ -46,14 +62,9 @@ module Bolt
       Puppet.initialize_settings(opts)
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: [BOLTLIB_PATH] + @config[:modulepath], facts: {}) do |pal|
         pal.with_script_compiler do |compiler|
+          add_target_spec(compiler)
           begin
-            result = yield compiler
-            # TODO: remove after PUP-8441 adds to_json to Errors
-            # This hack won't handle nested errors
-            if result.is_a? Puppet::DataTypes::Error
-              result = result._pcore_init_hash
-            end
-            result
+            yield compiler
           rescue Puppet::PreformattedError => err
             # Puppet sometimes rescues exceptions notes the location and reraises
             # For now return the original error. Exception cause support was added in Ruby 2.1
@@ -131,7 +142,11 @@ module Bolt
       task = in_bolt_compiler do |compiler|
         compiler.task_signature(task_name)
       end
-      raise Bolt::CLIError, "Could not find task #{task_name} in your modulepath" if task.nil?
+
+      if task.nil?
+        raise Bolt::CLIError, Bolt::Error.unknown_task(task_name)
+      end
+
       task.task_hash
     end
 
@@ -145,7 +160,11 @@ module Bolt
       plan = in_bolt_compiler do |compiler|
         compiler.plan_signature(plan_name)
       end
-      raise Bolt::CLIError, "Could not find plan #{plan_name} in your modulepath" if plan.nil?
+
+      if plan.nil?
+        raise Bolt::CLIError, Bolt::Error.unknown_plan(plan_name)
+      end
+
       elements = plan.params_type.elements
       {
         'name' => plan_name,

data/lib/bolt/target.rb

@@ -10,14 +10,6 @@ module Bolt
       new(hash['uri'], hash['options'])
     end
 
-    def self.from_uri(uri)
-      new(uri)
-    end
-
-    def self.parse_urls(urls)
-      urls.split(/[[:space:],]+/).reject(&:empty?).uniq.map { |url| from_uri(url) }
-    end
-
     def initialize(uri, options = nil)
       @uri = uri
       @uri_obj = parse(uri)

data/lib/bolt/transport/base.rb

@@ -0,0 +1,159 @@
+require 'logging'
+
+module Bolt
+  module Transport
+    # This class provides the default behavior for Transports. A Transport is
+    # responsible for uploading files and running commands, scripts, and tasks
+    # on Targets.
+    #
+    # Bolt executes work on the Transport in "batches". To do that, it calls
+    # the batches() method, which is responsible for dividing the list of
+    # Targets into batches according to how it wants to handle them. It will
+    # then call Transport#batch_task, or the corresponding method for another
+    # operation, passing a list of Targets. The Transport returns a list of
+    # Bolt::Result objects, one per Target. Each batch is executed on a
+    # separate thread, controlled by the `concurrency` setting, so many batches
+    # may be running in parallel.
+    #
+    # The default batch implementation splits the list of Targets into batches
+    # of 1. It then calls run_task(), or a corresponding method for other
+    # operations, passing in the single Target.
+    #
+    # Most Transport implementations, like the SSH and WinRM transports, don't
+    # need to do their own batching, since they only operate on a single Target
+    # at a time. Those Transports can implement the run_task() and related
+    # methods, which will automatically handle running many Targets in
+    # parallel, and will handle publishing start and finish events for each
+    # Target.
+    #
+    # Transports that need their own batching, like the Orch transport, can
+    # instead override the batches() method to split Targets into sets that can
+    # be executed together, and override the batch_task() and related methods
+    # to execute a batch of nodes. In that case, those Transports should accept
+    # a block argument and call it with a :node_start event for each Target
+    # before executing, and a :node_result event for each Target after
+    # execution.
+    class Base
+      attr_reader :logger
+
+      def initialize(_config)
+        @logger = Logging.logger[self]
+      end
+
+      def with_events(target, callback)
+        callback.call(type: :node_start, target: target) if callback
+        result = yield
+        @logger.debug("Result on #{target.uri}: #{JSON.dump(result.value)}")
+        callback.call(type: :node_result, result: result) if callback
+        result
+      rescue StandardError => ex
+        Bolt::Result.from_exception(target, ex)
+      end
+
+      def filter_options(target, options)
+        if target.options[:run_as]
+          options.reject { |k, _v| k == '_run_as' }
+        else
+          options
+        end
+      end
+
+      # Raises an error if more than one target was given in the batch.
+      #
+      # The default implementations of batch_* strictly assume the transport is
+      # using the default batch size of 1. This method ensures that is the
+      # case and raises an error if it's not.
+      def assert_batch_size_one(method, targets)
+        if targets.length > 1
+          message = "#{self.class.name} must implement #{method} to support batches (got #{targets.length} nodes)"
+          raise NotImplementedError, message
+        end
+      end
+
+      # Runs the given task on a batch of nodes.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_task(targets, task, arguments, options = {}, &callback)
+        assert_batch_size_one("batch_task()", targets)
+        target = targets.first
+        with_events(target, callback) do
+          @logger.debug { "Running task run '#{task}' on #{target.uri}" }
+          run_task(target, task, arguments, filter_options(target, options))
+        end
+      end
+
+      # Runs the given command on a batch of nodes.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_command(targets, command, options = {}, &callback)
+        assert_batch_size_one("batch_command()", targets)
+        target = targets.first
+        with_events(target, callback) do
+          @logger.debug("Running command '#{command}' on #{target.uri}")
+          run_command(target, command, filter_options(target, options))
+        end
+      end
+
+      # Runs the given script on a batch of nodes.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_script(targets, script, arguments, options = {}, &callback)
+        assert_batch_size_one("batch_script()", targets)
+        target = targets.first
+        with_events(target, callback) do
+          @logger.debug { "Running script '#{script}' on #{target.uri}" }
+          run_script(target, script, arguments, filter_options(target, options))
+        end
+      end
+
+      # Uploads the given source file to the destination location on a batch of nodes.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implement their own batch processing.
+      def batch_upload(targets, source, destination, options = {}, &callback)
+        assert_batch_size_one("batch_upload()", targets)
+        target = targets.first
+        with_events(target, callback) do
+          @logger.debug { "Uploading: '#{source}' to #{destination} on #{target.uri}" }
+          upload(target, source, destination, filter_options(target, options))
+        end
+      end
+
+      # Split the given list of targets into a list of batches. The default
+      # implementation returns single-node batches.
+      #
+      # Transports may override this method, and the corresponding batch_*
+      # methods, to implement their own batch processing.
+      def batches(targets)
+        targets.map { |target| [target] }
+      end
+
+      # Transports should override this method with their own implementation of running a command.
+      def run_command(*_args)
+        raise NotImplementedError, "run_command() must be implemented by the transport class"
+      end
+
+      # Transports should override this method with their own implementation of running a script.
+      def run_script(*_args)
+        raise NotImplementedError, "run_script() must be implemented by the transport class"
+      end
+
+      # Transports should override this method with their own implementation of running a task.
+      def run_task(*_args)
+        raise NotImplementedError, "run_task() must be implemented by the transport class"
+      end
+
+      # Transports should override this method with their own implementation of file upload.
+      def upload(*_args)
+        raise NotImplementedError, "upload() must be implemented by the transport class"
+      end
+    end
+  end
+end

data/lib/bolt/transport/orch.rb

@@ -0,0 +1,158 @@
+require 'base64'
+require 'concurrent'
+require 'json'
+require 'orchestrator_client'
+require 'bolt/transport/base'
+require 'bolt/result'
+
+module Bolt
+  module Transport
+    class Orch < Base
+      CONF_FILE = File.expand_path('~/.puppetlabs/client-tools/orchestrator.conf')
+      BOLT_MOCK_TASK = Struct.new(:name, :executable).new('bolt', 'bolt/tasks/init').freeze
+
+      def initialize(config)
+        super
+
+        client_keys = %i[service-url token-file cacert]
+        @client_opts = config.select { |k, _v| client_keys.include?(k) }
+      end
+
+      def create_client
+        OrchestratorClient.new(@client_opts, true)
+      end
+
+      def build_request(targets, task, arguments)
+        { task: task.name,
+          environment: targets.first.options[:orch_task_environment],
+          noop: arguments['_noop'],
+          params: arguments.reject { |k, _| k == '_noop' },
+          scope: {
+            nodes: targets.map(&:host)
+          } }
+      end
+
+      def process_run_results(targets, results)
+        targets_by_name = Hash[targets.map(&:host).zip(targets)]
+        results.map do |node_result|
+          target = targets_by_name[node_result['name']]
+          state = node_result['state']
+          result = node_result['result']
+
+          # If it's finished or already has a proper error simply pass it to the
+          # the result otherwise make sure an error is generated
+          if state == 'finished' || (result && result['_error'])
+            Bolt::Result.new(target, value: result)
+          elsif state == 'skipped'
+            Bolt::Result.new(
+              target,
+              value: { '_error' => {
+                'kind' => 'puppetlabs.tasks/skipped-node',
+                'msg' => "Node #{target.host} was skipped",
+                'details' => {}
+              } }
+            )
+          else
+            # Make a generic error with a unkown exit_code
+            Bolt::Result.for_task(target, result.to_json, '', 'unknown')
+          end
+        end
+      end
+
+      def batch_command(targets, command, _options = {}, &callback)
+        results = run_task_job(targets,
+                               BOLT_MOCK_TASK,
+                               action: 'command',
+                               command: command,
+                               &callback)
+        callback ||= proc {}
+        results.map! { |result| unwrap_bolt_result(result.target, result) }
+        results.each do |result|
+          callback.call(type: :node_result, result: result)
+        end
+      end
+
+      def batch_script(targets, script, arguments, _options = {}, &callback)
+        content = File.open(script, &:read)
+        content = Base64.encode64(content)
+        params = {
+          action: 'script',
+          content: content,
+          arguments: arguments
+        }
+        callback ||= proc {}
+        results = run_task_job(targets, BOLT_MOCK_TASK, params, &callback)
+        results.map! { |result| unwrap_bolt_result(result.target, result) }
+        results.each do |result|
+          callback.call(type: :node_result, result: result)
+        end
+      end
+
+      def batch_upload(targets, source, destination, _options = {}, &callback)
+        content = File.open(source, &:read)
+        content = Base64.encode64(content)
+        mode = File.stat(source).mode
+        params = {
+          action: 'upload',
+          path: destination,
+          content: content,
+          mode: mode
+        }
+        callback ||= proc {}
+        results = run_task_job(targets, BOLT_MOCK_TASK, params, &callback)
+        results.map! do |result|
+          if result.error_hash
+            result
+          else
+            Bolt::Result.for_upload(result.target, source, destination)
+          end
+        end
+        results.each do |result|
+          callback.call(type: :node_result, result: result) if callback
+        end
+      end
+
+      def batches(targets)
+        targets.group_by { |target| target.options[:orch_task_environment] }.values
+      end
+
+      def run_task_job(targets, task, arguments)
+        body = build_request(targets, task, arguments)
+
+        targets.each do |target|
+          yield(type: :node_start, target: target) if block_given?
+        end
+
+        begin
+          results = create_client.run_task(body)
+
+          process_run_results(targets, results)
+        rescue StandardError => e
+          targets.map do |target|
+            Bolt::Result.from_exception(target, e)
+          end
+        end
+      end
+
+      def batch_task(targets, task, arguments, _options = {}, &callback)
+        callback ||= proc {}
+        results = run_task_job(targets, task, arguments, &callback)
+        results.each do |result|
+          callback.call(type: :node_result, result: result)
+        end
+      end
+
+      # run_task generates a result that makes sense for a generic task which
+      # needs to be unwrapped to extract stdout/stderr/exitcode.
+      #
+      def unwrap_bolt_result(target, result)
+        if result.error_hash
+          # something went wrong return the failure
+          return result
+        end
+
+        Bolt::Result.for_command(target, result.value['stdout'], result.value['stderr'], result.value['exit_code'])
+      end
+    end
+  end
+end