bolt 3.12.0 → 3.16.0

data/lib/bolt/plan_result.rb

@@ -6,7 +6,7 @@ require 'bolt/util'
 
 module Bolt
   class PlanResult
-    attr_accessor :value, :status
+    attr_accessor :status, :value
 
     # This must be called from inside a compiler
     def self.from_pcore(result, status)

data/lib/bolt/plugin/task.rb

@@ -59,7 +59,7 @@ module Bolt
         run_opts = {}
         run_opts[:run_as] = opts['_run_as'] if opts['_run_as']
         begin
-          task = apply_prep.get_task(opts['task'], params)
+          task = @context.get_validated_task(opts['task'], params)
         rescue Bolt::Error => e
           raise Bolt::Plugin::PluginError::ExecutionError.new(e.message, name, 'puppet_library')
         end

data/lib/bolt/project.rb

@@ -209,12 +209,5 @@ module Bolt
         Bolt::Logger.warn("missing_project_name", message)
       end
     end
-
-    def check_deprecated_file
-      if (@path + 'project.yaml').file?
-        msg = "Project configuration file 'project.yaml' is deprecated; use 'bolt-project.yaml' instead."
-        Bolt::Logger.warn("project_yaml", msg)
-      end
-    end
   end
 end

data/lib/bolt/result_set.rb

@@ -2,7 +2,8 @@
 
 module Bolt
   class ResultSet
-    attr_reader :results
+    attr_accessor :elapsed_time
+    attr_reader   :results
 
     include Enumerable
 

data/lib/bolt/transport/local/connection.rb

@@ -10,6 +10,8 @@ module Bolt
   module Transport
     class Local < Simple
       class Connection
+        RUBY_ENV_VARS = %w[GEM_PATH GEM_HOME RUBYLIB RUBYLIB_PREFIX RUBYOPT RUBYPATH RUBYSHELL].freeze
+
         attr_accessor :user, :logger, :target
 
         def initialize(target)
@@ -68,7 +70,21 @@ module Bolt
             end
           end
 
-          Open3.popen3(*command)
+          # Only do this if bundled-ruby is set to false, not nil
+          ruby_env_vars = if target.transport_config['bundled-ruby'] == false
+                            RUBY_ENV_VARS.each_with_object({}) do |e, acc|
+                              acc[e] = ENV["BOLT_ORIG_#{e}"] if ENV["BOLT_ORIG_#{e}"]
+                            end
+                          end
+
+          if target.transport_config['bundled-ruby'] == false &&
+             Gem.loaded_specs.keys.include?('bundler')
+            Bundler.with_unbundled_env do
+              Open3.popen3(ruby_env_vars || {}, *command)
+            end
+          else
+            Open3.popen3(ruby_env_vars || {}, *command)
+          end
         end
 
         # This is used by the Bash shell to decide whether to `cd` before

data/lib/bolt/transport/orch/connection.rb

@@ -36,7 +36,8 @@ module Bolt
           end
           logger.debug("Creating orchestrator client for #{client_opts}")
           @client = OrchestratorClient.new(client_opts, true)
-          @plan_job = start_plan(plan_context)
+          @plan_context = plan_context
+          @plan_job = start_plan(@plan_context)
           logger.debug("Started plan #{@plan_job}")
           @environment = opts["task-environment"]
         end
@@ -87,6 +88,17 @@ module Bolt
         def run_task(targets, task, arguments, options)
           body = build_request(targets, task, arguments, options[:description])
           @client.run_task(body)
+        rescue OrchestratorClient::ApiError => e
+          if e.data['kind'] == 'puppetlabs.orchestrator/plan-already-finished'
+            @logger.debug("Retrying the task")
+            # Instead of recursing, just retry once
+            @plan_job = start_plan(@plan_context)
+            # Rebuild the request with the new plan job ID
+            body = build_request(targets, task, arguments, options[:description])
+            @client.run_task(body)
+          else
+            raise e
+          end
         end
 
         def query_inventory(targets)

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -47,7 +47,9 @@ module Bolt
           cmd = []
           # BatchMode is SSH's noninteractive option: if key authentication
           # fails it will error out instead of falling back to password prompt
-          cmd += %w[-o BatchMode=yes]
+          batch_mode = @target.transport_config['batch-mode'] ? 'yes' : 'no'
+          cmd += %W[-o BatchMode=#{batch_mode}]
+
           cmd += %W[-o Port=#{@target.port}] if @target.port
 
           if @target.transport_config.key?('host-key-check')

data/lib/bolt/util/format.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Bolt
+  module Util
+    module Format
+      class << self
+        # Stringifies an object, formatted as valid JSON.
+        #
+        # @param message [Object] The object to stringify.
+        # @return [String] The JSON string.
+        #
+        def stringify(message)
+          formatted = format_message(message)
+          if formatted.is_a?(Hash) || formatted.is_a?(Array)
+            ::JSON.pretty_generate(formatted)
+          else
+            formatted
+          end
+        end
+
+        # Recursively formats an object into a format that can be represented by
+        # JSON.
+        #
+        # @param message [Object] The object to stringify.
+        # @return [Array, Hash, String]
+        #
+        private def format_message(message)
+          case message
+          when Array
+            message.map { |item| format_message(item) }
+          when Bolt::ApplyResult
+            format_apply_result(message)
+          when Bolt::Result, Bolt::ResultSet
+            # This is equivalent to to_s, but formattable
+            message.to_data
+          when Bolt::RunFailure
+            formatted_resultset = message.result_set.to_data
+            message.to_h.merge('result_set' => formatted_resultset)
+          when Hash
+            message.each_with_object({}) do |(k, v), h|
+              h[format_message(k)] = format_message(v)
+            end
+          when Integer, Float, NilClass
+            message
+          else
+            message.to_s
+          end
+        end
+
+        # Formats a Bolt::ApplyResult object.
+        #
+        # @param result [Bolt::ApplyResult] The apply result.
+        # @return [Hash]
+        #
+        private def format_apply_result(result)
+          logs = result.resource_logs&.map do |log|
+            # Omit low-level info/debug messages
+            next if %w[info debug].include?(log['level'])
+            indent(2, format_log(log))
+          end
+          hash = result.to_data
+          hash['logs'] = logs unless logs.empty?
+          hash
+        end
+      end
+    end
+  end
+end

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.12.0'
+  VERSION = '3.16.0'
 end

data/lib/bolt_server/schemas/partials/target-ssh.json

@@ -62,6 +62,10 @@
     "interpreters": {
       "type": "object",
       "description": "Map of file extensions to remote executable"
+    },
+    "plugin_hooks": {
+      "type": "object",
+      "description": "Configuration for plugins to use"
     }
   },
   "oneOf": [

data/lib/bolt_server/schemas/partials/target-winrm.json

@@ -56,6 +56,10 @@
     "smb-port": {
       "type": "integer",
       "description": "Port for SMB protocol"
+    },
+    "plugin_hooks": {
+      "type": "object",
+      "description": "Configuration for plugins to use"
     }
   },
   "required": ["hostname", "user", "password"],

data/lib/bolt_server/transport_app.rb

@@ -120,9 +120,20 @@ module BoltServer
 
     def run_task(target, body)
       validate_schema(@schemas["action-run_task"], body)
+
       task_data = body['task']
       task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
       parameters = body['parameters'] || {}
+      # Wrap parameters marked with '"sensitive": true' in the task metadata with a
+      # Sensitive wrapper type. This way it's not shown in logs.
+      if (param_spec = task.parameters)
+        parameters.each do |k, v|
+          if param_spec[k] && param_spec[k]['sensitive']
+            parameters[k] = Puppet::Pops::Types::PSensitiveType::Sensitive.new(v)
+          end
+        end
+      end
+
       @executor.run_task(target, task, parameters).each do |result|
         value = result.value
         next unless value.is_a?(Hash)
@@ -378,6 +389,63 @@ module BoltServer
       end
     end
 
+    # The provided block takes a module object and returns the list
+    # of directories to search through. This is similar to
+    # Bolt::Applicator.build_plugin_tarball.
+    def build_project_plugins_tarball(versioned_project, &block)
+      start_time = Time.now
+
+      # Fetch the plugin files
+      plugin_files = in_bolt_project(versioned_project) do |context|
+        files = {}
+
+        # Bolt also sets plugin_modulepath to user modulepath so do it here too for
+        # consistency
+        plugin_modulepath = context[:pal].user_modulepath
+        Puppet.lookup(:current_environment).override_with(modulepath: plugin_modulepath).modules.each do |mod|
+          search_dirs = block.call(mod)
+
+          files[mod] ||= []
+          Find.find(*search_dirs).each do |file|
+            files[mod] << file if File.file?(file)
+          end
+        end
+
+        files
+      end
+
+      # Pack the plugin files
+      sio = StringIO.new
+      begin
+        output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
+
+        plugin_files.each do |mod, files|
+          tar_dir = Pathname.new(mod.name)
+          mod_dir = Pathname.new(mod.path)
+
+          files.each do |file|
+            tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
+            stat = File.stat(file)
+            content = File.binread(file)
+            output.tar.add_file_simple(
+              tar_path.to_s,
+              data: content,
+              size: content.size,
+              mode: stat.mode & 0o777,
+              mtime: stat.mtime
+            )
+          end
+        end
+
+        duration = Time.now - start_time
+        @logger.trace("Packed plugins in #{duration * 1000} ms")
+      ensure
+        output.close
+      end
+
+      Base64.encode64(sio.string)
+    end
+
     get '/' do
       200
     end
@@ -430,7 +498,7 @@ module BoltServer
         'uri' => target_hash['hostname'],
         'config' => {
           'transport' => 'ssh',
-          'ssh' => opts
+          'ssh' => opts.slice(*Bolt::Config::Transport::SSH.options)
         }
       }
 
@@ -468,7 +536,7 @@ module BoltServer
         'uri' => target_hash['hostname'],
         'config' => {
           'transport' => 'winrm',
-          'winrm' => opts
+          'winrm' => opts.slice(*Bolt::Config::Transport::WinRM.options)
         }
       }
 
@@ -688,60 +756,34 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      # Inspired by Bolt::Applicator.build_plugin_tarball
-      start_time = Time.now
-
-      # Fetch the plugin files
-      plugin_files = in_bolt_project(params['versioned_project']) do |context|
-        files = {}
-
-        # Bolt also sets plugin_modulepath to user modulepath so do it here too for
-        # consistency
-        plugin_modulepath = context[:pal].user_modulepath
-        Puppet.lookup(:current_environment).override_with(modulepath: plugin_modulepath).modules.each do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-
-          files[mod] ||= []
-          Find.find(*search_dirs).each do |file|
-            files[mod] << file if File.file?(file)
-          end
-        end
-
-        files
+      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
+        search_dirs = []
+        search_dirs << mod.plugins if mod.plugins?
+        search_dirs << mod.pluginfacts if mod.pluginfacts?
+        search_dirs
       end
 
-      # Pack the plugin files
-      sio = StringIO.new
-      begin
-        output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
-
-        plugin_files.each do |mod, files|
-          tar_dir = Pathname.new(mod.name)
-          mod_dir = Pathname.new(mod.path)
+      [200, plugins_tarball.to_json]
+    end
 
-          files.each do |file|
-            tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
-            stat = File.stat(file)
-            content = File.binread(file)
-            output.tar.add_file_simple(
-              tar_path.to_s,
-              data: content,
-              size: content.size,
-              mode: stat.mode & 0o777,
-              mtime: stat.mtime
-            )
-          end
-        end
+    # Returns the base64 encoded tar archive of _all_ plugin code for a project
+    #
+    # @param versioned_project [String] the versioned_project to build the plugin tarball from
+    get '/project_plugin_tarball' do
+      raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
+      content_type :json
 
-        duration = Time.now - start_time
-        @logger.trace("Packed plugins in #{duration * 1000} ms")
-      ensure
-        output.close
+      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
+        search_dirs = []
+        search_dirs << mod.plugins if mod.plugins?
+        search_dirs << mod.pluginfacts if mod.pluginfacts?
+        search_dirs << mod.files if mod.files?
+        type_files = "#{mod.path}/types"
+        search_dirs << type_files if File.exist?(type_files)
+        search_dirs
       end
 
-      [200, Base64.encode64(sio.string).to_json]
+      [200, plugins_tarball.to_json]
     end
 
     error 404 do

data/lib/bolt_spec/bolt_context.rb

@@ -191,6 +191,15 @@ module BoltSpec
       allow_out_message.expect_call
     end
 
+    def allow_out_verbose
+      executor.stub_out_verbose.add_stub
+    end
+    alias allow_any_out_verbose allow_out_verbose
+
+    def expect_out_verbose
+      allow_out_verbose.expect_call
+    end
+
     # Example helpers to mock other run functions
     # The with_targets method makes sense for all stubs
     # with_params could be reused for options

data/lib/bolt_spec/plans.rb

@@ -98,7 +98,7 @@ module BoltSpec
       Puppet[:tasks] = true
 
       # Ensure logger is initialized with Puppet levels so 'notice' works when running plan specs.
-      Logging.init :trace, :debug, :info, :notice, :warn, :error, :fatal, :any
+      Logging.init :trace, :debug, :info, :notice, :warn, :error, :fatal
     end
 
     # Provided as a class so expectations can be placed on it.

data/lib/bolt_spec/plans/mock_executor.rb

@@ -29,6 +29,7 @@ module BoltSpec
         @modulepath = [modulepath].flatten.map { |path| File.absolute_path(path) }
         MOCKED_ACTIONS.each { |action| instance_variable_set(:"@#{action}_doubles", {}) }
         @stub_out_message = nil
+        @stub_out_verbose = nil
         @transport_features = ['puppet-agent']
         @executor_real = Bolt::Executor.new
         # by default, we want to execute any plan that we come across without error
@@ -38,6 +39,7 @@ module BoltSpec
         # plans that are allowed to be executed by the @executor_real
         @allowed_exec_plans = {}
         @id = 0
+        @plan_futures = []
       end
 
       def module_file_id(file)
@@ -187,6 +189,7 @@ module BoltSpec
           end
         end
         @stub_out_message.assert_called('out::message') if @stub_out_message
+        @stub_out_verbose.assert_called('out::verbose') if @stub_out_verbose
       end
 
       MOCKED_ACTIONS.each do |action|
@@ -199,6 +202,10 @@ module BoltSpec
         @stub_out_message ||= ActionDouble.new(:PublishStub)
       end
 
+      def stub_out_verbose
+        @stub_out_verbose ||= ActionDouble.new(:PublishStub)
+      end
+
       def stub_apply
         @allow_apply = true
       end
@@ -220,12 +227,20 @@ module BoltSpec
       end
 
       def publish_event(event)
-        if event[:type] == :message
+        case event[:type]
+        when :message
           unless @stub_out_message
             @error_message = "Unexpected call to 'out::message(#{event[:message]})'"
             raise UnexpectedInvocation, @error_message
           end
           @stub_out_message.process(event[:message])
+
+        when :verbose
+          unless @stub_out_verbose
+            @error_message = "Unexpected call to 'out::verbose(#{event[:message]})'"
+            raise UnexpectedInvocation, @error_message
+          end
+          @stub_out_verbose.process(event[:message])
         end
       end
 
@@ -266,7 +281,7 @@ module BoltSpec
         false
       end
 
-      def create_future(scope: nil, name: nil)
+      def create_future(plan_id:, scope: nil, name: nil)
         newscope = nil
         if scope
           # Create the new scope
@@ -281,13 +296,18 @@ module BoltSpec
         # Execute "futures" serially when running in BoltSpec
         result = yield newscope
         @id += 1
-        future = Bolt::PlanFuture.new(nil, @id, name: name)
+        future = Bolt::PlanFuture.new(nil, @id, name: name, plan_id: plan_id)
         future.value = result
+        @plan_futures << future
         future
       end
 
-      def wait(results, **_kwargs)
-        results
+      def get_futures_for_plan(plan_id:)
+        @plan_futures.select { |future| future.plan_id == plan_id }
+      end
+
+      def wait(futures, **_kwargs)
+        futures.map(&:value)
       end
 
       # Since Futures are executed immediately once created, this will always
@@ -296,8 +316,12 @@ module BoltSpec
         true
       end
 
-      def plan_futures
-        []
+      def get_current_future(fiber)
+        @plan_futures.select { |f| f.fiber == fiber }&.first
+      end
+
+      def get_current_plan_id(fiber)
+        get_current_future(fiber)&.current_plan
       end
 
       # Public methods on Bolt::Executor that need to be mocked so there aren't