bolt 2.6.0 → 2.11.0

data/lib/bolt/transport/local/connection.rb

@@ -32,7 +32,8 @@ module Bolt
           if source.is_a?(StringIO)
             Tempfile.create(File.basename(dest)) do |f|
               f.write(source.read)
-              FileUtils.mv(t, dest)
+              f.close
+              FileUtils.mv(f, dest)
             end
           else
             # Mimic the behavior of `cp --remove-destination`
@@ -46,12 +47,11 @@ module Bolt
 
         def execute(command)
           if Bolt::Util.windows?
-            command += "\r\nif (!$?) { if($LASTEXITCODE) { exit $LASTEXITCODE } else { exit 1 } }"
-            script_file = Tempfile.new(['wrapper', '.ps1'], target.options['tmpdir'])
-            File.write(script_file, command)
-            script_file.close
-
-            command = ['powershell.exe', *Bolt::Shell::Powershell::PS_ARGS, script_file.path]
+            # If it's already a powershell command then invoke it normally.
+            # Otherwise, wrap it in powershell.exe.
+            unless command.start_with?('powershell.exe')
+              command = ['powershell.exe', *Bolt::Shell::Powershell::PS_ARGS, '-Command', command]
+            end
           end
 
           Open3.popen3(*command)
@@ -62,6 +62,12 @@ module Bolt
         def reset_cwd?
           false
         end
+
+        def max_command_length
+          if Bolt::Util.windows?
+            32000
+          end
+        end
       end
     end
   end

data/lib/bolt/transport/orch.rb

@@ -197,7 +197,7 @@ module Bolt
           end
         rescue StandardError => e
           targets.map do |target|
-            Bolt::Result.from_exception(target, e, 'task')
+            Bolt::Result.from_exception(target, e, action: 'task')
           end
         end
       end
@@ -210,6 +210,10 @@ module Bolt
         end
       end
 
+      def batch_task_with(_targets, _task, _target_mapping, _options = {})
+        raise NotImplementedError, "pcp transport does not support run_task_with()"
+      end
+
       def batch_connected?(targets)
         resp = get_connection(targets.first.options).query_inventory(targets)
         resp['items'].all? { |node| node['connected'] }

data/lib/bolt/transport/ssh.rb

@@ -16,13 +16,16 @@ module Bolt
         rescue LoadError
           logger.debug("Authentication method 'gssapi-with-mic' (Kerberos) is not available.")
         end
-
         @transport_logger = Logging.logger[Net::SSH]
         @transport_logger.level = :warn
       end
 
       def with_connection(target)
-        conn = Connection.new(target, @transport_logger)
+        conn = if target.transport_config['ssh-command']
+                 ExecConnection.new(target)
+               else
+                 Connection.new(target, @transport_logger)
+               end
         conn.connect
         yield conn
       ensure
@@ -37,3 +40,4 @@ module Bolt
 end
 
 require 'bolt/transport/ssh/connection'
+require 'bolt/transport/ssh/exec_connection'

data/lib/bolt/transport/ssh/connection.rb

@@ -108,6 +108,7 @@ module Bolt
           end
 
           @session = Net::SSH.start(target.host, @user, options)
+          validate_ssh_version
           @logger.debug { "Opened session" }
         rescue Net::SSH::AuthenticationFailed => e
           raise Bolt::Node::ConnectError.new(
@@ -206,6 +207,10 @@ module Bolt
             err_wr.close
           end
           [in_wr, out_rd, err_rd, th]
+        rescue Errno::EMFILE => e
+          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+            "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+          raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end
 
         def copy_file(source, destination)
@@ -242,7 +247,11 @@ module Bolt
         end
 
         def shell
-          @shell ||= Bolt::Shell::Bash.new(target, self)
+          @shell ||= if target.options['login-shell'] == 'powershell'
+                       Bolt::Shell::Powershell.new(target, self)
+                     else
+                       Bolt::Shell::Bash.new(target, self)
+                     end
         end
 
         # This is used by the Bash shell to decide whether to `cd` before
@@ -250,6 +259,22 @@ module Bolt
         def reset_cwd?
           true
         end
+
+        def max_command_length
+          if target.options['login-shell'] == 'powershell'
+            32000
+          end
+        end
+
+        def validate_ssh_version
+          remote_version = @session.transport.server_version.version
+          return unless target.options['login-shell'] && remote_version
+
+          match = remote_version.match(/OpenSSH_for_Windows_(\d+\.\d+)/)
+          if match && match[1].to_f < 7.9
+            raise "Powershell over SSH requires OpenSSH server >= 7.9, target is running #{match[1]}"
+          end
+        end
       end
     end
   end

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Bolt
+  module Transport
+    class SSH < Simple
+      class ExecConnection
+        attr_reader :user, :target
+
+        def initialize(target)
+          raise Bolt::ValidationError, "Target #{target.safe_name} does not have a host" unless target.host
+
+          @target = target
+          ssh_config = Net::SSH::Config.for(target.host)
+          @user = @target.user || ssh_config[:user] || Etc.getlogin
+          @logger = Logging.logger[self]
+        end
+
+        # This is used to verify we can connect to targets with `connected?`
+        def connect
+          cmd = build_ssh_command('exit')
+          _, err, stat = Open3.capture3(*cmd)
+          unless stat.success?
+            raise Bolt::Node::ConnectError.new(
+              "Failed to connect to #{@target.safe_name}: #{err}",
+              'CONNECT_ERROR'
+            )
+          end
+        end
+
+        def disconnect; end
+
+        def shell
+          Bolt::Shell::Bash.new(@target, self)
+        end
+
+        def userhost
+          "#{@user}@#{@target.host}"
+        end
+
+        def ssh_opts
+          cmd = []
+          # BatchMode is SSH's noninteractive option: if key authentication
+          # fails it will error out instead of falling back to password prompt
+          cmd += %w[-o BatchMode=yes]
+          cmd += %W[-o Port=#{@target.port}] if @target.port
+
+          if @target.transport_config.key?('host-key-check')
+            hkc = @target.transport_config['host-key-check'] ? 'yes' : 'no'
+            cmd += %W[-o StrictHostKeyChecking=#{hkc}]
+          end
+
+          if (key = target.transport_config['private-key'])
+            cmd += ['-i', key]
+          end
+          cmd
+        end
+
+        def build_ssh_command(command)
+          ssh_conf = @target.transport_config['ssh-command']
+          ssh_cmd = Array(ssh_conf)
+          ssh_cmd += ssh_opts
+          ssh_cmd << userhost
+          ssh_cmd << command
+        end
+
+        def copy_file(source, dest)
+          @logger.debug { "Uploading #{source}, to #{userhost}:#{dest}" } unless source.is_a?(StringIO)
+
+          cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
+          cp_cmd = Array(cp_conf)
+          cp_cmd += ssh_opts
+
+          _, err, stat = if source.is_a?(StringIO)
+                           Tempfile.create(File.basename(dest)) do |f|
+                             f.write(source.read)
+                             f.close
+                             cp_cmd << f.path
+                             cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                             Open3.capture3(*cp_cmd)
+                           end
+                         else
+                           cp_cmd << source
+                           cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                           Open3.capture3(*cp_cmd)
+                         end
+
+          if stat.success?
+            @logger.debug "Successfully uploaded #{source} to #{dest}"
+          else
+            message = "Could not copy file to #{dest}: #{err}"
+            raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
+          end
+        end
+
+        def execute(command)
+          cmd_array = build_ssh_command(command)
+          Open3.popen3(*cmd_array)
+        end
+
+        # This is used by the Bash shell to decide whether to `cd` before
+        # executing commands as a run-as user
+        def reset_cwd?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/bolt/transport/winrm/connection.rb

@@ -108,8 +108,8 @@ module Bolt
           # it will fail if the shell attempts to provide stdin
           inp.close
 
-          out_rd, out_wr = IO.pipe
-          err_rd, err_wr = IO.pipe
+          out_rd, out_wr = IO.pipe('UTF-8')
+          err_rd, err_wr = IO.pipe('UTF-8')
           th = Thread.new do
             result = @session.run(command)
             out_wr << result.stdout
@@ -120,6 +120,10 @@ module Bolt
           end
 
           [inp, out_rd, err_rd, th]
+        rescue Errno::EMFILE => e
+          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+            "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+          raise Bolt::Error.new(msg, 'bolt/too-many-files')
         rescue StandardError
           @logger.debug { "Command aborted" }
           raise
@@ -175,6 +179,10 @@ module Bolt
           @shell ||= Bolt::Shell::Powershell.new(target, self)
         end
 
+        def max_command_length
+          nil
+        end
+
         private
 
         def smb_client_login

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '2.6.0'
+  VERSION = '2.11.0'
 end

data/lib/bolt_server/pe/pal.rb

@@ -51,50 +51,13 @@ module BoltServer
         basemodulepath = plan_executor_config['basemodulepath'] || "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
 
         with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
-          modulepath_dirs = []
-          modulepath_setting_from_bolt = nil
           environment = Puppet.lookup(:environments).get!(environment_name)
-          path_to_env = environment.configuration.path_to_env
-
-          # In the instance where the environment is "production" but no production dir
-          # exists, the lookup will succeed, but the configuration will be mostly empty.
-          # For other environments the lookup will fail, but for production we don't
-          # want cryptic messages sent to the user about combining `nil` with a string.
-          # Thus if we do get here and `path_to_env` is empty, just assume it's the
-          # default production environment and continue.
-          #
-          # This should hopefully match puppet's behavior for the default 'production'
-          # environment: _technically_ that environment always exists, but if the dir
-          # isn't there it won't find the module and fail with "plan not found" rather
-          # than "environment doesn't exist"
-          if path_to_env
-            bolt_yaml = File.join(environment.configuration.path_to_env, 'bolt.yaml')
-            modulepath_setting_from_bolt = Bolt::Util.read_optional_yaml_hash(bolt_yaml, 'config')['modulepath']
-          end
-
-          # If we loaded a bolt.yaml in the environment root and it contained a modulepath setting:
-          # we will use that modulepath rather than the one loaded through puppet. modulepath will
-          # be the _only_ setting that will work from bolt.yaml in plans in PE.
-          if modulepath_setting_from_bolt
-            modulepath_setting_from_bolt.split(File::PATH_SEPARATOR).each do |path|
-              if Pathname.new(path).absolute? && File.exist?(path)
-                modulepath_dirs << path
-              elsif File.exist?(File.join(path_to_env, path))
-                modulepath_dirs << File.join(path_to_env, path)
-              end
-            end
-
-            # Append the basemodulepath to include "built-in" modules.
-            modulepath_dirs.concat(basemodulepath.split(File::PATH_SEPARATOR))
-          else
-            modulepath_dirs = environment.modulepath
-          end
-
           # A new modulepath is created from scratch (rather than using super's @modulepath)
           # so that we can have full control over all the entries in modulepath. In the future
           # it's likely we will need to preceed _both_ Bolt::PAL::BOLTLIB_PATH _and_
           # Bolt::PAL::MODULES_PATH which would be more complex if we tried to use @modulepath since
           # we need to append our modulepaths and exclude modules shiped in bolt gem code
+          modulepath_dirs = environment.modulepath
           @original_modulepath = modulepath_dirs
           @modulepath = [PE_BOLTLIB_PATH, Bolt::PAL::BOLTLIB_PATH, *modulepath_dirs]
         end

data/lib/bolt_server/transport_app.rb

@@ -57,8 +57,8 @@ module BoltServer
     end
 
     def scrub_stack_trace(result)
-      if result.dig(:value, '_error', 'details', 'stack_trace')
-        result[:value]['_error']['details'].reject! { |k| k == 'stack_trace' }
+      if result.dig('value', '_error', 'details', 'stack_trace')
+        result['value']['_error']['details'].reject! { |k| k == 'stack_trace' }
       end
       result
     end
@@ -87,14 +87,14 @@ module BoltServer
     # If the `result_set` contains only one item, it will be returned
     # as a single result object. Set `aggregate` to treat it as a set
     # of results with length 1 instead.
-    def result_set_to_status_hash(result_set, aggregate: false)
+    def result_set_to_data(result_set, aggregate: false)
       scrubbed_results = result_set.map do |result|
-        scrub_stack_trace(result.status_hash)
+        scrub_stack_trace(result.to_data)
       end
 
       if aggregate || scrubbed_results.length > 1
         # For actions that act on multiple targets, construct a status hash for the aggregate result
-        all_succeeded = scrubbed_results.all? { |r| r[:status] == 'success' }
+        all_succeeded = scrubbed_results.all? { |r| r['status'] == 'success' }
         {
           status: all_succeeded ? 'success' : 'failure',
           result: scrubbed_results
@@ -297,7 +297,7 @@ module BoltServer
       return [400, error.to_json] unless error.nil?
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     def make_winrm_target(target_hash)
@@ -337,7 +337,7 @@ module BoltServer
       return [400, error.to_json] if error
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     # Fetches the metadata for a single plan

data/lib/bolt_spec/bolt_context.rb

@@ -138,21 +138,18 @@ module BoltSpec
     # Override in your tests
     def config
       @config ||= begin
-                    conf = Bolt::Config.new(Bolt::Boltdir.new('.'), {})
+                    conf = Bolt::Config.new(Bolt::Project.new('.'), {})
                     conf.modulepath = [modulepath].flatten
                     conf
                   end
     end
 
     def plugins
-      @plugins ||= Bolt::Plugin.setup(config,
-                                      pal,
-                                      nil,
-                                      Bolt::Analytics::NoopClient.new)
+      @plugins ||= Bolt::Plugin.setup(config, pal)
     end
 
     def pal
-      @pal ||= Bolt::PAL.new(config.modulepath, config.hiera_config, config.boltdir.resource_types)
+      @pal ||= Bolt::PAL.new(config.modulepath, config.hiera_config, config.project.resource_types)
     end
 
     BoltSpec::Plans::MOCKED_ACTIONS.each do |action|

data/lib/bolt_spec/plans.rb

@@ -40,6 +40,23 @@ require 'bolt/pal'
 #  an otherwise empty bolt config and inventory. To create your own values for
 #  these override the modulepath, config, or inventory methods.
 #
+# Sub-plan Execution
+#
+#  When testing a plan, often times those plans call other plans in order to
+#  build complex workflows. To support this we offer running in two different
+#  modes:
+#    execute_any_plan (default) - This mode will execute any plan that is encountered
+#      without having to be stubbed/mocked. This default mode allows for plan control
+#      flow to behave as normal. If you choose to stub/mock out a sub-plan in this mode
+#      that will be honored and the sub-plan will not be executed. We will use the modifiers
+#      on the stub to check for the conditions specified (example: be_called_times(3))
+#
+#    execute_no_plan - This mode will not execute a plans that it encounters. Instead, when
+#      a plan is encountered it will throw an error unless the plan is mocked out. This
+#      mode is useful for ensuring that there are no plans called that you do not expect.
+#      This plan requires authors to mock out all sub-plans that may be invoked when running
+#      tests.
+#
 #  TODO:
 #  - Allow description based stub matching
 #  - Better testing of plan errors
@@ -47,15 +64,20 @@ require 'bolt/pal'
 #  - Allow stubbing with a block(at the double level? As a matched stub?)
 #  - package code so that it can be used for testing modules outside of this repo
 #  - set subject from describe and provide matchers similar to rspec puppets function tests
+#  - Allow specific plans to be executed when running in execute_no_plan mode.
 #
 #  MAYBE TODO?:
-#  - allow stubbing for subplans
 #  - validate call expectations at the end of the example instead of in run_plan
 #  - resultset matchers to help testing canary like plans?
 #  - inventory matchers to help testing plans that change inventory
 #
+# Flags:
+# - execute_any_plan: execute any plan that is encountered unless it is mocked (default)
+# - execute_no_plan: throw an error if a plan is encountered that is not stubbed
+#
 # Stubs:
 # - allow_command(cmd), expect_command(cmd): expect the exact command
+# - allow_plan(plan), expect_plan(plan): expect the named plan
 # - allow_script(script), expect_script(script): expect the script as <module>/path/to/file
 # - allow_task(task), expect_task(task): expect the named task
 # - allow_upload(file), expect_upload(file): expect the identified source file
@@ -69,22 +91,28 @@ require 'bolt/pal'
 #                       if expected, fail unless the action is called 'n' times
 # - not_be_called: fail if the action is called
 # - with_targets(targets): target or list of targets that you expect to be passed to the action
+#                          plan: does not support this modifier
 # - with_params(params): list of params and metaparams (or options) that you expect to be passed to the action.
 #                        Corresponds to the action's last argument.
 # - with_destination(dest): for upload_file, the expected destination path
 # - always_return(value): return a Bolt::ResultSet of Bolt::Result objects with the specified value Hash
+#                         plan: returns a Bolt::PlanResult with the specified value with a status of 'success'
 #                         command and script: only accept 'stdout' and 'stderr' keys
 #                         upload: does not support this modifier
 # - return_for_targets(targets_to_values): return a Bolt::ResultSet of Bolt::Result objects from the Hash mapping
 #                                          targets to their value Hashes
 #                                          command and script: only accept 'stdout' and 'stderr' keys
 #                                          upload: does not support this modifier
+#                                          plan: does not support this modifier
 # - return(&block): invoke the block to construct a Bolt::ResultSet. The blocks parameters differ based on action
 #                   command: `{ |targets:, command:, params:| ... }`
+#                   plan: `{ |plan:, params:| ... }`
 #                   script: `{ |targets:, script:, params:| ... }`
 #                   task: `{ |targets:, task:, params:| ... }`
 #                   upload: `{ |targets:, source:, destination:, params:| ... }`
 # - error_with(err): return a failing Bolt::ResultSet, with Bolt::Result objects with the identified err hash
+#                    plans will throw a Bolt::PlanFailure that will be returned as the value of
+#                    the Bolt::PlanResult object with a status of 'failure'.
 #
 # Example:
 #   describe "my_plan" do
@@ -128,12 +156,38 @@ require 'bolt/pal'
 #       end
 #       expect(run_plan('my_plan', { 'param1' => 10 })).to eq(10)
 #     end
-
+#
 #     it 'expects multiple messages to out::message' do
 #       expect_out_message.be_called_times(2).with_params(message)
 #       result = run_plan(plan_name, 'messages' => [message, message])
 #       expect(result).to be_ok
 #     end
+#
+#     it 'expects a sub-plan to be called' do
+#       expect_plan('module::sub_plan').with_params('targets' => ['foo']).be_called_times(1)
+#       result = run_plan('module::main_plan', 'targets' => ['foo'])
+#       expect(result).to be_ok
+#       expect(result.class).to eq(Bolt::PlanResult)
+#       expect(result.value).to eq('foo' => 'is_good')
+#       expect(result.status).to eq('success')
+#     end
+#
+#     it 'error when sub-plan is called' do
+#       execute_no_plan
+#       err = 'Unexpected call to 'run_plan(module::sub_plan, {\"targets\"=>[\"foo\"]})'
+#       expect { run_plan('module::main_plan', 'targets' => ['foo']) }
+#         .to raise_error(RuntimeError, err)
+#     end
+#
+#     it 'errors when plan calls fail_plan()' do
+#       result = run_plan('module::calls_fail_plan', {})
+#       expect(result).not_to be_ok
+#       expect(result.class).to eq(Bolt::PlanResult)
+#       expect(result.status).to eq('failure')
+#       expect(result.value.class).to eq(Bolt::PlanFailure)
+#       expect(result.value.msg).to eq('failure message passed to fail_plan()')
+#       expect(result.value.kind).to eq('bolt/plan-failure')
+#     end
 #   end
 #
 # See spec/bolt_spec/plan_spec.rb for more examples.
@@ -165,8 +219,10 @@ module BoltSpec
     end
 
     def run_plan(name, params)
-      pal = Bolt::PAL.new(config.modulepath, config.hiera_config, config.boltdir.resource_types)
-      result = pal.run_plan(name, params, executor, inventory, puppetdb_client)
+      pal = Bolt::PAL.new(config.modulepath, config.hiera_config, config.project.resource_types)
+      result = executor.with_plan_allowed_exec(name, params) do
+        pal.run_plan(name, params, executor, inventory, puppetdb_client)
+      end
 
       if executor.error_message
         raise executor.error_message
@@ -175,7 +231,7 @@ module BoltSpec
       begin
         executor.assert_call_expectations
       rescue StandardError => e
-        raise "#{e.message}\nPlan result: #{result}"
+        raise "#{e.message}\nPlan result: #{result}\n#{e.backtrace.join("\n")}"
       end
 
       result
@@ -196,9 +252,23 @@ module BoltSpec
       nil
     end
 
-    # Plan execution does not flow through the executor mocking may make sense but
-    # will be a separate effort.
-    # def allow_plan(plan_name)
+    # Flag for the default behavior of executing sub-plans during testing
+    # By *default* we allow any sub-plan to be executed, no mocking required.
+    # Users can still mock out plans in this mode and the mocks will check for
+    # parameters and return values like normal. However, if a plan isn't explicitly
+    # mocked out, it will be executed.
+    def execute_any_plan
+      executor.execute_any_plan = true
+    end
+
+    # If you want to explicitly mock out all of the sub-plan calls, then
+    # call this prior to calling `run_plan()` along with setting up any
+    # mocks that you require.
+    # In this mode, any plan that is not explicitly mocked out will not be executed
+    # and an error will be thrown.
+    def execute_no_plan
+      executor.execute_any_plan = false
+    end
 
     # intended to be private below here
   end