bolt 2.42.0 → 3.3.0

data/lib/bolt/project_manager.rb

@@ -143,7 +143,7 @@ module Bolt
       @outputter.print_message("Migrating project #{@config.project.path}\n\n")
 
       @outputter.print_action_step(
-        "Migrating a Bolt project may make irreversible changes to the project's "\
+        "Migrating a Bolt project might make irreversible changes to the project's "\
         "configuration and inventory files. Before continuing, make sure the "\
         "project has a backup or uses a version control system."
       )
@@ -166,10 +166,11 @@ module Bolt
     # Migrates the project-level configuration file to the latest version.
     #
     private def migrate_config
-      migrator = ConfigMigrator.new(@outputter)
+      migrator   = ConfigMigrator.new(@outputter)
+      configfile = @config.project.path + 'bolt.yaml'
 
       migrator.migrate(
-        @config.project.config_file,
+        configfile,
         @config.project.project_file,
         @config.inventoryfile || @config.project.inventory_file,
         @config.project.backup_dir
@@ -194,7 +195,7 @@ module Bolt
 
       migrator.migrate(
         @config.project,
-        @config.modulepath
+        @config.modulepath[0...-1]
       )
     end
   end

data/lib/bolt/project_manager/module_migrator.rb

@@ -6,19 +6,20 @@ module Bolt
   class ProjectManager
     class ModuleMigrator < Migrator
       def migrate(project, configured_modulepath)
-        return true unless project.modules.nil?
+        return true if project.managed_moduledir.exist?
 
         @outputter.print_message "Migrating project modules\n\n"
 
         config            = project.project_file
         puppetfile        = project.puppetfile
         managed_moduledir = project.managed_moduledir
-        modulepath        = [(project.path + 'modules').to_s,
+        new_modulepath    = [(project.path + 'modules').to_s]
+        old_modulepath    = [(project.path + 'modules').to_s,
                              (project.path + 'site-modules').to_s,
                              (project.path + 'site').to_s]
 
         # Notify user to manually migrate modules if using non-default modulepath
-        if configured_modulepath != modulepath
+        if configured_modulepath != new_modulepath && configured_modulepath != old_modulepath
           @outputter.print_action_step(
             "Project has a non-default configured modulepath, unable to automatically "\
             "migrate project modules. To migrate project modules manually, see "\
@@ -27,10 +28,10 @@ module Bolt
           true
         # Migrate modules from Puppetfile
         elsif File.exist?(puppetfile)
-          migrate_modules_from_puppetfile(config, puppetfile, managed_moduledir, modulepath)
+          migrate_modules_from_puppetfile(config, puppetfile, managed_moduledir, old_modulepath)
         # Migrate modules to updated modulepath
         else
-          consolidate_modules(modulepath)
+          consolidate_modules(old_modulepath)
           update_project_config([], config)
         end
       end
@@ -65,7 +66,7 @@ module Bolt
         # Attempt to resolve dependencies
         begin
           @outputter.print_message('')
-          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          @outputter.print_action_step("Resolving module dependencies, this might take a moment")
           puppetfile = Bolt::ModuleInstaller::Resolver.new.resolve(specs)
         rescue Bolt::Error => e
           @outputter.print_action_error("#{e.message}\nSkipping module migration.")

data/lib/bolt/rerun.rb

@@ -49,7 +49,7 @@ module Bolt
         end
       end
     rescue StandardError => e
-      Bolt::Logger.warn_once('unwriteable_file', "Failed to save result to #{@path}: #{e.message}")
+      Bolt::Logger.warn_once("unwriteable_file", "Failed to save result to #{@path}: #{e.message}")
     end
   end
 end

data/lib/bolt/result.rb

@@ -203,12 +203,17 @@ module Bolt
     end
 
     def to_data
+      serialized_value = safe_value
+      if serialized_value.key?('_sensitive') &&
+         serialized_value['_sensitive'].is_a?(Puppet::Pops::Types::PSensitiveType::Sensitive)
+        serialized_value['_sensitive'] = serialized_value['_sensitive'].to_s
+      end
       {
         "target" => @target.name,
         "action" => action,
         "object" => object,
         "status" => status,
-        "value" => safe_value
+        "value" => serialized_value
       }
     end
 

data/lib/bolt/shell.rb

@@ -8,6 +8,22 @@ module Bolt
       @target = target
       @conn = conn
       @logger = Bolt::Logger.logger(@target.safe_name)
+
+      if Bolt::Logger.stream
+        Bolt::Logger.warn_once("stream_experimental",
+                               "The 'stream' option is experimental, and might "\
+                               "include breaking changes between minor versions.")
+        @stream_logger = Bolt::Logger.logger(:stream)
+        # Don't send stream messages to the parent logger
+        @stream_logger.additive = false
+
+        # Log stream messages without any other data or color
+        pattern = Logging.layouts.pattern(pattern: '%m\n')
+        @stream_logger.appenders = Logging.appenders.stdout(
+          'console',
+          layout: pattern
+        )
+      end
     end
 
     def run_command(*_args)

data/lib/bolt/shell/bash.rb

@@ -12,7 +12,6 @@ module Bolt
         super
 
         @run_as = nil
-
         @sudo_id = SecureRandom.uuid
         @sudo_password = @target.options['sudo-password'] || @target.password
       end
@@ -54,19 +53,35 @@ module Bolt
 
       def download(source, destination, options = {})
         running_as(options[:run_as]) do
-          # Target OS may be either Unix or Windows. Without knowing the target OS before-hand
-          # we can't assume whether the path separator is '/' or '\'. Assume we're connecting
-          # to a target with Unix and then check if the path exists after downloading.
           download = File.join(destination, Bolt::Util.unix_basename(source))
 
-          conn.download_file(source, destination, download)
+          # If using run-as, the file is copied to a tmpdir and chowned to the
+          # connecting user. This is a workaround for limitations in net-ssh that
+          # only allow for downloading files as the connecting user, which is a
+          # problem for users who cannot connect to targets as the root user.
+          # This temporary copy should *always* be deleted.
+          if run_as
+            with_tmpdir(force_cleanup: true) do |dir|
+              tmpfile = File.join(dir.to_s, Bolt::Util.unix_basename(source))
+
+              result = execute(['cp', '-r', source, dir.to_s], sudoable: true)
+
+              if result.exit_code != 0
+                message = "Could not copy file '#{source}' to temporary directory '#{dir}': #{result.stderr.string}"
+                raise Bolt::Node::FileError.new(message, 'CP_ERROR')
+              end
+
+              # We need to force the chown, otherwise this will just return
+              # without doing anything since the chown user is the same as the
+              # connecting user.
+              dir.chown(conn.user, force: true)
 
-          # If the download path doesn't exist, then the file was likely downloaded from Windows
-          # using a source path with backslashes (e.g. 'C:\Users\Administrator\foo'). The file
-          # should be saved to the expected location, so update the download path assuming a
-          # Windows basename so the result shows the correct local path.
-          unless File.exist?(download)
-            download = File.join(destination, Bolt::Util.windows_basename(source))
+              conn.download_file(tmpfile, destination, download)
+            end
+          # If not using run-as, we can skip creating a temporary copy and just
+          # download the file directly.
+          else
+            conn.download_file(source, destination, download)
           end
 
           Bolt::Result.for_download(target, source, destination, download)
@@ -145,7 +160,10 @@ module Bolt
 
             dir.chown(run_as)
 
-            execute_options[:stdin] = stdin
+            # Don't pass parameters on stdin if using a tty, as the parameters are
+            # already part of the wrapper script.
+            execute_options[:stdin] = stdin unless stdin && target.options['tty']
+
             execute_options[:sudoable] = true if run_as
             output = execute(remote_task_path, **execute_options)
           end
@@ -291,15 +309,15 @@ module Bolt
 
       # A helper to create and delete a tmpdir on the remote system. Yields the
       # directory name.
-      def with_tmpdir
+      def with_tmpdir(force_cleanup: false)
         dir = make_tmpdir
         yield dir
       ensure
         if dir
-          if target.options['cleanup']
+          if target.options['cleanup'] || force_cleanup
             dir.delete
           else
-            @logger.warn("Skipping cleanup of tmpdir #{dir}")
+            Bolt::Logger.warn("skip_cleanup", "Skipping cleanup of tmpdir #{dir}")
           end
         end
       end
@@ -331,10 +349,15 @@ module Bolt
         # together multiple commands into a single sh invocation
         commands = [inject_interpreter(options[:interpreter], command)]
 
+        # Let the transport handle adding environment variables if it's custom.
         if options[:environment]
-          env_decl = options[:environment].map do |env, val|
-            "#{env}=#{Shellwords.shellescape(val)}"
-          end.join(' ')
+          if defined? conn.add_env_vars
+            conn.add_env_vars(options[:environment])
+          else
+            env_decl = options[:environment].map do |env, val|
+              "#{env}=#{Shellwords.shellescape(val)}"
+            end.join(' ')
+          end
         end
 
         if escalate
@@ -385,14 +408,23 @@ module Bolt
           # See if we can read from out or err, or write to in
           ready_read, ready_write, = select(read_streams.keys, write_stream, nil, timeout)
 
-          # Read from out and err
           ready_read&.each do |stream|
+            stream_name = stream == out ? 'out' : 'err'
             # Check for sudo prompt
-            read_streams[stream] << if use_sudo
-                                      check_sudo(stream, inp, options[:stdin])
-                                    else
-                                      stream.readpartial(CHUNK_SIZE)
-                                    end
+            to_print = if use_sudo
+                         check_sudo(stream, inp, options[:stdin])
+                       else
+                         stream.readpartial(CHUNK_SIZE)
+                       end
+
+            if !to_print.chomp.empty? && @stream_logger
+              formatted = to_print.lines.map do |msg|
+                "[#{@target.safe_name}] #{stream_name}: #{msg.chomp}"
+              end.join("\n")
+              @stream_logger.warn(formatted)
+            end
+
+            read_streams[stream] << to_print
           rescue EOFError
           end
 
@@ -440,7 +472,7 @@ module Bolt
         when 0
           @logger.trace { "Command `#{command_str}` returned successfully" }
         when 126
-          msg = "\n\nThis may be caused by the default tmpdir being mounted "\
+          msg = "\n\nThis might be caused by the default tmpdir being mounted "\
             "using 'noexec'. See http://pup.pt/task-failure for details and workarounds."
           result_output.stderr << msg
           @logger.trace { "Command #{command_str} failed with exit code #{result_output.exit_code}" }

data/lib/bolt/shell/bash/tmpdir.rb

@@ -24,8 +24,8 @@ module Bolt
           end
         end
 
-        def chown(owner)
-          return if owner.nil? || owner == @owner
+        def chown(owner, force: false)
+          return if owner.nil? || (owner == @owner && !force)
 
           result = @shell.execute(['id', '-g', owner])
           if result.exit_code != 0
@@ -48,7 +48,10 @@ module Bolt
         def delete
           result = @shell.execute(['rm', '-rf', @path], sudoable: true, run_as: @owner)
           if result.exit_code != 0
-            @logger.warn("Failed to clean up tmpdir '#{@path}': #{result.stderr.string}")
+            Bolt::Logger.warn(
+              "fail_cleanup",
+              "Failed to clean up tmpdir '#{@path}': #{result.stderr.string}"
+            )
           end
           # For testing
           result.stderr.string

data/lib/bolt/shell/powershell.rb

@@ -23,11 +23,10 @@ module Bolt
           # This lets us know how many targets have Powershell 2, and lets the
           # user know how many targets they have with PS2
           msg = "Detected PowerShell 2 on one or more targets.\nPowerShell 2 "\
-            "is deprecated, and support will be removed in Bolt 3.0. See "\
-            "bolt-debug.log or run with '--log-level debug' to see the full "\
+            "is unsupported. See bolt-debug.log or run with '--log-level debug' to see the full "\
             "list of targets with PowerShell 2."
 
-          Bolt::Logger.deprecation_warning("PowerShell 2", msg)
+          Bolt::Logger.deprecate_once("powershell_2", msg)
           @logger.debug("Detected PowerShell 2 on #{target}.")
         end
       end
@@ -163,7 +162,7 @@ module Bolt
           if target.options['cleanup']
             rmdir(@tmpdir)
           else
-            @logger.warn("Skipping cleanup of tmpdir '#{@tmpdir}'")
+            Bolt::Logger.warn("Skipping cleanup of tmpdir '#{@tmpdir}'", "skip_cleanup")
           end
         end
       end
@@ -209,16 +208,21 @@ module Bolt
         arguments = unwrap_sensitive_args(arguments)
         with_tmpdir do |dir|
           script_path = write_executable(dir, script)
-          command = if powershell_file?(script_path)
+          command = if powershell_file?(script_path) && options[:pwsh_params]
+                      # Scripts run with pwsh_params can be run like tasks
+                      Snippets.ps_task(script_path, options[:pwsh_params])
+                    elsif powershell_file?(script_path)
                       Snippets.run_script(arguments, script_path)
                     else
                       path, args = *process_from_extension(script_path)
                       args += escape_arguments(arguments)
                       execute_process(path, args)
                     end
-          command = [*env_declarations(options[:env_vars]), command].join("\r\n") if options[:env_vars]
+          env_assignments = options[:env_vars] ? env_declarations(options[:env_vars]) : []
+          shell_init = options[:pwsh_params] ? Snippets.shell_init : ''
+
+          output = execute([shell_init, *env_assignments, command].join("\r\n"))
 
-          output = execute(command)
           Bolt::Result.for_command(target,
                                    output.stdout.string,
                                    output.stderr.string,
@@ -275,7 +279,12 @@ module Bolt
                               []
                             end
 
-          output = execute([Snippets.shell_init, *env_assignments, command].join("\n"))
+          output = execute([
+            Snippets.shell_init,
+            Snippets.append_ps_module_path(dir),
+            *env_assignments,
+            command
+          ].join("\n"))
 
           Bolt::Result.for_task(target, output.stdout.string,
                                 output.stderr.string,
@@ -306,12 +315,26 @@ module Bolt
           # the proper encoding so the string isn't later misinterpreted
           encoding = out.external_encoding
           out.binmode
-          result.stdout << out.read.force_encoding(encoding)
+          to_print = out.read.force_encoding(encoding)
+          if !to_print.chomp.empty? && @stream_logger
+            formatted = to_print.lines.map do |msg|
+              "[#{@target.safe_name}] out: #{msg.chomp}"
+            end.join("\n")
+            @stream_logger.warn(formatted)
+          end
+          result.stdout << to_print
         end
         stderr = Thread.new do
           encoding = err.external_encoding
           err.binmode
-          result.stderr << err.read.force_encoding(encoding)
+          to_print = err.read.force_encoding(encoding)
+          if !to_print.chomp.empty? && @stream_logger
+            formatted = to_print.lines.map do |msg|
+              "[#{@target.safe_name}] err: #{msg.chomp}"
+            end.join("\n")
+            @stream_logger.warn(formatted)
+          end
+          result.stderr << to_print
         end
 
         stdout.join

data/lib/bolt/shell/powershell/snippets.rb

@@ -55,27 +55,59 @@ module Bolt
             }
             #{build_arg_list}
 
+            switch -regex ( Get-ExecutionPolicy )
+            {
+              '^AllSigned'
+              {
+                if ((Get-AuthenticodeSignature -File "#{script_path}").Status -ne 'Valid') {
+                  $Host.UI.WriteErrorLine("Error: Target host Powershell ExecutionPolicy is set to ${_} and script '#{script_path}' does not contain a valid signature.")
+                  exit 1;
+                }
+              }
+              '^Restricted'
+              {
+                $Host.UI.WriteErrorLine("Error: Target host Powershell ExecutionPolicy is set to ${_} which denies running any scripts on the target.")
+                exit 1;
+              }
+            }
+
+            if([string]::IsNullOrEmpty($invokeArgs.ScriptBlock)){
+              $Host.UI.WriteErrorLine("Error: Failed to obtain scriptblock from '#{script_path}'. Running scripts might be disabled on this system. For more information, see about_Execution_Policies at https:/go.microsoft.com/fwlink/?LinkID=135170");
+              exit 1;
+            }
+
             try
             {
               Invoke-Command @invokeArgs
             }
             catch
             {
-              Write-Error $_.Exception
-              exit 1
+              $Host.UI.WriteErrorLine("[$($_.FullyQualifiedErrorId)] Exception $($_.InvocationInfo.PositionMessage).`n$($_.Exception.Message)");
+              exit 1;
             }
             PS
           end
 
+          def append_ps_module_path(directory)
+            <<~PS
+            $env:PSModulePath += ";#{directory}"
+            PS
+          end
+
           def ps_task(path, arguments)
             <<~PS
             $private:tempArgs = Get-ContentAsJson (
-              $utf8.GetString([System.Convert]::FromBase64String('#{Base64.encode64(JSON.dump(arguments))}'))
+              [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String('#{Base64.encode64(JSON.dump(arguments))}'))
             )
             $allowedArgs = (Get-Command "#{path}").Parameters.Keys
             $private:taskArgs = @{}
             $private:tempArgs.Keys | ? { $allowedArgs -contains $_ } | % { $private:taskArgs[$_] = $private:tempArgs[$_] }
-            try { & "#{path}" @taskArgs } catch { Write-Error $_.Exception; exit 1 }
+            try {
+              & "#{path}" @taskArgs
+            } catch {
+              $Host.UI.WriteErrorLine("[$($_.FullyQualifiedErrorId)] Exception $($_.InvocationInfo.PositionMessage).`n$($_.Exception.Message)");
+              exit 1;
+            }
             PS
           end
 
@@ -102,151 +134,11 @@ module Bolt
             "${boltBaseDir}\\hiera\\lib;" +
             $ENV:RUBYLIB
 
-            Add-Type -AssemblyName System.ServiceModel.Web, System.Runtime.Serialization
-            $utf8 = [System.Text.Encoding]::UTF8
-
-            function Write-Stream {
-            PARAM(
-              [Parameter(Position=0)] $stream,
-              [Parameter(ValueFromPipeline=$true)] $string
-            )
-            PROCESS {
-              $bytes = $utf8.GetBytes($string)
-              $stream.Write( $bytes, 0, $bytes.Length )
-            }
-            }
-
-            function Convert-JsonToXml {
-            PARAM([Parameter(ValueFromPipeline=$true)] [string[]] $json)
-            BEGIN {
-              $mStream = New-Object System.IO.MemoryStream
-            }
-            PROCESS {
-              $json | Write-Stream -Stream $mStream
-            }
-            END {
-              $mStream.Position = 0
-              try {
-                $jsonReader = [System.Runtime.Serialization.Json.JsonReaderWriterFactory]::CreateJsonReader($mStream,[System.Xml.XmlDictionaryReaderQuotas]::Max)
-                $xml = New-Object Xml.XmlDocument
-                $xml.Load($jsonReader)
-                $xml
-              } finally {
-                $jsonReader.Close()
-                $mStream.Dispose()
-              }
-            }
-            }
-
-            Function ConvertFrom-Xml {
-            [CmdletBinding(DefaultParameterSetName="AutoType")]
-            PARAM(
-              [Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=1)] [Xml.XmlNode] $xml,
-              [Parameter(Mandatory=$true,ParameterSetName="ManualType")] [Type] $Type,
-              [Switch] $ForceType
-            )
-            PROCESS{
-              if (Get-Member -InputObject $xml -Name root) {
-                return $xml.root.Objects | ConvertFrom-Xml
-              } elseif (Get-Member -InputObject $xml -Name Objects) {
-                return $xml.Objects | ConvertFrom-Xml
-              }
-              $propbag = @{}
-              foreach ($name in Get-Member -InputObject $xml -MemberType Properties | Where-Object{$_.Name -notmatch "^(__.*|type)$"} | Select-Object -ExpandProperty name) {
-                Write-Debug "$Name Type: $($xml.$Name.type)" -Debug:$false
-                $propbag."$Name" = Convert-Properties $xml."$name"
-              }
-              if (!$Type -and $xml.HasAttribute("__type")) { $Type = $xml.__Type }
-              if ($ForceType -and $Type) {
-                try {
-                  $output = New-Object $Type -Property $propbag
-                } catch {
-                  $output = New-Object PSObject -Property $propbag
-                  $output.PsTypeNames.Insert(0, $xml.__type)
-                }
-              } elseif ($propbag.Count -ne 0) {
-                $output = New-Object PSObject -Property $propbag
-                if ($Type) {
-                  $output.PsTypeNames.Insert(0, $Type)
-                }
-              }
-              return $output
-            }
-            }
-
-            Function Convert-Properties {
-            PARAM($InputObject)
-            switch ($InputObject.type) {
-              "object" {
-                return (ConvertFrom-Xml -Xml $InputObject)
-              }
-              "string" {
-                $MightBeADate = $InputObject.get_InnerText() -as [DateTime]
-                ## Strings that are actually dates (*grumble* JSON is crap)
-                if ($MightBeADate -and $propbag."$Name" -eq $MightBeADate.ToString("G")) {
-                  return $MightBeADate
-                } else {
-                  return $InputObject.get_InnerText()
-                }
-              }
-              "number" {
-                $number = $InputObject.get_InnerText()
-                if ($number -eq ($number -as [int])) {
-                  return $number -as [int]
-                } elseif ($number -eq ($number -as [double])) {
-                  return $number -as [double]
-                } else {
-                  return $number -as [decimal]
-                }
-              }
-              "boolean" {
-                return [bool]::parse($InputObject.get_InnerText())
-              }
-              "null" {
-                return $null
-              }
-              "array" {
-                [object[]]$Items = $(foreach( $item in $InputObject.GetEnumerator() ) {
-                  Convert-Properties $item
-                })
-                return $Items
-              }
-              default {
-                return $InputObject
-              }
-            }
-            }
-
-            Function ConvertFrom-Json2 {
-            [CmdletBinding()]
-            PARAM(
-              [Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=1)] [string] $InputObject,
-              [Parameter(Mandatory=$true)] [Type] $Type,
-              [Switch] $ForceType
-            )
-            PROCESS {
-              $null = $PSBoundParameters.Remove("InputObject")
-              [Xml.XmlElement]$xml = (Convert-JsonToXml $InputObject).Root
-              if ($xml) {
-                if ($xml.Objects) {
-                  $xml.Objects.Item.GetEnumerator() | ConvertFrom-Xml @PSBoundParameters
-                } elseif ($xml.Item -and $xml.Item -isnot [System.Management.Automation.PSParameterizedProperty]) {
-                  $xml.Item | ConvertFrom-Xml @PSBoundParameters
-                } else {
-                  $xml | ConvertFrom-Xml @PSBoundParameters
-                }
-              } else {
-                Write-Error "Failed to parse JSON with JsonReader" -Debug:$false
-              }
-            }
-            }
-
             function ConvertFrom-PSCustomObject
             {
             PARAM([Parameter(ValueFromPipeline = $true)] $InputObject)
             PROCESS {
               if ($null -eq $InputObject) { return $null }
-
               if ($InputObject -is [System.Collections.IEnumerable] -and $InputObject -isnot [string]) {
                 $collection = @(
                   foreach ($object in $InputObject) { ConvertFrom-PSCustomObject $object }
@@ -274,12 +166,7 @@ module Bolt
               [Parameter(Mandatory = $false)] [Text.Encoding] $Encoding = [Text.Encoding]::UTF8
             )
 
-            # using polyfill cmdlet on PS2, so pass type info
-            if ($PSVersionTable.PSVersion -lt [Version]'3.0') {
-              $Text | ConvertFrom-Json2 -Type PSObject | ConvertFrom-PSCustomObject
-            } else {
-              $Text | ConvertFrom-Json | ConvertFrom-PSCustomObject
-            }
+            $Text | ConvertFrom-Json | ConvertFrom-PSCustomObject
             }
             PS
           end