bolt 3.0.1 → 3.6.0

data/lib/bolt/transport/lxd.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'bolt/logger'
+require 'bolt/node/errors'
+require 'bolt/transport/simple'
+
+module Bolt
+  module Transport
+    class LXD < Simple
+      def provided_features
+        ['shell']
+      end
+
+      def with_connection(target, options = {})
+        Bolt::Logger.warn_once("lxd_experimental",
+                               "The LXD transport is experimental, and might "\
+                               "include breaking changes between minor versions.")
+        conn = Connection.new(target, options)
+        conn.connect
+        yield conn
+      end
+    end
+  end
+end
+
+require 'bolt/transport/lxd/connection'

data/lib/bolt/transport/lxd/connection.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'logging'
+require 'bolt/node/errors'
+
+module Bolt
+  module Transport
+    class LXD < Simple
+      class Connection
+        attr_reader :user, :target
+
+        def initialize(target, options)
+          raise Bolt::ValidationError, "Target #{target.safe_name} does not have a host" unless target.host
+
+          @target = target
+          @user = ENV['USER'] || Etc.getlogin
+          @options = options
+          @logger = Bolt::Logger.logger(target.safe_name)
+          @logger.trace("Initializing LXD connection to #{target.safe_name}")
+        end
+
+        def shell
+          Bolt::Shell::Bash.new(target, self)
+        end
+
+        def container_id
+          "#{@target.transport_config['remote']}:#{@target.host}"
+        end
+
+        def connect
+          out, err, status = execute_local_command(%W[list #{container_id} --format json])
+          unless status.exitstatus.zero?
+            raise "Error listing available containers: #{err}"
+          end
+          containers = JSON.parse(out)
+          if containers.empty?
+            raise "Could not find a container with name or ID matching '#{container_id}'"
+          end
+          @logger.trace("Opened session")
+          true
+        rescue StandardError => e
+          raise Bolt::Node::ConnectError.new(
+            "Failed to connect to #{container_id}: #{e.message}",
+            'CONNECT_ERROR'
+          )
+        end
+
+        def add_env_vars(env_vars)
+          @env_vars = env_vars.each_with_object([]) do |env_var, acc|
+            acc << "--env"
+            acc << "#{env_var[0]}=#{Shellwords.shellescape(env_var[1])}"
+          end
+        end
+
+        def execute(command)
+          lxc_command = %w[lxc exec]
+          lxc_command += @env_vars if @env_vars
+          lxc_command += %W[#{container_id} -- sh -c #{Shellwords.shellescape(command)}]
+
+          @logger.trace { "Executing: #{lxc_command.join(' ')}" }
+          Open3.popen3(lxc_command.join(' '))
+        end
+
+        private def execute_local_command(command)
+          Open3.capture3('lxc', *command, { binmode: true })
+        end
+
+        def upload_file(source, destination)
+          @logger.trace { "Uploading #{source} to #{destination}" }
+          args = %w[--create-dirs]
+          if File.directory?(source)
+            args << '--recursive'
+            # If we don't do this, LXD will upload to
+            # /tmp/d2020-11/d2020-11/dir instead of /tmp/d2020-11/dir
+            destination = Pathname.new(destination).dirname.to_s
+          end
+          cmd = %w[file push] + args + %W[#{source} #{container_id}#{destination}]
+          _out, err, stat = execute_local_command(cmd)
+          unless stat.exitstatus.zero?
+            raise "Error writing to #{container_id}: #{err}"
+          end
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
+        def download_file(source, destination, _download)
+          @logger.trace { "Downloading #{source} to #{destination}" }
+          FileUtils.mkdir_p(destination)
+          _out, err, stat = execute_local_command(%W[file pull --recursive #{container_id}#{source} #{destination}])
+          unless stat.exitstatus.zero?
+            raise "Error downloading content from container #{container_id}: #{err}"
+          end
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+      end
+    end
+  end
+end

data/lib/bolt/transport/orch.rb

@@ -59,6 +59,18 @@ module Bolt
           # the result otherwise make sure an error is generated
           if state == 'finished' || (result && result['_error'])
             if result['_error']
+              unless result['_error'].is_a?(Hash)
+                result['_error'] = { 'kind' => 'puppetlabs.tasks/task-error',
+                                     'issue_code' => 'TASK_ERROR',
+                                     'msg' => result['_error'],
+                                     'details' => {} }
+              end
+
+              result['_error']['details'] ||= {}
+              unless result['_error']['details'].is_a?(Hash)
+                deets = result['_error']['details']
+                result['_error']['details'] = { 'msg' => deets }
+              end
               file_line = %w[file line].zip(position).to_h.compact
               result['_error']['details'].merge!(file_line) unless result['_error']['details']['file']
             end
@@ -252,11 +264,7 @@ module Bolt
 
         # If we get here, there's no error so we don't need the file or line
         # number
-        Bolt::Result.for_command(target,
-                                 result.value['stdout'],
-                                 result.value['stderr'],
-                                 result.value['exit_code'],
-                                 action, obj, [])
+        Bolt::Result.for_command(target, result.value, action, obj, [])
       end
     end
   end

data/lib/bolt/transport/podman.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'shellwords'
+require 'bolt/transport/base'
+
+module Bolt
+  module Transport
+    class Podman < Docker
+      def with_connection(target)
+        conn = Connection.new(target)
+        conn.connect
+        yield conn
+      end
+    end
+  end
+end
+
+require 'bolt/transport/podman/connection'

data/lib/bolt/transport/podman/connection.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'logging'
+require 'bolt/node/errors'
+
+module Bolt
+  module Transport
+    class Podman < Docker
+      class Connection < Connection
+        attr_reader :user, :target
+
+        def initialize(target)
+          raise Bolt::ValidationError, "Target #{target.safe_name} does not have a host" unless target.host
+          @target = target
+          @user = ENV['USER'] || Etc.getlogin
+          @logger = Bolt::Logger.logger(target.safe_name)
+          @container_info = {}
+          @logger.trace("Initializing podman connection to #{target.safe_name}")
+        end
+
+        def run_cmd(cmd, env_vars)
+          Bolt::Util.exec_podman(cmd, env_vars)
+        end
+
+        def shell
+          @shell ||= if Bolt::Util.windows?
+                       Bolt::Shell::Powershell.new(target, self)
+                     else
+                       Bolt::Shell::Bash.new(target, self)
+                     end
+        end
+
+        def connect
+          # We don't actually have a connection, but we do need to
+          # check that the container exists and is running.
+          ps = execute_local_json_command('ps')
+          container = Array(ps).find { |item|
+            item["ID"].to_s.eql?(@target.host) ||
+              item["Id"].to_s.start_with?(@target.host) ||
+              Array(item["Names"]).include?(@target.host)
+          }
+          raise "Could not find a container with name or ID matching '#{@target.host}'" if container.nil?
+          # Now find the indepth container information
+          id = container["ID"] || container["Id"]
+          output = execute_local_json_command('inspect', [id])
+          # Store the container information for later
+          @container_info = output.first
+          @logger.trace { "Opened session" }
+          true
+        rescue StandardError => e
+          raise Bolt::Node::ConnectError.new(
+            "Failed to connect to #{target.safe_name}: #{e.message}",
+            'CONNECT_ERROR'
+          )
+        end
+
+        # Executes a command inside the target container. This is called from the shell class.
+        #
+        # @param command [string] The command to run
+        def execute(command)
+          args = []
+          args += %w[--interactive]
+          args += %w[--tty] if target.options['tty']
+          args += @env_vars if @env_vars
+
+          if target.options['shell-command'] && !target.options['shell-command'].empty?
+            # escape any double quotes in command
+            command = command.gsub('"', '\"')
+            command = "#{target.options['shell-command']} \"#{command}\""
+          end
+
+          podman_command = %w[podman exec] + args + [container_id] + Shellwords.split(command)
+          @logger.trace { "Executing: #{podman_command.join(' ')}" }
+
+          Open3.popen3(*podman_command)
+        rescue StandardError
+          @logger.trace { "Command aborted" }
+          raise
+        end
+
+        # Converts the JSON encoded STDOUT string from the podman cli into ruby objects
+        #
+        # @param stdout [String] The string to convert
+        # @return [Object] Ruby object representation of the JSON string
+        private def extract_json(stdout)
+          # Podman renders the output in pretty JSON, which results in a newline
+          # appearing in the output before the closing bracket.
+          # should we only get a single line with no newline at all, we also
+          # assume it is a single minified JSON object
+          stdout.strip!
+          newline = stdout.index("\n") || -1
+          bracket = stdout.index('}') || -1
+          JSON.parse(stdout) if bracket > newline
+        end
+      end
+    end
+  end
+end

data/lib/bolt/transport/ssh/connection.rb

@@ -230,7 +230,7 @@ module Bolt
           end
           [in_wr, out_rd, err_rd, th]
         rescue Errno::EMFILE => e
-          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+          msg = "#{e.message}. This might be resolved by increasing your user limit "\
             "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
           raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end

data/lib/bolt/transport/winrm/connection.rb

@@ -130,7 +130,7 @@ module Bolt
 
           [inp, out_rd, err_rd, th]
         rescue Errno::EMFILE => e
-          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+          msg = "#{e.message}. This might be resolved by increasing your user limit "\
             "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
           raise Bolt::Error.new(msg, 'bolt/too-many-files')
         rescue StandardError

data/lib/bolt/util.rb

@@ -77,6 +77,14 @@ module Bolt
         File.exist?(path) ? read_yaml_hash(path, file_name) : {}
       end
 
+      def first_runs_free
+        Bolt::Config.user_path + '.first_runs_free'
+      end
+
+      def first_run?
+        Bolt::Config.user_path && !File.exist?(first_runs_free)
+      end
+
       # Accepts a path with either 'plans' or 'tasks' in it and determines
       # the name of the module
       def module_name(path)
@@ -324,6 +332,40 @@ module Bolt
         end
       end
 
+      # Executes a Docker CLI command. This is useful for running commands as
+      # part of this class without having to go through the `execute`
+      # function and manage pipes.
+      #
+      # @param cmd [String] The docker command and arguments to run
+      #   e.g. 'cp <src> <dest>' for `docker cp <src> <dest>`
+      # @return [String, String, Process::Status] The output of the command: STDOUT, STDERR, Process Status
+      def exec_docker(cmd, env = {})
+        Open3.capture3(env, 'docker', *cmd, { binmode: true })
+      end
+
+      # Executes a Podman CLI command. This is useful for running commands as
+      # part of this class without having to go through the `execute`
+      # function and manage pipes.
+      #
+      # @param cmd [String] The podman command and arguments to run
+      #   e.g. 'cp <src> <dest>' for `podman cp <src> <dest>`
+      # @return [String, String, Process::Status] The output of the command: STDOUT, STDERR, Process Status
+      def exec_podman(cmd, env = {})
+        Open3.capture3(env, 'podman', *cmd, { binmode: true })
+      end
+
+      # Formats a map of environment variables to be passed to a command that
+      # accepts repeated `--env` flags
+      #
+      # @param env_vars [Hash] A map of environment variables keys and their values
+      # @return [String]
+      def format_env_vars_for_cli(env_vars)
+        @env_vars = env_vars.each_with_object([]) do |(key, value), acc|
+          acc << "--env"
+          acc << "#{key}=#{value}"
+        end
+      end
+
       def unix_basename(path)
         raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
         path.split('/').last

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.0.1'
+  VERSION = '3.6.0'
 end

data/lib/bolt_server/transport_app.rb

@@ -133,7 +133,14 @@ module BoltServer
       task_data = body['task']
       task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
       parameters = body['parameters'] || {}
-      [@executor.run_task(target, task, parameters), nil]
+      task_result = @executor.run_task(target, task, parameters)
+      task_result.each do |result|
+        value = result.value
+        next unless value.is_a?(Hash)
+        next unless value.key?('_sensitive')
+        value['_sensitive'] = value['_sensitive'].unwrap
+      end
+      [task_result, nil]
     end
 
     def run_command(target, body)
@@ -275,15 +282,19 @@ module BoltServer
       Bolt::Config.from_project(project, { log: { 'bolt-debug.log' => 'disable' } })
     end
 
+    def pal_from_project_bolt_config(bolt_config)
+      modulepath_object = Bolt::Config::Modulepath.new(
+        bolt_config.modulepath,
+        boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH],
+        builtin_content_path: @config['builtin-content-dir']
+      )
+      Bolt::PAL.new(modulepath_object, nil, nil, nil, nil, nil, bolt_config.project)
+    end
+
     def in_bolt_project(versioned_project)
       @pal_mutex.synchronize do
         bolt_config = config_from_project(versioned_project)
-        modulepath_object = Bolt::Config::Modulepath.new(
-          bolt_config.modulepath,
-          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH],
-          builtin_content_path: @config['builtin-content-dir']
-        )
-        pal = Bolt::PAL.new(modulepath_object, nil, nil, nil, nil, nil, bolt_config.project)
+        pal = pal_from_project_bolt_config(bolt_config)
         context = {
           pal: pal,
           config: bolt_config
@@ -351,8 +362,8 @@ module BoltServer
       }
     end
 
-    def allowed_helper(metadata, allowlist)
-      allowed = allowlist.nil? || allowlist.include?(metadata['name'])
+    def allowed_helper(pal, metadata, allowlist)
+      allowed = !pal.filter_content([metadata['name']], allowlist).empty?
       metadata.merge({ 'allowed' => allowed })
     end
 
@@ -366,21 +377,27 @@ module BoltServer
       plans.map { |plan_name| { 'name' => plan_name } }
     end
 
-    def file_metadatas(pal, module_name, file)
-      pal.in_bolt_compiler do
-        mod = Puppet.lookup(:current_environment).module(module_name)
-        raise ArgumentError, "`module_name`: #{module_name} does not exist" unless mod
-        abs_file_path = mod.file(file)
-        raise ArgumentError, "`file`: #{file} does not exist inside the module's 'files' directory" unless abs_file_path
-        fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
-        Puppet::FileServing::Fileset.merge(fileset).collect do |relative_file_path, base_path|
-          metadata = Puppet::FileServing::Metadata.new(base_path, relative_path: relative_file_path)
-          metadata.checksum_type = 'sha256'
-          metadata.links = 'follow'
-          metadata.collect
-          metadata.to_data_hash
+    def file_metadatas(versioned_project, module_name, file)
+      abs_file_path = @pal_mutex.synchronize do
+        bolt_config = config_from_project(versioned_project)
+        pal = pal_from_project_bolt_config(bolt_config)
+        pal.in_bolt_compiler do
+          mod = Puppet.lookup(:current_environment).module(module_name)
+          raise ArgumentError, "`module_name`: #{module_name} does not exist" unless mod
+          mod.file(file)
         end
       end
+
+      raise ArgumentError, "`file`: #{file} does not exist inside the module's 'files' directory" unless abs_file_path
+
+      fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
+      Puppet::FileServing::Fileset.merge(fileset).collect do |relative_file_path, base_path|
+        metadata = Puppet::FileServing::Metadata.new(base_path, relative_path: relative_file_path)
+        metadata.checksum_type = 'sha256'
+        metadata.links = 'follow'
+        metadata.collect
+        metadata.to_data_hash
+      end
     end
 
     get '/' do
@@ -520,7 +537,7 @@ module BoltServer
       return MISSING_VERSIONED_PROJECT_RESPONSE if params['versioned_project'].nil?
       in_bolt_project(params['versioned_project']) do |context|
         plan_info = pe_plan_info(context[:pal], params[:module_name], params[:plan_name])
-        plan_info = allowed_helper(plan_info, context[:config].project.plans)
+        plan_info = allowed_helper(context[:pal], plan_info, context[:config].project.plans)
         [200, plan_info.to_json]
       end
     rescue Bolt::Error => e
@@ -550,7 +567,7 @@ module BoltServer
           'versioned_project' => params['versioned_project']
         }
         task_info = pe_task_info(context[:pal], params[:module_name], params[:task_name], ps_parameters)
-        task_info = allowed_helper(task_info, context[:config].project.tasks)
+        task_info = allowed_helper(context[:pal], task_info, context[:config].project.tasks)
         [200, task_info.to_json]
       end
     rescue Bolt::Error => e
@@ -590,7 +607,7 @@ module BoltServer
         plans_response = plan_list(context[:pal])
 
         # Dig in context for the allowlist of plans from project object
-        plans_response.map! { |metadata| allowed_helper(metadata, context[:config].project.plans) }
+        plans_response.map! { |metadata| allowed_helper(context[:pal], metadata, context[:config].project.plans) }
 
         # We structure this array of plans to be an array of hashes so that it matches the structure
         # returned by the puppetserver API that serves data like this. Structuring the output this way
@@ -626,7 +643,7 @@ module BoltServer
         tasks_response = task_list(context[:pal])
 
         # Dig in context for the allowlist of tasks from project object
-        tasks_response.map! { |metadata| allowed_helper(metadata, context[:config].project.tasks) }
+        tasks_response.map! { |metadata| allowed_helper(context[:pal], metadata, context[:config].project.tasks) }
 
         # We structure this array of tasks to be an array of hashes so that it matches the structure
         # returned by the puppetserver API that serves data like this. Structuring the output this way
@@ -642,12 +659,11 @@ module BoltServer
     #
     # @param versioned_project [String] the versioned_project to fetch the file metadatas from
     get '/project_file_metadatas/:module_name/*' do
-      return MISSING_VERSIONED_PROJECT_RESPONSE if params['versioned_project'].nil?
-      in_bolt_project(params['versioned_project']) do |context|
-        file = params[:splat].first
-        metadatas = file_metadatas(context[:pal], params[:module_name], file)
-        [200, metadatas.to_json]
-      end
+      versioned_project = params['versioned_project']
+      return MISSING_VERSIONED_PROJECT_RESPONSE if versioned_project.nil?
+      file = params[:splat].first
+      metadatas = file_metadatas(versioned_project, params[:module_name], file)
+      [200, metadatas.to_json]
     rescue Bolt::Error => e
       [400, e.to_json]
     rescue ArgumentError => e
@@ -674,11 +690,26 @@ module BoltServer
         Bolt::Util.validate_file('inventory file', context[:config].project.inventory_file)
 
         begin
+          # Set the default puppet_library plugin hook if it has not already been
+          # set
+          context[:config].data['plugin-hooks']['puppet_library'] ||= {
+            'plugin'     => 'task',
+            'task'       => 'puppet_agent::install',
+            'parameters' => {
+              'stop_service' => true
+            }
+          }
+
           connect_plugin = BoltServer::Plugin::PuppetConnectData.new(body['puppet_connect_data'])
           plugins = Bolt::Plugin.setup(context[:config], context[:pal], load_plugins: false)
           plugins.add_plugin(connect_plugin)
+          %w[aws_inventory azure_inventory gcloud_inventory].each do |plugin_name|
+            plugins.add_module_plugin(plugin_name) if plugins.known_plugin?(plugin_name)
+          end
           inventory = Bolt::Inventory.from_config(context[:config], plugins)
-          target_list = inventory.get_targets('all').map { |targ| targ.to_h.merge({ 'transport' => targ.transport }) }
+          target_list = inventory.get_targets('all').map do |targ|
+            targ.to_h.merge({ 'transport' => targ.transport, 'plugin_hooks' => targ.plugin_hooks })
+          end
         rescue Bolt::Plugin::PluginError::LoadingDisabled => e
           msg = "Cannot load plugin #{e.details['plugin_name']}: plugin not supported"
           raise BoltServer::Plugin::PluginNotSupported.new(msg, e.details['plugin_name'])