bolt 2.19.0 → 2.24.0

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -65,8 +65,8 @@ module Bolt
           ssh_cmd << command
         end
 
-        def copy_file(source, dest)
-          @logger.debug { "Uploading #{source}, to #{userhost}:#{dest}" } unless source.is_a?(StringIO)
+        def upload_file(source, dest)
+          @logger.trace { "Uploading #{source} to #{dest}" } unless source.is_a?(StringIO)
 
           cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
           cp_cmd = Array(cp_conf)
@@ -87,7 +87,28 @@ module Bolt
                          end
 
           if stat.success?
-            @logger.debug "Successfully uploaded #{source} to #{dest}"
+            @logger.trace "Successfully uploaded #{source} to #{dest}"
+          else
+            message = "Could not copy file to #{dest}: #{err}"
+            raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
+          end
+        end
+
+        def download_file(source, dest, _download)
+          @logger.trace { "Downloading #{userhost}:#{source} to #{dest}" }
+
+          FileUtils.mkdir_p(dest)
+
+          cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
+          cp_cmd = Array(cp_conf)
+          cp_cmd += ssh_opts
+          cp_cmd << "#{userhost}:#{Shellwords.escape(source)}"
+          cp_cmd << dest
+
+          _, err, stat = Open3.capture3(*cp_cmd)
+
+          if stat.success?
+            @logger.trace "Successfully downloaded #{userhost}:#{source} to #{dest}"
           else
             message = "Could not copy file to #{dest}: #{err}"
             raise Bolt::Node::FileError.new(message, 'COPY_ERROR')

data/lib/bolt/transport/winrm/connection.rb

@@ -19,7 +19,7 @@ module Bolt
           # Build set of extensions from extensions config as well as interpreters
 
           @logger = Logging.logger[@target.safe_name]
-          logger.debug("Initializing winrm connection to #{@target.safe_name}")
+          logger.trace("Initializing winrm connection to #{@target.safe_name}")
           @transport_logger = transport_logger
         end
 
@@ -55,7 +55,7 @@ module Bolt
 
             @session = @connection.shell(:powershell)
             @session.run('$PSVersionTable.PSVersion')
-            @logger.debug { "Opened session" }
+            @logger.trace { "Opened session" }
           end
         rescue Timeout::Error
           # If we're using the default port with SSL, a timeout probably means the
@@ -97,11 +97,11 @@ module Bolt
         def disconnect
           @session&.close
           @client&.disconnect!
-          @logger.debug { "Closed session" }
+          @logger.trace { "Closed session" }
         end
 
         def execute(command)
-          @logger.debug { "Executing command: #{command}" }
+          @logger.trace { "Executing command: #{command}" }
 
           inp = StringIO.new
           # This transport doesn't accept stdin, so close the stream to ensure
@@ -111,12 +111,21 @@ module Bolt
           out_rd, out_wr = IO.pipe('UTF-8')
           err_rd, err_wr = IO.pipe('UTF-8')
           th = Thread.new do
+            # By default, any exception raised in a thread will be reported to
+            # stderr as a stacktrace. Since we know these errors are going to
+            # propagate to the main thread via the shell, there's no chance
+            # they will be unhandled, so the default stack trace is unneeded.
+            Thread.current.report_on_exception = false
             result = @session.run(command)
             out_wr << result.stdout
             err_wr << result.stderr
             out_wr.close
             err_wr.close
             result.exitcode
+          ensure
+            # Close the streams to avoid the caller deadlocking
+            out_wr.close
+            err_wr.close
           end
 
           [inp, out_rd, err_rd, th]
@@ -125,27 +134,27 @@ module Bolt
             "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
           raise Bolt::Error.new(msg, 'bolt/too-many-files')
         rescue StandardError
-          @logger.debug { "Command aborted" }
+          @logger.trace { "Command aborted" }
           raise
         end
 
-        def copy_file(source, destination)
-          @logger.debug { "Uploading #{source}, to #{destination}" }
+        def upload_file(source, destination)
+          @logger.trace { "Uploading #{source} to #{destination}" }
           if target.options['file-protocol'] == 'smb'
-            copy_file_smb(source, destination)
+            upload_file_smb(source, destination)
           else
-            copy_file_winrm(source, destination)
+            upload_file_winrm(source, destination)
           end
         end
 
-        def copy_file_winrm(source, destination)
+        def upload_file_winrm(source, destination)
           fs = ::WinRM::FS::FileManager.new(@connection)
           fs.upload(source, destination)
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
-        def copy_file_smb(source, destination)
+        def upload_file_smb(source, destination)
           # lazy-load expensive gem code
           require 'ruby_smb'
 
@@ -165,7 +174,7 @@ module Bolt
           client = smb_client_login
           tree = client.tree_connect(path)
           begin
-            copy_file_smb_recursive(tree, source, dest)
+            upload_file_smb_recursive(tree, source, dest)
           ensure
             tree.disconnect!
           end
@@ -175,6 +184,61 @@ module Bolt
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
+        def download_file(source, destination, download)
+          @logger.trace { "Downloading #{source} to #{destination}" }
+          if target.options['file-protocol'] == 'smb'
+            download_file_smb(source, destination)
+          else
+            download_file_winrm(source, destination, download)
+          end
+        end
+
+        def download_file_winrm(source, destination, download)
+          # The winrm gem doesn't create the destination directory if it's missing,
+          # so create it here
+          FileUtils.mkdir_p(destination)
+          fs = ::WinRM::FS::FileManager.new(@connection)
+          # params: source, destination, chunksize, first
+          # first needs to be set to false, otherwise if the source is a directory it
+          # will be nested inside a directory with the same name
+          fs.download(source, download, 1024 * 1024, false)
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
+        end
+
+        def download_file_smb(source, destination)
+          # lazy-load expensive gem code
+          require 'ruby_smb'
+
+          win_source = source.tr('/', '\\')
+          if (md = win_source.match(/^([a-z]):\\(.*)/i))
+            # if drive, use admin share for that drive, so path is '\\host\C$'
+            path = "\\\\#{@target.host}\\#{md[1]}$"
+            src  = md[2]
+          elsif (md = win_source.match(/^(\\\\[^\\]+\\[^\\]+)\\(.*)/))
+            # if unc, path is '\\host\share'
+            path = md[1]
+            src  = md[2]
+          else
+            raise ArgumentError, "Unknown source '#{source}'"
+          end
+
+          client = smb_client_login
+          tree = client.tree_connect(path)
+
+          begin
+            # Make sure the root download directory for the target exists
+            FileUtils.mkdir_p(destination)
+            download_file_smb_recursive(tree, src, destination)
+          ensure
+            tree.disconnect!
+          end
+        rescue ::RubySMB::Error::UnexpectedStatusCode => e
+          raise Bolt::Node::FileError.new("SMB Error: #{e.message}", 'DOWNLOAD_ERROR')
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'DOWNLOAD_ERROR')
+        end
+
         def shell
           @shell ||= Bolt::Shell::Powershell.new(target, self)
         end
@@ -193,7 +257,7 @@ module Bolt
           status = @client.login
           case status
           when WindowsError::NTStatus::STATUS_SUCCESS
-            @logger.debug { "Connected to #{@client.dns_host_name}" }
+            @logger.trace { "Connected to #{@client.dns_host_name}" }
           when WindowsError::NTStatus::STATUS_LOGON_FAILURE
             raise Bolt::Node::ConnectError.new(
               "SMB authentication failed for #{target.safe_name}",
@@ -230,13 +294,13 @@ module Bolt
           )
         end
 
-        def copy_file_smb_recursive(tree, source, dest)
+        def upload_file_smb_recursive(tree, source, dest)
           if Dir.exist?(source)
             tree.open_directory(directory: dest, write: true, disposition: ::RubySMB::Dispositions::FILE_OPEN_IF)
 
             Dir.children(source).each do |child|
               child_dest = dest + '\\' + child
-              copy_file_smb_recursive(tree, File.join(source, child), child_dest)
+              upload_file_smb_recursive(tree, File.join(source, child), child_dest)
             end
             return
           end
@@ -255,6 +319,52 @@ module Bolt
             file.close
           end
         end
+
+        def download_file_smb_recursive(tree, source, destination)
+          dest = File.expand_path(Bolt::Util.windows_basename(source), destination)
+
+          # Check if the source is a directory by attempting to list its children.
+          # If the source is a directory, create the directory on the host and then
+          # recurse through the children.
+          if (children = list_directory_children_smb(tree, source))
+            FileUtils.mkdir_p(dest)
+
+            children.each do |child|
+              # File names are encoded UTF_16LE.
+              filename = child.file_name.encode(Encoding::UTF_8)
+
+              next if %w[. ..].include?(filename)
+
+              src = source + '\\' + filename
+              download_file_smb_recursive(tree, src, dest)
+            end
+          # If the source wasn't a directory and just returns 'STATUS_NOT_A_DIRECTORY, then
+          # it is a file. Write it to the host.
+          else
+            begin
+              file = tree.open_file(filename: source)
+              data = file.read
+
+              # Files may be encoded UTF_16LE
+              data = data.encode(Encoding::UTF_8) if data.encoding == Encoding::UTF_16LE
+
+              File.write(dest, data)
+            ensure
+              file.close
+            end
+          end
+        end
+
+        # Lists the children of a directory using rb_smb
+        # Returns an array of RubySMB::Fscc::FileInformation::FileIdFullDirectoryInformation objects
+        # if the source is a directory, or raises RubySMB::Error::UnexpectedStatusCode otherwise.
+        def list_directory_children_smb(tree, source)
+          tree.list(directory: source)
+        rescue RubySMB::Error::UnexpectedStatusCode => e
+          unless e.message == 'STATUS_NOT_A_DIRECTORY'
+            raise e
+          end
+        end
       end
     end
   end

data/lib/bolt/util.rb

@@ -13,7 +13,7 @@ module Bolt
           msg = "Invalid content for #{file_name} file: #{path} should be a Hash or empty, not #{content.class}"
           raise Bolt::FileError.new(msg, path)
         end
-        logger.debug("Loaded #{file_name} from #{path}")
+        logger.trace("Loaded #{file_name} from #{path}")
         content
       rescue Errno::ENOENT
         raise Bolt::FileError.new("Could not read #{file_name} file: #{path}", path)
@@ -107,12 +107,13 @@ module Bolt
       # Accepts a Data object and returns a copy with all hash keys
       # modified by block. use &:to_s to stringify keys or &:to_sym to symbolize them
       def walk_keys(data, &block)
-        if data.is_a? Hash
+        case data
+        when Hash
           data.each_with_object({}) do |(k, v), acc|
             v = walk_keys(v, &block)
             acc[yield(k)] = v
           end
-        elsif data.is_a? Array
+        when Array
           data.map { |v| walk_keys(v, &block) }
         else
           data
@@ -124,9 +125,10 @@ module Bolt
       # their descendants are.
       def walk_vals(data, skip_top = false, &block)
         data = yield(data) unless skip_top
-        if data.is_a? Hash
+        case data
+        when Hash
           data.transform_values { |v| walk_vals(v, &block) }
-        elsif data.is_a? Array
+        when Array
           data.map { |v| walk_vals(v, &block) }
         else
           data
@@ -137,9 +139,10 @@ module Bolt
       # modified by the given block. Descendants are modified before their
       # parents.
       def postwalk_vals(data, skip_top = false, &block)
-        new_data = if data.is_a? Hash
+        new_data = case data
+                   when Hash
                      data.transform_values { |v| postwalk_vals(v, &block) }
-                   elsif data.is_a? Array
+                   when Array
                      data.map { |v| postwalk_vals(v, &block) }
                    else
                      data
@@ -193,11 +196,12 @@ module Bolt
           cloned[obj.object_id] = cl
           cloned[cl.object_id] = cl
 
-          if cl.is_a? Hash
+          case cl
+          when Hash
             obj.each { |k, v| cl[k] = deep_clone(v, cloned) }
-          elsif cl.is_a? Array
+          when Array
             cl.collect! { |v| deep_clone(v, cloned) }
-          elsif cl.is_a? Struct
+          when Struct
             obj.each_pair { |k, v| cl[k] = deep_clone(v, cloned) }
           end
 
@@ -257,14 +261,25 @@ module Bolt
 
       # Recursively searches a data structure for plugin references
       def references?(input)
-        if input.is_a?(Hash)
+        case input
+        when Hash
           input.key?('_plugin') || input.values.any? { |v| references?(v) }
-        elsif input.is_a?(Array)
+        when Array
           input.any? { |v| references?(v) }
         else
           false
         end
       end
+
+      def unix_basename(path)
+        raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
+        path.split('/').last
+      end
+
+      def windows_basename(path)
+        raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
+        path.split(%r{[/\\]}).last
+      end
     end
   end
 end

data/lib/bolt/util/puppet_log_level.rb

@@ -4,9 +4,10 @@ module Bolt
   module Util
     module PuppetLogLevel
       MAPPING = {
-        debug: :debug,
-        info: :info,
-        notice: :notice,
+        # Demote Puppet's logs by one level, since Puppet is an implementation detail of Bolt
+        debug: :trace,
+        info: :debug,
+        notice: :info,
         warning: :warn,
         err: :error,
         # The following are used by Puppet functions of the same name, and are all treated as

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '2.19.0'
+  VERSION = '2.24.0'
 end

data/lib/bolt_server/base_config.rb

@@ -16,7 +16,7 @@ module BoltServer
 
     def defaults
       { 'host' => '127.0.0.1',
-        'loglevel' => 'notice',
+        'loglevel' => 'warn',
         'ssl-cipher-suites' => %w[ECDHE-ECDSA-AES256-GCM-SHA384
                                   ECDHE-RSA-AES256-GCM-SHA384
                                   ECDHE-ECDSA-CHACHA20-POLY1305

data/lib/bolt_server/pe/pal.rb

@@ -58,7 +58,7 @@ module BoltServer
           # Bolt::PAL::MODULES_PATH which would be more complex if we tried to use @modulepath since
           # we need to append our modulepaths and exclude modules shiped in bolt gem code
           modulepath_dirs = environment.modulepath
-          @original_modulepath = modulepath_dirs
+          @user_modulepath = modulepath_dirs
           @modulepath = [PE_BOLTLIB_PATH, Bolt::PAL::BOLTLIB_PATH, *modulepath_dirs]
         end
       end

data/lib/bolt_server/transport_app.rb

@@ -151,13 +151,14 @@ module BoltServer
         sha256 = file['sha256']
         kind = file['kind']
         path = File.join(cache_dir, relative_path)
-        if kind == 'file'
+        case kind
+        when 'file'
           # The parent should already be created by `directory` entries,
           # but this is to be on the safe side.
           parent = File.dirname(path)
           FileUtils.mkdir_p(parent)
           @file_cache.serial_execute { @file_cache.download_file(path, sha256, uri) }
-        elsif kind == 'directory'
+        when 'directory'
           # Create directory in cache so we can move files in.
           FileUtils.mkdir_p(path)
         else
@@ -220,6 +221,56 @@ module BoltServer
       plan_info
     end
 
+    def build_puppetserver_uri(file_identifier, module_name, environment)
+      segments = file_identifier.split('/', 3)
+      if segments.size == 1
+        {
+          'path' => "/puppet/v3/file_content/tasks/#{module_name}/#{file_identifier}",
+          'params' => {
+            'environment' => environment
+          }
+        }
+      else
+        module_segment, mount_segment, name_segment = *segments
+        {
+          'path' => case mount_segment
+                    when 'files'
+                      "/puppet/v3/file_content/modules/#{module_segment}/#{name_segment}"
+                    when 'tasks'
+                      "/puppet/v3/file_content/tasks/#{module_segment}/#{name_segment}"
+                    when 'lib'
+                      "/puppet/v3/file_content/plugins/#{name_segment}"
+                    end,
+          'params' => {
+            'environment' => environment
+          }
+        }
+      end
+    end
+
+    def pe_task_info(pal, module_name, task_name, environment)
+      # Handle case where task name is simply module name with special `init` task
+      task_name = if task_name == 'init' || task_name.nil?
+                    module_name
+                  else
+                    "#{module_name}::#{task_name}"
+                  end
+      task = pal.get_task(task_name)
+      files = task.files.map do |file_hash|
+        {
+          'filename' => file_hash['name'],
+          'sha256' => Digest::SHA256.hexdigest(File.read(file_hash['path'])),
+          'size_bytes' => File.size(file_hash['path']),
+          'uri' => build_puppetserver_uri(file_hash['name'], module_name, environment)
+        }
+      end
+      {
+        'metadata' => task.metadata,
+        'name' => task.name,
+        'files' => files
+      }
+    end
+
     get '/' do
       200
     end
@@ -350,6 +401,16 @@ module BoltServer
       end
     end
 
+    # Fetches the metadata for a single task
+    #
+    # @param environment [String] the environment to fetch the task from
+    get '/tasks/:module_name/:task_name' do
+      in_pe_pal_env(params['environment']) do |pal|
+        task_info = pe_task_info(pal, params[:module_name], params[:task_name], params['environment'])
+        [200, task_info.to_json]
+      end
+    end
+
     # Fetches the list of plans for an environment, optionally fetching all metadata for each plan
     #
     # @param environment [String] the environment to fetch the list of plans from
@@ -374,6 +435,22 @@ module BoltServer
       end
     end
 
+    # Fetches the list of tasks for an environment
+    #
+    # @param environment [String] the environment to fetch the list of tasks from
+    get '/tasks' do
+      in_pe_pal_env(params['environment']) do |pal|
+        tasks = pal.list_tasks
+        tasks_response = tasks.map { |task_name, _description| { 'name' => task_name } }.to_json
+
+        # We structure this array of tasks to be an array of hashes so that it matches the structure
+        # returned by the puppetserver API that serves data like this. Structuring the output this way
+        # makes switching between puppetserver and bolt-server easier, which makes changes to switch
+        # to bolt-server smaller/simpler.
+        [200, tasks_response]
+      end
+    end
+
     error 404 do
       err = Bolt::Error.new("Could not find route #{request.path}",
                             'boltserver/not-found')