bolt 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e8d1a8d37a48ddf246cd058bbe4b071732c000c90592da48aac5ed7285143cf
-  data.tar.gz: 9c1aeade908ced86c6d6b6f3e459949c438e0831e4dfc2401daaca6124663077
+  metadata.gz: 5955d8922691cc5c2875f1614d69181289c0ba9561e6016fb8c18bcecd09391a
+  data.tar.gz: 0077fbd497fbbabf23c72a7916517b660a9c21553e8a6bc04ffaddc8cc283e53
 SHA512:
-  metadata.gz: e060cf1af91bfe4f3f6ea97d83e38fd4ed5a65a2fdc1e3d1af28337bb59835fa2fba62cc34d98d7000c37426f5ea76b83e7bd2d540f48c09a40aef8f37cdf202
-  data.tar.gz: d01e9efae28d08b0dffaab1bd6ca6b2679c01b7f390473a09557f61118a957d909f509e0aa480999bc0b8df3dced6b00bea2806ad6a89fb9f1d8bd5ccf0c4e5c
+  metadata.gz: a92a53a9e95ff635ff3adf36e594d6ba38d130cbfd117ecfe7111404cba7f453c63e483269120ecf51d1cd1b58f96cd8bfa6087d88e4ed3c1692c7dfe804ece9
+  data.tar.gz: f2df408dcd301c87b0d598b0f5848576b832a815f2bf13aa5784e85a2384a098b3a34a0b12e3b7b1d80292e334913ab553848826a52e712cb563fc3814abc24d

data/lib/bolt/catalog.rb

@@ -12,6 +12,10 @@ require 'bolt/catalog/logging'
 
 module Bolt
   class Catalog
+    def initialize(log_level = 'debug')
+      @log_level = log_level
+    end
+
     def with_puppet_settings(hiera_config = {})
       Dir.mktmpdir('bolt') do |dir|
         cli = []
@@ -24,7 +28,7 @@ module Bolt
 
         # Use a special logdest that serializes all log messages and their level to stderr.
         Puppet::Util::Log.newdestination(:stderr)
-        Puppet.settings[:log_level] = 'debug'
+        Puppet.settings[:log_level] = @log_level
         yield
       end
     end

data/lib/bolt/cli.rb

@@ -298,7 +298,7 @@ module Bolt
               if dest.nil?
                 raise Bolt::CLIError, "A destination path must be specified"
               end
-              validate_file('source file', src)
+              validate_file('source file', src, true)
               executor.upload_file(targets, src, dest, executor_opts) do |event|
                 outputter.print_event(event)
               end
@@ -392,7 +392,7 @@ module Bolt
       @pal ||= Bolt::PAL.new(config.modulepath, config.hiera_config, config.compile_concurrency)
     end
 
-    def validate_file(type, path)
+    def validate_file(type, path, allow_dir = false)
       if path.nil?
         raise Bolt::CLIError, "A #{type} must be specified"
       end
@@ -401,8 +401,14 @@ module Bolt
 
       if !stat.readable?
         raise Bolt::FileError.new("The #{type} '#{path}' is unreadable", path)
-      elsif !stat.file?
-        raise Bolt::FileError.new("The #{type} '#{path}' is not a file", path)
+      elsif !stat.file? && (!allow_dir || !stat.directory?)
+        expected = allow_dir ? 'file or directory' : 'file'
+        raise Bolt::FileError.new("The #{type} '#{path}' is not a #{expected}", path)
+      elsif stat.directory?
+        Dir.foreach(path) do |file|
+          next if %w[. ..].include?(file)
+          validate_file(type, File.join(path, file), allow_dir)
+        end
       end
     rescue Errno::ENOENT
       raise Bolt::FileError.new("The #{type} '#{path}' does not exist", path)

data/lib/bolt/executor.rb

@@ -79,9 +79,14 @@ module Bolt
               Array(results).each do |result|
                 result_promises[result.target].set(result)
               end
-            # NotImplementedError can be thrown if the transport is implemented improperly
+            # NotImplementedError can be thrown if the transport is not implemented improperly
             rescue StandardError, NotImplementedError => e
               result_promises.each do |target, promise|
+                # If an exception happens while running, the result won't be logged
+                # by the CLI. Log a warning, as this is probably a problem with the transport.
+                # If batch_* commands are used from the Base transport, then exceptions
+                # normally shouldn't reach here.
+                @logger.warn(e)
                 promise.set(Bolt::Result.from_exception(target, e))
               end
             ensure

data/lib/bolt/pal.rb

@@ -179,9 +179,11 @@ module Bolt
     def list_tasks
       in_bolt_compiler do |compiler|
         tasks = compiler.list_tasks
-        tasks.map(&:name).sort.map do |task_name|
+        tasks.map(&:name).sort.each_with_object([]) do |task_name, data|
           task_sig = compiler.task_signature(task_name)
-          [task_name, task_sig.task_hash['metadata']['description']]
+          unless task_sig.task_hash['metadata']['private']
+            data << [task_name, task_sig.task_hash['metadata']['description']]
+          end
         end
       end
     end

data/lib/bolt/task.rb

@@ -58,7 +58,20 @@ module Bolt
       impl['input_method'] = inmethod unless inmethod.nil?
 
       mfiles = impl.fetch('files', []) + metadata.fetch('files', [])
-      impl['files'] = mfiles.map { |file| { 'name' => file, 'path' => file_map[file] } } unless mfiles.empty?
+      dirnames, filenames = mfiles.partition { |file| file.end_with?('/') }
+      impl['files'] = filenames.map do |file|
+        path = file_map[file]
+        raise "No file found for reference #{file}" if path.nil?
+        { 'name' => file, 'path' => path }
+      end
+
+      unless dirnames.empty?
+        files.each do |file|
+          if dirnames.any? { |dirname| file['name'].start_with?(dirname) }
+            impl['files'] << file
+          end
+        end
+      end
 
       impl
     end

data/lib/bolt/transport/base.rb

@@ -60,7 +60,7 @@ module Bolt
 
         result = begin
                    yield
-                 rescue StandardError => ex
+                 rescue StandardError, NotImplementedError => ex
                    Bolt::Result.from_exception(target, ex)
                  end
 

data/lib/bolt/transport/local.rb

@@ -36,7 +36,7 @@ module Bolt
       private :in_tmpdir
 
       def copy_file(source, destination)
-        FileUtils.copy_file(source, destination)
+        FileUtils.cp_r(source, destination, remove_destination: true)
       rescue StandardError => e
         raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
       end
@@ -83,14 +83,25 @@ module Bolt
         implementation = task.select_implementation(target, PROVIDED_FEATURES)
         executable = implementation['path']
         input_method = implementation['input_method'] || 'both'
-
-        # unpack any Sensitive data, write it to a separate variable because
-        # we log 'arguments' below
-        unwrapped_arguments = unwrap_sensitive_args(arguments)
-        stdin = STDIN_METHODS.include?(input_method) ? JSON.dump(unwrapped_arguments) : nil
-        env = ENVIRONMENT_METHODS.include?(input_method) ? envify_params(unwrapped_arguments) : nil
+        extra_files = implementation['files']
 
         with_tmpscript(executable, target.options['tmpdir']) do |script, dir|
+          unless extra_files.empty?
+            installdir = File.join(dir, '_installdir')
+            arguments['_installdir'] = installdir
+            FileUtils.mkdir_p(extra_files.map { |file| File.join(installdir, File.dirname(file['name'])) })
+
+            extra_files.each do |file|
+              copy_file(file['path'], File.join(installdir, file['name']))
+            end
+          end
+
+          # unpack any Sensitive data, write it to a separate variable because
+          # we log 'arguments' below
+          unwrapped_arguments = unwrap_sensitive_args(arguments)
+          stdin = STDIN_METHODS.include?(input_method) ? JSON.dump(unwrapped_arguments) : nil
+          env = ENVIRONMENT_METHODS.include?(input_method) ? envify_params(unwrapped_arguments) : nil
+
           # log the arguments with sensitive data redacted, do NOT log unwrapped_arguments
           logger.debug("Running '#{script}' with #{arguments}")
 

data/lib/bolt/transport/orch.rb

@@ -2,8 +2,12 @@
 
 require 'base64'
 require 'concurrent'
+require 'find'
 require 'json'
+require 'minitar'
 require 'orchestrator_client'
+require 'pathname'
+require 'zlib'
 require 'bolt/transport/base'
 require 'bolt/transport/orch/connection'
 
@@ -114,14 +118,50 @@ module Bolt
         end
       end
 
+      def pack(directory)
+        start_time = Time.now
+        io = StringIO.new
+        output = Minitar::Output.new(Zlib::GzipWriter.new(io))
+        Find.find(directory) do |file|
+          next unless File.file?(file)
+
+          tar_path = Pathname.new(file).relative_path_from(Pathname.new(directory))
+          @logger.debug("Packing #{file} to #{tar_path}")
+          stat = File.stat(file)
+          content = File.binread(file)
+          output.tar.add_file_simple(
+            tar_path.to_s,
+            data: content,
+            size: content.size,
+            mode: stat.mode & 0o777,
+            mtime: stat.mtime
+          )
+        end
+
+        duration = Time.now - start_time
+        @logger.debug("Packed upload in #{duration * 1000} ms")
+
+        output.close
+        io.string
+      ensure
+        # Closes both tar and sgz.
+        output&.close
+      end
+
       def batch_upload(targets, source, destination, options = {}, &callback)
-        content = File.open(source, &:read)
+        stat = File.stat(source)
+        content = if stat.directory?
+                    pack(source)
+                  else
+                    File.open(source, &:read)
+                  end
         content = Base64.encode64(content)
         mode = File.stat(source).mode
         params = {
           'path' => destination,
           'content' => content,
-          'mode' => mode
+          'mode' => mode,
+          'directory' => stat.directory?
         }
         callback ||= proc {}
         results = run_task_job(targets, BOLT_UPLOAD_TASK, params, options, &callback)

data/lib/bolt/transport/ssh.rb

@@ -126,10 +126,12 @@ module Bolt
           executable = task.file['filename']
           file_content = Base64.decode64(task.file['file_content'])
           input_method = task.metadata['input_method']
+          extra_files = []
         else
           implementation = task.select_implementation(target, PROVIDED_FEATURES)
           executable = implementation['path']
           input_method = implementation['input_method']
+          extra_files = implementation['files']
         end
         input_method ||= 'both'
 
@@ -138,18 +140,9 @@ module Bolt
         with_connection(target, options.fetch('_load_config', true)) do |conn|
           conn.running_as(options['_run_as']) do
             stdin, output = nil
-
             command = []
             execute_options = {}
 
-            if STDIN_METHODS.include?(input_method)
-              stdin = JSON.dump(arguments)
-            end
-
-            if ENVIRONMENT_METHODS.include?(input_method)
-              execute_options[:environment] = envify_params(arguments)
-            end
-
             conn.with_remote_tempdir do |dir|
               remote_task_path = if from_api?(task)
                                    conn.write_executable_from_content(dir, file_content, executable)
@@ -157,6 +150,24 @@ module Bolt
                                    conn.write_remote_executable(dir, executable)
                                  end
 
+              unless extra_files.empty?
+                # TODO: optimize upload of directories
+                installdir = File.join(dir.to_s, '_installdir')
+                arguments['_installdir'] = installdir
+                dir.mkdirs(extra_files.map { |file| File.join('_installdir', File.dirname(file['name'])) })
+                extra_files.each do |file|
+                  conn.write_remote_file(file['path'], File.join(installdir, file['name']))
+                end
+              end
+
+              if STDIN_METHODS.include?(input_method)
+                stdin = JSON.dump(arguments)
+              end
+
+              if ENVIRONMENT_METHODS.include?(input_method)
+                execute_options[:environment] = envify_params(arguments)
+              end
+
               if conn.run_as && stdin
                 wrapper = make_wrapper_stringio(remote_task_path, stdin)
                 remote_wrapper_path = conn.write_remote_executable(dir, wrapper, 'wrapper.sh')

data/lib/bolt/transport/ssh/connection.rb

@@ -22,6 +22,15 @@ module Bolt
             @path
           end
 
+          def mkdirs(subdirs)
+            abs_subdirs = subdirs.map { |subdir| File.join(@path, subdir) }
+            result = @node.execute(['mkdir', '-p'] + abs_subdirs)
+            if result.exit_code != 0
+              message = "Could not create subdirectories in '#{@path}': #{result.stderr.string}"
+              raise Bolt::Node::FileError.new(message, 'MKDIR_ERROR')
+            end
+          end
+
           def chown(owner)
             return if owner.nil? || owner == @owner
 
@@ -92,8 +101,16 @@ module Bolt
 
           options[:port] = target.port if target.port
           options[:password] = target.password if target.password
+          # Support both net-ssh 4 and 5. We use 5 in packaging, but Beaker pins to 4 so we
+          # want the gem to be compatible with version 4.
           options[:verify_host_key] = if target.options['host-key-check']
-                                        Net::SSH::Verifiers::Secure.new
+                                        if defined?(Net::SSH::Verifiers::Always)
+                                          Net::SSH::Verifiers::Always.new
+                                        else
+                                          Net::SSH::Verifiers::Secure.new
+                                        end
+                                      elsif defined?(Net::SSH::Verifiers::Never)
+                                        Net::SSH::Verifiers::Never.new
                                       else
                                         Net::SSH::Verifiers::Null.new
                                       end
@@ -281,7 +298,7 @@ module Bolt
         end
 
         def write_remote_file(source, destination)
-          @session.scp.upload!(source, destination)
+          @session.scp.upload!(source, destination, recursive: true)
         rescue StandardError => e
           raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end

data/lib/bolt/transport/winrm.rb

@@ -112,36 +112,49 @@ catch
           executable = task.file['filename']
           file_content = StringIO.new(Base64.decode64(task.file['file_content']))
           input_method = task.metadata['input_method']
+          extra_files = []
         else
           implementation = task.select_implementation(target, PROVIDED_FEATURES)
           executable = implementation['path']
           input_method = implementation['input_method']
+          extra_files = implementation['files']
         end
         input_method ||= powershell_file?(executable) ? 'powershell' : 'both'
 
         # unpack any Sensitive data
         arguments = unwrap_sensitive_args(arguments)
         with_connection(target) do |conn|
-          if STDIN_METHODS.include?(input_method)
-            stdin = JSON.dump(arguments)
-          end
-
-          if ENVIRONMENT_METHODS.include?(input_method)
-            envify_params(arguments).each do |(arg, val)|
-              cmd = "[Environment]::SetEnvironmentVariable('#{arg}', @'\n#{val}\n'@)"
-              result = conn.execute(cmd)
-              if result.exit_code != 0
-                raise Bolt::Node::EnvironmentVarError.new(arg, val)
-              end
-            end
-          end
-
           conn.with_remote_tempdir do |dir|
             remote_task_path = if from_api?(task)
                                  conn.write_executable_from_content(dir, file_content, executable)
                                else
                                  conn.write_remote_executable(dir, executable)
                                end
+
+            unless extra_files.empty?
+              # TODO: optimize upload of directories
+              installdir = File.join(dir, '_installdir')
+              arguments['_installdir'] = installdir
+              conn.mkdirs(extra_files.map { |file| File.join(installdir, File.dirname(file['name'])) })
+              extra_files.each do |file|
+                conn.write_remote_file(file['path'], File.join(installdir, file['name']))
+              end
+            end
+
+            if STDIN_METHODS.include?(input_method)
+              stdin = JSON.dump(arguments)
+            end
+
+            if ENVIRONMENT_METHODS.include?(input_method)
+              envify_params(arguments).each do |(arg, val)|
+                cmd = "[Environment]::SetEnvironmentVariable('#{arg}', @'\n#{val}\n'@)"
+                result = conn.execute(cmd)
+                if result.exit_code != 0
+                  raise Bolt::Node::EnvironmentVarError.new(arg, val)
+                end
+              end
+            end
+
             conn.shell_init
             output =
               if powershell_file?(remote_task_path) && stdin.nil?

data/lib/bolt/transport/winrm/connection.rb

@@ -334,6 +334,14 @@ exit $LASTEXITCODE
 PS
         end
 
+        def mkdirs(dirs)
+          result = execute("mkdir -Force #{dirs.uniq.sort.join(',')}")
+          if result.exit_code != 0
+            message = "Could not create directories: #{result.stderr}"
+            raise Bolt::Node::FileError.new(message, 'MKDIR_ERROR')
+          end
+        end
+
         def write_remote_file(source, destination)
           fs = ::WinRM::FS::FileManager.new(@connection)
           fs.upload(source, destination)

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

data/lib/bolt_ext/schemas/ssh-run_task.json

@@ -45,6 +45,10 @@
           "type": "string",
           "description": "The directory to upload and execute temporary files on the target"
         },
+        "tty": {
+          "type": "boolean",
+          "description": "Should bolt use pseudo tty to meet sudoer restrictions"
+        },
         "host-key-check": {
           "type": "boolean",
           "description": "Whether to perform host key validation when connecting over SSH"
@@ -65,7 +69,7 @@
     "parameters": {
       "type": "object",
       "description": "JSON formatted parameters to be provided to task"
-    }  
+    }
   },
   "required": ["target", "task"]
-}
+}

data/lib/bolt_ext/server.rb

@@ -2,6 +2,7 @@
 
 require 'sinatra'
 require 'bolt'
+require 'bolt/target'
 require 'bolt/task'
 require 'json'
 require 'json-schema'
@@ -50,8 +51,10 @@ class TransportAPI < Sinatra::Base
     schema_error = JSON::Validator.fully_validate(@schemas["ssh-run_task"], body)
     return [400, schema_error.join] if schema_error.any?
 
-    keys = %w[user password port ssh-key-content connect-timeout run-as-command run-as
-              tmpdir host-key-check known-hosts-content private-key-content sudo-password]
+    # CODEREVIEW: the schema is additionalProperties false do we need this?
+    keys = %w[user password port connect-timeout run-as-command run-as
+              tmpdir host-key-check known-hosts-content private-key-content sudo-password
+              tty]
     opts = body['target'].select { |k, _| keys.include? k }
 
     if opts['private-key-content']

data/lib/bolt_ext/server_config.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'hocon'
+require 'bolt/error'
 
 class TransportConfig
   attr_accessor :host, :port, :ssl_cert, :ssl_key, :ssl_ca_cert, :ssl_cipher_suites,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-08 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -84,16 +84,16 @@ dependencies:
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: orchestrator_client
   requirement: !ruby/object:Gem::Requirement