bolt 0.22.0 → 0.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe765b38afea737ae2308098b790fde05b73da5d4eaa661c2121313ccb675d48
-  data.tar.gz: 3280f5388127bce11a71f3aed0db819e7869f0d6751dedb3d0ec815c83623674
+  metadata.gz: c7e3c1e13c0b81acdf948d563eb5acc20710ba52c6aacd14580defbb722fa6ca
+  data.tar.gz: 5a382bffe6ac5eb12e7214a5f0f64581366a4e224f0a1968e5db8a656994f5a1
 SHA512:
-  metadata.gz: 50c22855422b4a2cba254cb43647c8ead003b53816b082173a4383f143cf44e3b6a9ca20ff1792d986597e7256fffe875a7130fe036689860f8931f7c5622b29
-  data.tar.gz: 101c06ea0ad6836d3c773ab877fbc74f972f38c14a33d92f00d0da52e83cd797c58a1b502fc79c0b8edd85df3cccc9bfbbb972ce54d64d09de4a6d7ca1a1f31f
+  metadata.gz: 5793f401349ef2a8985b97c4938cf61497c51abb39fcd65ca2fbc3c4173719366d659eeb302fb145e1e198e415ed62b50728743f0bf98440b9d8d837ce28617c
+  data.tar.gz: c9dc1835008deda159c9e317134dc68c67a24d5cab366cf72d608fd067d854d58ce1084f289aacb7764008a085d26363c82b186b6885ecf2bdab6ddf0eecc464

data/lib/bolt/applicator.rb

@@ -12,10 +12,11 @@ require 'bolt/util/puppet_log_level'
 
 module Bolt
   class Applicator
-    def initialize(inventory, executor, modulepath, pdb_client, hiera_config, max_compiles)
+    def initialize(inventory, executor, modulepath, plugin_dirs, pdb_client, hiera_config, max_compiles)
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath
+      @plugin_dirs = plugin_dirs
       @pdb_client = pdb_client
       @hiera_config = hiera_config ? validate_hiera_config(hiera_config) : nil
 
@@ -222,7 +223,7 @@ module Bolt
       sio = StringIO.new
       output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
 
-      Puppet.lookup(:current_environment).modules.each do |mod|
+      Puppet.lookup(:current_environment).override_with(modulepath: @plugin_dirs).modules.each do |mod|
         search_dirs = yield mod
 
         parent = Pathname.new(mod.path).parent

data/lib/bolt/bolt_option_parser.rb

@@ -202,7 +202,10 @@ Available options are:
       end
       define('--modulepath MODULES',
              "List of directories containing modules, separated by '#{File::PATH_SEPARATOR}'") do |modulepath|
-        @options[:modulepath] = modulepath.split(File::PATH_SEPARATOR)
+        # When specified from the CLI, modulepath entries are relative to pwd
+        @options[:modulepath] = modulepath.split(File::PATH_SEPARATOR).map do |moduledir|
+          File.expand_path(moduledir)
+        end
       end
       define('--boltdir FILEPATH',
              'Specify what Boltdir to load config from (default: autodiscovered from current working dir)') do |path|
@@ -217,7 +220,7 @@ Available options are:
         if ENV.include?(Bolt::Inventory::ENVIRONMENT_VAR)
           raise Bolt::CLIError, "Cannot pass inventory file when #{Bolt::Inventory::ENVIRONMENT_VAR} is set"
         end
-        @options[:inventoryfile] = path
+        @options[:inventoryfile] = File.expand_path(path)
       end
 
       separator 'Transports:'

data/lib/bolt/boltdir.rb

@@ -25,7 +25,7 @@ module Bolt
       @path = Pathname.new(path).expand_path
       @config_file = @path + 'bolt.yaml'
       @inventory_file = @path + 'inventory.yaml'
-      @modulepath = [(@path + 'modules').to_s]
+      @modulepath = [(@path + 'modules').to_s, (@path + 'site').to_s]
       @hiera_config = @path + 'hiera.yaml'
       @puppetfile = @path + 'Puppetfile'
     end

data/lib/bolt/config.rb

@@ -148,11 +148,17 @@ module Bolt
         update_logs(data['log'])
       end
 
-      @modulepath = data['modulepath'].split(File::PATH_SEPARATOR) if data.key?('modulepath')
+      # Expand paths relative to the Boltdir. Any settings that came from the
+      # CLI will already be absolute, so the expand will be skipped.
+      if data.key?('modulepath')
+        @modulepath = data['modulepath'].split(File::PATH_SEPARATOR).map do |moduledir|
+          File.expand_path(moduledir, @boltdir.path)
+        end
+      end
 
-      @inventoryfile = data['inventoryfile'] if data.key?('inventoryfile')
+      @inventoryfile = File.expand_path(data['inventoryfile'], @boltdir.path) if data.key?('inventoryfile')
 
-      @hiera_config = data['hiera-config'] if data.key?('hiera-config')
+      @hiera_config = File.expand_path(data['hiera-config'], @boltdir.path) if data.key?('hiera-config')
       @compile_concurrency = data['compile-concurrency'] if data.key?('compile-concurrency')
 
       %w[concurrency format puppetdb color transport].each do |key|

data/lib/bolt/executor.rb

@@ -40,7 +40,11 @@ module Bolt
 
       @noop = noop
       @run_as = nil
-      @pool = Concurrent::ThreadPoolExecutor.new(max_threads: concurrency)
+      @pool = if concurrency > 0
+                Concurrent::ThreadPoolExecutor.new(max_threads: concurrency)
+              else
+                Concurrent.global_immediate_executor
+              end
       @logger.debug { "Started with #{concurrency} max thread(s)" }
       @notifier = Bolt::Notifier.new
     end

data/lib/bolt/pal.rb

@@ -8,8 +8,8 @@ require 'bolt/applicator'
 
 module Bolt
   class PAL
-    BOLTLIB_PATH = File.join(__dir__, '../../bolt-modules')
-    MODULES_PATH = File.join(__dir__, '../../modules')
+    BOLTLIB_PATH = File.expand_path('../../bolt-modules', __dir__)
+    MODULES_PATH = File.expand_path('../../modules', __dir__)
 
     # PALError is used to convert errors from executing puppet code into
     # Bolt::Errors
@@ -44,9 +44,15 @@ module Bolt
       # This makes sure we don't accidentally create puppet dirs
       with_puppet_settings { |_| nil }
 
+      @original_modulepath = modulepath
       @modulepath = [BOLTLIB_PATH, *modulepath, MODULES_PATH]
       @hiera_config = hiera_config
       @max_compiles = max_compiles
+
+      @logger = Logging.logger[self]
+      if modulepath && !modulepath.empty?
+        @logger.info("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
+      end
     end
 
     # Puppet logging is global so this is class method to avoid confusion
@@ -123,6 +129,10 @@ module Bolt
           inventory,
           executor,
           @modulepath,
+          # Skip syncing built-in plugins, since we vendor some Puppet 6
+          # versions of "core" types, which are already present on the agent,
+          # but may cause issues on Puppet 5 agents.
+          @original_modulepath,
           pdb_client,
           @hiera_config,
           @max_compiles

data/lib/bolt/target.rb

@@ -107,15 +107,11 @@ module Bolt
     end
 
     def user
-      Addressable::URI.unencode_component(
-        @uri_obj.user || @user
-      )
+      Addressable::URI.unencode_component(@uri_obj.user) || @user
     end
 
     def password
-      Addressable::URI.unencode_component(
-        @uri_obj.password || @password
-      )
+      Addressable::URI.unencode_component(@uri_obj.password) || @password
     end
   end
 end

data/lib/bolt/transport/base.rb

@@ -58,10 +58,10 @@ module Bolt
         callback&.call(type: :node_start, target: target)
 
         result = begin
-          yield
-        rescue StandardError => ex
-          Bolt::Result.from_exception(target, ex)
-        end
+                   yield
+                 rescue StandardError => ex
+                   Bolt::Result.from_exception(target, ex)
+                 end
 
         callback&.call(type: :node_result, result: result)
         result

data/lib/bolt/transport/ssh.rb

@@ -174,7 +174,7 @@ module Bolt
       def make_wrapper_stringio(task_path, stdin)
         StringIO.new(<<-SCRIPT)
 #!/bin/sh
-'#{task_path}' <<EOF
+'#{task_path}' <<'EOF'
 #{stdin}
 EOF
 SCRIPT

data/lib/bolt/transport/winrm.rb

@@ -73,7 +73,8 @@ module Bolt
         arguments = unwrap_sensitive_args(arguments)
 
         with_connection(target) do |conn|
-          conn.with_remote_file(script) do |remote_path|
+          conn.with_remote_tempdir do |dir|
+            remote_path = conn.write_remote_executable(dir, script)
             if powershell_file?(remote_path)
               mapped_args = arguments.map do |a|
                 "$invokeArgs.ArgumentList += @'\n#{a}\n'@"
@@ -107,13 +108,9 @@ catch
 
       def run_task(target, task, arguments, _options = {})
         if from_api?(task)
-          # TODO: Remove as part of BOLT-664
-          dir = Dir.mktmpdir
-          executable = File.join(dir, task.file['filename'])
-          File.open(executable, 'w') { |f|
-            f.write(Base64.decode64(task.file['file_content']))
-          }
-          task.input_method = powershell_file?(executable) ? 'powershell' : 'both'
+          task.input_method = powershell_file?(task["file"]["filename"]) ? 'powershell' : 'both'
+          executable = { filename: task["file"]["filename"],
+                         file_content: StringIO.new(Base64.decode64(task.file['file_content'])) }
         else
           executable = target.select_impl(task, PROVIDED_FEATURES)
           raise "No suitable implementation of #{task.name} for #{target.name}" unless executable
@@ -139,9 +136,17 @@ catch
             end
           end
 
-          conn.with_remote_file(executable) do |remote_path|
+          conn.with_remote_tempdir do |dir|
+            remote_task_path = if from_api?(task)
+                                 conn.write_executable_from_content(dir,
+                                                                    executable[:file_content],
+                                                                    executable[:filename])
+                               else
+                                 conn.write_remote_executable(dir, executable)
+                               end
+            conn.shell_init
             output =
-              if powershell_file?(remote_path) && stdin.nil?
+              if powershell_file?(remote_task_path) && stdin.nil?
                 # NOTE: cannot redirect STDIN to a .ps1 script inside of PowerShell
                 # must create new powershell.exe process like other interpreters
                 # fortunately, using PS with stdin input_method should never happen
@@ -150,22 +155,19 @@ catch
 $private:tempArgs = Get-ContentAsJson (
   $utf8.GetString([System.Convert]::FromBase64String('#{Base64.encode64(JSON.dump(arguments))}'))
 )
-$allowedArgs = (Get-Command "#{remote_path}").Parameters.Keys
+$allowedArgs = (Get-Command "#{remote_task_path}").Parameters.Keys
 $private:taskArgs = @{}
 $private:tempArgs.Keys | ? { $allowedArgs -contains $_ } | % { $private:taskArgs[$_] = $private:tempArgs[$_] }
-try { & "#{remote_path}" @taskArgs } catch { Write-Error $_.Exception; exit 1 }
+try { & "#{remote_task_path}" @taskArgs } catch { Write-Error $_.Exception; exit 1 }
               PS
                 else
-                  conn.execute(%(try { & "#{remote_path}" } catch { Write-Error $_.Exception; exit 1 }))
+                  conn.execute(%(try { & "#{remote_task_path}" } catch { Write-Error $_.Exception; exit 1 }))
                 end
               else
-                path, args = *process_from_extension(remote_path)
+                path, args = *process_from_extension(remote_task_path)
                 conn.execute_process(path, args, stdin)
               end
 
-            if from_api?(task)
-              FileUtils.remove_entry dir
-            end
             Bolt::Result.for_task(target, output.stdout.string,
                                   output.stderr.string,
                                   output.exit_code)

data/lib/bolt/transport/winrm/connection.rb

@@ -99,6 +99,7 @@ module Bolt
           return nil if @shell_initialized
           result = execute(<<-PS)
 $ENV:PATH += ";${ENV:ProgramFiles}\\Puppet Labs\\Puppet\\bin\\;" +
+"${ENV:ProgramFiles}\\Puppet Labs\\Puppet\\puppet\\bin;" +
 "${ENV:ProgramFiles}\\Puppet Labs\\Puppet\\sys\\ruby\\bin\\"
 $ENV:RUBYLIB = "${ENV:ProgramFiles}\\Puppet Labs\\Puppet\\puppet\\lib;" +
 "${ENV:ProgramFiles}\\Puppet Labs\\Puppet\\facter\\lib;" +
@@ -335,8 +336,9 @@ PS
 
         def write_remote_file(source, destination)
           fs = ::WinRM::FS::FileManager.new(@connection)
-          # TODO: raise FileError here if this fails
           fs.upload(source, destination)
+        rescue StandardError => e
+          raise Bolt::Node::FileError.new(e.message, 'WRITE_ERROR')
         end
 
         def make_tempdir
@@ -354,25 +356,35 @@ PS
           result.stdout.string.chomp
         end
 
-        def with_remote_file(file)
-          ext = File.extname(file)
+        def with_remote_tempdir
+          dir = make_tempdir
+          yield dir
+        ensure
+          execute(<<-PS)
+Remove-Item -Force -Recurse -Path "#{dir}"
+PS
+        end
+
+        def validate_extensions(ext)
           unless @extensions.include?(ext)
             raise Bolt::Node::FileError.new("File extension #{ext} is not enabled, "\
                                 "to run it please add to 'winrm: extensions'", 'FILETYPE_ERROR')
           end
-          file_base = File.basename(file)
-          dir = make_tempdir
-          dest = "#{dir}\\#{file_base}"
-          begin
-            write_remote_file(file, dest)
-            shell_init
-            yield dest
-          ensure
-            execute(<<-PS)
-Remove-Item -Force "#{dest}"
-Remove-Item -Force "#{dir}"
-PS
-          end
+        end
+
+        def write_remote_executable(dir, file, filename = nil)
+          filename ||= File.basename(file)
+          validate_extensions(File.extname(filename))
+          remote_path = "#{dir}\\#{filename}"
+          write_remote_file(file, remote_path)
+          remote_path
+        end
+
+        def write_executable_from_content(dir, content, filename)
+          validate_extensions(File.extname(filename))
+          remote_path = "#{dir}\\#{filename}"
+          write_remote_file(content, remote_path)
+          remote_path
         end
       end
     end

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '0.22.0'
+  VERSION = '0.23.0'
 end

data/lib/bolt_ext/schemas/ssh-run_task.json

@@ -0,0 +1,71 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "SSH run_task request",
+  "description": "POST ssh/run_task request schema",
+  "type": "object",
+  "properties": {
+    "target": {
+      "type": "object",
+      "description": "Target information to run task on",
+      "properties": {
+        "hostname": {
+          "type": "string",
+          "description": "Target identifier"
+        },
+        "user": {
+          "type": "string",
+          "description": "Login user"
+        },
+        "password": {
+          "type": "string",
+          "description": "Password for SSH transport authentication"
+        },
+        "private-key-content": {
+          "type": "string",
+          "description": "Contents of private key for SSH"
+        },
+        "port": {
+          "type": "integer",
+          "description": "Connection port"
+        },
+        "connect-timeout": {
+          "type": "integer",
+          "description": "How long Bolt should wait when establishing connections"
+        },
+        "run-as-command": {
+          "type": "array",
+          "description": "Command elevate permissions",
+          "items": { "type": "string" }
+        },
+        "run-as": {
+          "type": "string",
+          "description": "A different user to run commands as after login"
+        },
+        "tmpdir": {
+          "type": "string",
+          "description": "The directory to upload and execute temporary files on the target"
+        },
+        "host-key-check": {
+          "type": "boolean",
+          "description": "Whether to perform host key validation when connecting over SSH"
+        },
+        "sudo-password": {
+          "type": "string",
+          "description": "Password to use when changing users via run-as"
+        }
+      },
+      "oneOf": [
+        { "required": ["password"] },
+        { "required": ["private-key-content"] }
+      ],
+      "required": ["hostname", "user"],
+      "additionalProperties": false
+    },
+    "task": { "$ref": "file:task"},
+    "parameters": {
+      "type": "object",
+      "description": "JSON formatted parameters to be provided to task"
+    }  
+  },
+  "required": ["target", "task"]
+}

data/lib/bolt_ext/schemas/task.json

@@ -0,0 +1,57 @@
+{
+  "id": "file:task",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Task",
+  "description": "Task schema for bolt-server",
+  "type": "object",
+  "description": "The task is a JSON object which includes the following keys",
+  "properties": {
+    "name": {
+      "type": "string",
+       "description": "Task name"
+    },
+    "metadata": {
+      "type": "object",
+      "description": "The metadata object is optional, and contains metadata about the task being run",
+      "properties": {
+        "description": {
+          "type": "string",
+          "description": "The task description from it's metadata"
+        },
+        "parameters": {
+          "type": "object",
+          "description": "Object whose keys are parameter names, and values are objects",
+          "properties": {
+            "description": {
+              "type": "string",
+              "description": "Parameter description"
+            },
+            "type": {
+              "type": "string",
+              "description": "The type the parameter should accept"
+            }
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "file": {
+      "type": "object",
+      "description": "File name and content",
+      "properties": {
+        "filename": {
+          "type": "string",
+          "description": "Name of the task file"
+        },
+        "file_content": {
+          "type": "string",
+          "description": "Task's base64 encoded file content"
+        }
+      },
+      "required": ["filename", "file_content"],
+      "additionalProperties": false
+    }
+  },
+  "required": ["name", "file"],
+  "additionalProperties": false
+}

data/lib/bolt_ext/schemas/winrm-run_task.json

@@ -0,0 +1,62 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "WinRm run_task request",
+  "description": "POST winrm/run_task request schema",
+  "type": "object",
+  "properties": {
+    "target": {
+      "type": "object",
+      "description": "Target information to run task on",
+      "properties": {
+        "hostname": {
+          "type": "string",
+          "description": "Target identifier"
+        },
+        "user": {
+          "type": "string",
+          "description": "Login user"
+        },
+        "password": {
+          "type": "string",
+          "description": "Password for SSH transport authentication"
+        },
+        "port": {
+          "type": "integer",
+          "description": "Connection port"
+        },
+        "connect-timeout": {
+          "type": "integer",
+          "description": "How long Bolt should wait when establishing connections"
+        },
+        "tmpdir": {
+          "type": "string",
+          "description": "The directory to upload and execute temporary files on the target"
+        },
+        "ssl": {
+          "type": "boolean",
+          "description": "When true, Bolt will use https connections for WinRM"
+        },
+        "ssl-verify": {
+          "type": "boolean",
+          "description": "When true, verifies the targets certificate matches the cacert"
+        },
+        "cacert": {
+          "type": "string",
+          "description": "The path to the CA certificate"
+        },
+        "extensions": {
+          "type": "array",
+          "description": "List of file extensions that are accepted for scripts or tasks"
+        }
+      },
+      "required": ["hostname", "user", "password"],
+      "additionalProperties": false
+    },
+    "task": { "$ref": "file:task"},
+    "parameters": {
+      "type": "object",
+      "description": "JSON formatted parameters to be provided to task"
+    }  
+  },
+  "required": ["target", "task"]
+}

data/lib/bolt_ext/server.rb

@@ -4,15 +4,41 @@ require 'sinatra'
 require 'bolt'
 require 'bolt/task'
 require 'json'
+require 'json-schema'
 
 class TransportAPI < Sinatra::Base
   # This disables Sinatra's error page generation
   set :show_exceptions, false
 
+  def initialize(app = nil)
+    @schemas = {
+      "ssh-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ssh-run_task.json'))),
+      "winrm-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'winrm-run_task.json')))
+    }
+    shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
+                                     Addressable::URI.parse("file:task"))
+    JSON::Validator.add_schema(shared_schema)
+
+    @executor = Bolt::Executor.new(0, load_config: false)
+
+    super(app)
+  end
+
   get '/' do
     200
   end
 
+  if ENV['RACK_ENV'] == 'dev'
+    get '/admin/gc' do
+      GC.start
+      200
+    end
+  end
+
+  get '/admin/gc_stat' do
+    [200, GC.stat.to_json]
+  end
+
   get '/500_error' do
     raise 'Unexpected error'
   end
@@ -21,13 +47,13 @@ class TransportAPI < Sinatra::Base
     content_type :json
 
     body = JSON.parse(request.body.read)
+    schema_error = JSON::Validator.fully_validate(@schemas["ssh-run_task"], body)
+    return [400, schema_error.join] if schema_error.any?
+
     keys = %w[user password port ssh-key-content connect-timeout run-as-command run-as
               tmpdir host-key-check known-hosts-content private-key-content sudo-password]
     opts = body['target'].select { |k, _| keys.include? k }
 
-    if opts['private-key-content'] && opts['password']
-      return [400, "Only include one of 'password' and 'private-key-content'"]
-    end
     if opts['private-key-content']
       opts['private-key'] = { 'key-data' => opts['private-key-content'] }
       opts.delete('private-key-content')
@@ -37,10 +63,8 @@ class TransportAPI < Sinatra::Base
     task = Bolt::Task.new(body['task'])
     parameters = body['parameters'] || {}
 
-    executor = Bolt::Executor.new(load_config: false)
-
     # Since this will only be on one node we can just return the first result
-    results = executor.run_task(target, task, parameters)
+    results = @executor.run_task(target, task, parameters)
     [200, results.first.to_json]
   end
 
@@ -48,6 +72,9 @@ class TransportAPI < Sinatra::Base
     content_type :json
 
     body = JSON.parse(request.body.read)
+    schema_error = JSON::Validator.fully_validate(@schemas["winrm-run_task"], body)
+    return [400, schema_error.join] if schema_error.any?
+
     keys = %w[user password port connect-timeout ssl ssl-verify tmpdir cacert extensions]
     opts = body['target'].select { |k, _| keys.include? k }
     opts['protocol'] = 'winrm'
@@ -55,10 +82,8 @@ class TransportAPI < Sinatra::Base
     task = Bolt::Task.new(body['task'])
     parameters = body['parameters'] || {}
 
-    executor = Bolt::Executor.new(load_config: false)
-
     # Since this will only be on one node we can just return the first result
-    results = executor.run_task(target, task, parameters)
+    results = @executor.run_task(target, task, parameters)
     [200, results.first.to_json]
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 0.22.0
+  version: 0.23.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-05 00:00:00.000000000 Z
+date: 2018-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: CFPropertyList
   requirement: !ruby/object:Gem::Requirement
@@ -357,6 +357,9 @@ files:
 - lib/bolt/util/puppet_log_level.rb
 - lib/bolt/version.rb
 - lib/bolt_ext/puppetdb_inventory.rb
+- lib/bolt_ext/schemas/ssh-run_task.json
+- lib/bolt_ext/schemas/task.json
+- lib/bolt_ext/schemas/winrm-run_task.json
 - lib/bolt_ext/server.rb
 - lib/bolt_ext/server_acl.rb
 - lib/bolt_ext/server_config.rb