bolt 3.13.0 → 3.16.1

data/lib/bolt/plugin/task.rb

@@ -59,7 +59,7 @@ module Bolt
         run_opts = {}
         run_opts[:run_as] = opts['_run_as'] if opts['_run_as']
         begin
-          task = apply_prep.get_task(opts['task'], params)
+          task = @context.get_validated_task(opts['task'], params)
         rescue Bolt::Error => e
           raise Bolt::Plugin::PluginError::ExecutionError.new(e.message, name, 'puppet_library')
         end

data/lib/bolt/project.rb

@@ -209,12 +209,5 @@ module Bolt
         Bolt::Logger.warn("missing_project_name", message)
       end
     end
-
-    def check_deprecated_file
-      if (@path + 'project.yaml').file?
-        msg = "Project configuration file 'project.yaml' is deprecated; use 'bolt-project.yaml' instead."
-        Bolt::Logger.warn("project_yaml", msg)
-      end
-    end
   end
 end

data/lib/bolt/result_set.rb

@@ -2,7 +2,8 @@
 
 module Bolt
   class ResultSet
-    attr_reader :results
+    attr_accessor :elapsed_time
+    attr_reader   :results
 
     include Enumerable
 

data/lib/bolt/transport/local/connection.rb

@@ -10,6 +10,8 @@ module Bolt
   module Transport
     class Local < Simple
       class Connection
+        RUBY_ENV_VARS = %w[GEM_PATH GEM_HOME RUBYLIB RUBYLIB_PREFIX RUBYOPT RUBYPATH RUBYSHELL].freeze
+
         attr_accessor :user, :logger, :target
 
         def initialize(target)
@@ -68,7 +70,21 @@ module Bolt
             end
           end
 
-          Open3.popen3(*command)
+          # Only do this if bundled-ruby is set to false, not nil
+          ruby_env_vars = if target.transport_config['bundled-ruby'] == false
+                            RUBY_ENV_VARS.each_with_object({}) do |e, acc|
+                              acc[e] = ENV["BOLT_ORIG_#{e}"] if ENV["BOLT_ORIG_#{e}"]
+                            end
+                          end
+
+          if target.transport_config['bundled-ruby'] == false &&
+             Gem.loaded_specs.keys.include?('bundler')
+            Bundler.with_unbundled_env do
+              Open3.popen3(ruby_env_vars || {}, *command)
+            end
+          else
+            Open3.popen3(ruby_env_vars || {}, *command)
+          end
         end
 
         # This is used by the Bash shell to decide whether to `cd` before

data/lib/bolt/transport/orch/connection.rb

@@ -36,7 +36,8 @@ module Bolt
           end
           logger.debug("Creating orchestrator client for #{client_opts}")
           @client = OrchestratorClient.new(client_opts, true)
-          @plan_job = start_plan(plan_context)
+          @plan_context = plan_context
+          @plan_job = start_plan(@plan_context)
           logger.debug("Started plan #{@plan_job}")
           @environment = opts["task-environment"]
         end
@@ -87,6 +88,17 @@ module Bolt
         def run_task(targets, task, arguments, options)
           body = build_request(targets, task, arguments, options[:description])
           @client.run_task(body)
+        rescue OrchestratorClient::ApiError => e
+          if e.data['kind'] == 'puppetlabs.orchestrator/plan-already-finished'
+            @logger.debug("Retrying the task")
+            # Instead of recursing, just retry once
+            @plan_job = start_plan(@plan_context)
+            # Rebuild the request with the new plan job ID
+            body = build_request(targets, task, arguments, options[:description])
+            @client.run_task(body)
+          else
+            raise e
+          end
         end
 
         def query_inventory(targets)

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -47,7 +47,9 @@ module Bolt
           cmd = []
           # BatchMode is SSH's noninteractive option: if key authentication
           # fails it will error out instead of falling back to password prompt
-          cmd += %w[-o BatchMode=yes]
+          batch_mode = @target.transport_config['batch-mode'] ? 'yes' : 'no'
+          cmd += %W[-o BatchMode=#{batch_mode}]
+
           cmd += %W[-o Port=#{@target.port}] if @target.port
 
           if @target.transport_config.key?('host-key-check')

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.13.0'
+  VERSION = '3.16.1'
 end

data/lib/bolt_server/file_cache.rb

@@ -182,5 +182,17 @@ module BoltServer
         end
       end
     end
+
+    def get_cached_project_file(versioned_project, file_name)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.read(file_path) if File.exist?(file_path) }
+    end
+
+    def cache_project_file(versioned_project, file_name, data)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
+    end
   end
 end

data/lib/bolt_server/schemas/action-apply.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply request",
+  "description": "POST <transport>/apply request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "string",
+      "description": "Project from which to load code"
+    },
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "catalog": {
+          "type": "object",
+          "description": "Compiled catalog to apply"
+        },
+        "apply_options": {
+          "type": "object",
+          "description": "Options for application of a catalog"
+        }
+      }
+    },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    },
+    "target": { "$ref": "partial:target-any" }
+  },
+  "required": ["target", "versioned_project", "parameters", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/schemas/action-apply_prep.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply_prep request",
+  "description": "POST <transport>/apply_prep request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "String",
+      "description": "Project from which to load code"
+    },
+    "target": { "$ref": "partial:target-any" },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    }
+  },
+  "required": ["target", "versioned_project", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/schemas/partials/target-ssh.json

@@ -62,6 +62,10 @@
     "interpreters": {
       "type": "object",
       "description": "Map of file extensions to remote executable"
+    },
+    "plugin_hooks": {
+      "type": "object",
+      "description": "Configuration for plugins to use"
     }
   },
   "oneOf": [

data/lib/bolt_server/schemas/partials/target-winrm.json

@@ -56,6 +56,10 @@
     "smb-port": {
       "type": "integer",
       "description": "Port for SMB protocol"
+    },
+    "plugin_hooks": {
+      "type": "object",
+      "description": "Configuration for plugins to use"
     }
   },
   "required": ["hostname", "user", "password"],

data/lib/bolt_server/transport_app.rb

@@ -43,6 +43,8 @@ module BoltServer
       transport-ssh
       transport-winrm
       connect-data
+      action-apply_prep
+      action-apply
     ].freeze
 
     # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
@@ -75,6 +77,12 @@ module BoltServer
       # This is needed until the PAL is threadsafe.
       @pal_mutex = Mutex.new
 
+      # Avoid redundant plugin tarbal construction
+      @plugin_mutex = Mutex.new
+
+      # Avoid redundant project_task metadata construction
+      @task_metadata_mutex = Mutex.new
+
       @logger = Bolt::Logger.logger(self)
 
       super(nil)
@@ -118,12 +126,7 @@ module BoltServer
       end
     end
 
-    def run_task(target, body)
-      validate_schema(@schemas["action-run_task"], body)
-
-      task_data = body['task']
-      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
-      parameters = body['parameters'] || {}
+    def task_helper(target, task, parameters)
       # Wrap parameters marked with '"sensitive": true' in the task metadata with a
       # Sensitive wrapper type. This way it's not shown in logs.
       if (param_spec = task.parameters)
@@ -142,6 +145,101 @@ module BoltServer
       end
     end
 
+    def run_task(target, body)
+      validate_schema(@schemas["action-run_task"], body)
+
+      task_data = body['task']
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'] || {})
+    end
+
+    def extract_install_task(target)
+      unless target.plugin_hooks['puppet_library']['task']
+        raise BoltServer::RequestError,
+              "Target must have 'task' plugin hook"
+      end
+      install_task = target.plugin_hooks['puppet_library']['task'].split('::', 2)
+      install_task << 'init' if install_task.count == 1
+      install_task
+    end
+
+    # This helper is responsible for computing or retrieving from the cache a plugin tarball. There are
+    # two supported plugin types 'fact_plugins', and 'all_plugins'. Note that this is cached based on
+    # versioned_project as there are no plugins in the "builtin content" directory
+    def plugin_tarball(versioned_project, tarball_type)
+      tarball_types = %w[fact_plugins all_plugins]
+      unless tarball_types.include?(tarball_type)
+        raise ArgumentError,
+              "tarball_type must be one of: #{tarball_types.join(', ')}"
+      end
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint
+      # the tarball computation only happens once (all the other targets will just need to read the cached data)
+      @plugin_mutex.synchronize do
+        if (tarball = @file_cache.get_cached_project_file(versioned_project, tarball_type))
+          tarball
+        else
+          new_tarball = build_project_plugins_tarball(versioned_project) do |mod|
+            search_dirs = []
+            search_dirs << mod.plugins if mod.plugins?
+            search_dirs << mod.pluginfacts if mod.pluginfacts?
+            if tarball_type == 'all_plugins'
+              search_dirs << mod.files if mod.files?
+              type_files = "#{mod.path}/types"
+              search_dirs << type_files if File.exist?(type_files)
+            end
+            search_dirs
+          end
+          @file_cache.cache_project_file(versioned_project, tarball_type, new_tarball)
+          new_tarball
+        end
+      end
+    end
+
+    # This helper is responsible for computing or retrieving task metadata for a project.
+    # It expects task name in segments and uses the combination of task name and versioned_project
+    # as a unique identifier for caching in addition to the job_id. The job id is added to protect against
+    # a case where the buildtin content is update (where the apply_helpers would be managed)
+    def project_task_metadata(versioned_project, task_name_segments, job_id)
+      cached_file_name = "#{task_name_segments.join('_')}_#{job_id}"
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint the
+      # metadata computation will only be computed once, then the cache will be read.
+      @task_metadata_mutex.synchronize do
+        if (metadata = @file_cache.get_cached_project_file(versioned_project, cached_file_name))
+          JSON.parse(metadata)
+        else
+          new_metadata = in_bolt_project(versioned_project) do |context|
+            ps_parameters = {
+              'versioned_project' => versioned_project
+            }
+            pe_task_info(context[:pal], *task_name_segments, ps_parameters)
+          end
+          @file_cache.cache_project_file(versioned_project, cached_file_name, new_metadata.to_json)
+          new_metadata
+        end
+      end
+    end
+
+    def apply_prep(target, body)
+      validate_schema(@schemas["action-apply_prep"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'fact_plugins')
+      install_task_segments = extract_install_task(target.first)
+      task_data = project_task_metadata(body['versioned_project'], install_task_segments, body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      install_task_result = task_helper(target, task, target.first.plugin_hooks['puppet_library']['parameters'] || {})
+      return install_task_result unless install_task_result.ok
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers custom_facts], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, { 'plugins' => plugins_tarball })
+    end
+
+    def apply(target, body)
+      validate_schema(@schemas["action-apply"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'all_plugins')
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers apply_catalog], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'].merge({ 'plugins' => plugins_tarball }))
+    end
+
     def run_command(target, body)
       validate_schema(@schemas["action-run_command"], body)
       command = body['command']
@@ -389,6 +487,63 @@ module BoltServer
       end
     end
 
+    # The provided block takes a module object and returns the list
+    # of directories to search through. This is similar to
+    # Bolt::Applicator.build_plugin_tarball.
+    def build_project_plugins_tarball(versioned_project, &block)
+      start_time = Time.now
+
+      # Fetch the plugin files
+      plugin_files = in_bolt_project(versioned_project) do |context|
+        files = {}
+
+        # Bolt also sets plugin_modulepath to user modulepath so do it here too for
+        # consistency
+        plugin_modulepath = context[:pal].user_modulepath
+        Puppet.lookup(:current_environment).override_with(modulepath: plugin_modulepath).modules.each do |mod|
+          search_dirs = block.call(mod)
+
+          files[mod] ||= []
+          Find.find(*search_dirs).each do |file|
+            files[mod] << file if File.file?(file)
+          end
+        end
+
+        files
+      end
+
+      # Pack the plugin files
+      sio = StringIO.new
+      begin
+        output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
+
+        plugin_files.each do |mod, files|
+          tar_dir = Pathname.new(mod.name)
+          mod_dir = Pathname.new(mod.path)
+
+          files.each do |file|
+            tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
+            stat = File.stat(file)
+            content = File.binread(file)
+            output.tar.add_file_simple(
+              tar_path.to_s,
+              data: content,
+              size: content.size,
+              mode: stat.mode & 0o777,
+              mtime: stat.mtime
+            )
+          end
+        end
+
+        duration = Time.now - start_time
+        @logger.trace("Packed plugins in #{duration * 1000} ms")
+      ensure
+        output.close
+      end
+
+      Base64.encode64(sio.string)
+    end
+
     get '/' do
       200
     end
@@ -419,6 +574,8 @@ module BoltServer
       run_task
       run_script
       upload_file
+      apply
+      apply_prep
     ].freeze
 
     def make_ssh_target(target_hash)
@@ -441,8 +598,9 @@ module BoltServer
         'uri' => target_hash['hostname'],
         'config' => {
           'transport' => 'ssh',
-          'ssh' => opts
-        }
+          'ssh' => opts.slice(*Bolt::Config::Transport::SSH.options)
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -479,8 +637,9 @@ module BoltServer
         'uri' => target_hash['hostname'],
         'config' => {
           'transport' => 'winrm',
-          'winrm' => opts
-        }
+          'winrm' => opts.slice(*Bolt::Config::Transport::WinRM.options)
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -699,60 +858,21 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      # Inspired by Bolt::Applicator.build_plugin_tarball
-      start_time = Time.now
-
-      # Fetch the plugin files
-      plugin_files = in_bolt_project(params['versioned_project']) do |context|
-        files = {}
-
-        # Bolt also sets plugin_modulepath to user modulepath so do it here too for
-        # consistency
-        plugin_modulepath = context[:pal].user_modulepath
-        Puppet.lookup(:current_environment).override_with(modulepath: plugin_modulepath).modules.each do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-
-          files[mod] ||= []
-          Find.find(*search_dirs).each do |file|
-            files[mod] << file if File.file?(file)
-          end
-        end
-
-        files
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'fact_plugins')
 
-      # Pack the plugin files
-      sio = StringIO.new
-      begin
-        output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
-
-        plugin_files.each do |mod, files|
-          tar_dir = Pathname.new(mod.name)
-          mod_dir = Pathname.new(mod.path)
+      [200, plugins_tarball.to_json]
+    end
 
-          files.each do |file|
-            tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
-            stat = File.stat(file)
-            content = File.binread(file)
-            output.tar.add_file_simple(
-              tar_path.to_s,
-              data: content,
-              size: content.size,
-              mode: stat.mode & 0o777,
-              mtime: stat.mtime
-            )
-          end
-        end
+    # Returns the base64 encoded tar archive of _all_ plugin code for a project
+    #
+    # @param versioned_project [String] the versioned_project to build the plugin tarball from
+    get '/project_plugin_tarball' do
+      raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
+      content_type :json
 
-        duration = Time.now - start_time
-        @logger.trace("Packed plugins in #{duration * 1000} ms")
-      ensure
-        output.close
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'all_plugins')
 
-      [200, Base64.encode64(sio.string).to_json]
+      [200, plugins_tarball.to_json]
     end
 
     error 404 do