bolt 3.15.0 → 3.18.0

data/lib/bolt/transport/local/connection.rb

@@ -3,13 +3,15 @@
 require 'open3'
 require 'fileutils'
 require 'tempfile'
-require 'bolt/node/output'
-require 'bolt/util'
+require_relative '../../../bolt/node/output'
+require_relative '../../../bolt/util'
 
 module Bolt
   module Transport
     class Local < Simple
       class Connection
+        RUBY_ENV_VARS = %w[GEM_PATH GEM_HOME RUBYLIB RUBYLIB_PREFIX RUBYOPT RUBYPATH RUBYSHELL].freeze
+
         attr_accessor :user, :logger, :target
 
         def initialize(target)
@@ -68,7 +70,21 @@ module Bolt
             end
           end
 
-          Open3.popen3(*command)
+          # Only do this if bundled-ruby is set to false, not nil
+          ruby_env_vars = if target.transport_config['bundled-ruby'] == false
+                            RUBY_ENV_VARS.each_with_object({}) do |e, acc|
+                              acc[e] = ENV["BOLT_ORIG_#{e}"] if ENV["BOLT_ORIG_#{e}"]
+                            end
+                          end
+
+          if target.transport_config['bundled-ruby'] == false &&
+             Gem.loaded_specs.keys.include?('bundler')
+            Bundler.with_unbundled_env do
+              Open3.popen3(ruby_env_vars || {}, *command)
+            end
+          else
+            Open3.popen3(ruby_env_vars || {}, *command)
+          end
         end
 
         # This is used by the Bash shell to decide whether to `cd` before

data/lib/bolt/transport/local.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'bolt/logger'
-require 'bolt/transport/simple'
+require_relative '../../bolt/logger'
+require_relative '../../bolt/transport/simple'
 
 module Bolt
   module Transport
@@ -21,4 +21,4 @@ module Bolt
   end
 end
 
-require 'bolt/transport/local/connection'
+require_relative 'local/connection'

data/lib/bolt/transport/lxd/connection.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'logging'
-require 'bolt/node/errors'
+require_relative '../../../bolt/node/errors'
 
 module Bolt
   module Transport

data/lib/bolt/transport/lxd.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'bolt/logger'
-require 'bolt/node/errors'
-require 'bolt/transport/simple'
+require_relative '../../bolt/logger'
+require_relative '../../bolt/node/errors'
+require_relative '../../bolt/transport/simple'
 
 module Bolt
   module Transport
@@ -23,4 +23,4 @@ module Bolt
   end
 end
 
-require 'bolt/transport/lxd/connection'
+require_relative 'lxd/connection'

data/lib/bolt/transport/orch/connection.rb

@@ -36,7 +36,8 @@ module Bolt
           end
           logger.debug("Creating orchestrator client for #{client_opts}")
           @client = OrchestratorClient.new(client_opts, true)
-          @plan_job = start_plan(plan_context)
+          @plan_context = plan_context
+          @plan_job = start_plan(@plan_context)
           logger.debug("Started plan #{@plan_job}")
           @environment = opts["task-environment"]
         end
@@ -87,6 +88,17 @@ module Bolt
         def run_task(targets, task, arguments, options)
           body = build_request(targets, task, arguments, options[:description])
           @client.run_task(body)
+        rescue OrchestratorClient::ApiError => e
+          if e.data['kind'] == 'puppetlabs.orchestrator/plan-already-finished'
+            @logger.debug("Retrying the task")
+            # Instead of recursing, just retry once
+            @plan_job = start_plan(@plan_context)
+            # Rebuild the request with the new plan job ID
+            body = build_request(targets, task, arguments, options[:description])
+            @client.run_task(body)
+          else
+            raise e
+          end
         end
 
         def query_inventory(targets)

data/lib/bolt/transport/orch.rb

@@ -4,8 +4,8 @@ require 'base64'
 require 'find'
 require 'json'
 require 'pathname'
-require 'bolt/transport/base'
-require 'bolt/transport/orch/connection'
+require_relative '../../bolt/transport/base'
+require_relative 'orch/connection'
 
 module Bolt
   module Transport

data/lib/bolt/transport/podman/connection.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'logging'
-require 'bolt/node/errors'
+require_relative '../../../bolt/node/errors'
 
 module Bolt
   module Transport

data/lib/bolt/transport/podman.rb

@@ -2,7 +2,7 @@
 
 require 'json'
 require 'shellwords'
-require 'bolt/transport/base'
+require_relative '../../bolt/transport/base'
 
 module Bolt
   module Transport
@@ -16,4 +16,4 @@ module Bolt
   end
 end
 
-require 'bolt/transport/podman/connection'
+require_relative 'podman/connection'

data/lib/bolt/transport/remote.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'bolt/task'
-require 'bolt/transport/base'
+require_relative '../../bolt/task'
+require_relative '../../bolt/transport/base'
 
 module Bolt
   module Transport

data/lib/bolt/transport/simple.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 require 'logging'
-require 'bolt/result'
-require 'bolt/shell'
-require 'bolt/transport/base'
+require_relative '../../bolt/result'
+require_relative '../../bolt/shell'
+require_relative '../../bolt/transport/base'
 
 module Bolt
   module Transport

data/lib/bolt/transport/ssh/connection.rb

@@ -2,9 +2,9 @@
 
 require 'logging'
 require 'shellwords'
-require 'bolt/node/errors'
-require 'bolt/node/output'
-require 'bolt/util'
+require_relative '../../../bolt/node/errors'
+require_relative '../../../bolt/node/output'
+require_relative '../../../bolt/util'
 
 module Bolt
   module Transport

data/lib/bolt/transport/ssh.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'bolt/logger'
-require 'bolt/node/errors'
-require 'bolt/transport/simple'
+require_relative '../../bolt/logger'
+require_relative '../../bolt/node/errors'
+require_relative '../../bolt/transport/simple'
 
 module Bolt
   module Transport
@@ -44,5 +44,5 @@ module Bolt
   end
 end
 
-require 'bolt/transport/ssh/connection'
-require 'bolt/transport/ssh/exec_connection'
+require_relative 'ssh/connection'
+require_relative 'ssh/exec_connection'

data/lib/bolt/transport/winrm/connection.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'bolt/node/errors'
-require 'bolt/node/output'
+require_relative '../../../bolt/node/errors'
+require_relative '../../../bolt/node/output'
 
 module Bolt
   module Transport

data/lib/bolt/transport/winrm.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'bolt/node/errors'
-require 'bolt/transport/base'
+require_relative '../../bolt/node/errors'
+require_relative '../../bolt/transport/base'
 
 module Bolt
   module Transport
@@ -30,4 +30,4 @@ module Bolt
   end
 end
 
-require 'bolt/transport/winrm/connection'
+require_relative 'winrm/connection'

data/lib/bolt/util.rb

@@ -103,7 +103,7 @@ module Bolt
       # absolute path or Puppet module syntax lookup. Returns the path to the
       # file if found, or nil.
       #
-      def find_file_from_scope(file, scope, fallback = false)
+      def find_file_from_scope(file, scope)
         # If we got an absolute path, just return that.
         return file if Pathname.new(file).absolute?
 
@@ -112,44 +112,21 @@ module Bolt
         mod_path = scope.compiler.environment.module(module_name)&.path
 
         # Search the module for the file, falling back to new-style paths if enabled.
-        find_file_in_module(mod_path, file_pattern, fallback) if mod_path
+        search_module(mod_path, file_pattern) if mod_path
       end
 
-      # This method is used by Bolt to find files when provided a
-      # module-style path without loading Puppet. It takes the absolute path to
-      # the module root and the module-style path minus the module name and
-      # searches subdirectories in the module in order of precedence.
+      # This searches a module for files under 'files/' or 'scripts/', falling
+      # back to the new style of file loading. It takes the absolute path to the
+      # module root and the relative path provided by the user.
       #
-      def find_file_in_module(module_path, module_file, fallback = false)
-        # If the first part of the path is 'scripts' or 'files', the path may
-        # be a new-style file location and should fall back to the new path.
-        subdir_or_file = split_path(module_file).first
-        case subdir_or_file
-          # For any subdirs that may indicate the user passed a new-style path,
-          # first look in 'mymod/files/<relative_path>' (old-style) then fall
-          # back to 'mymod/<relative_path>' (new-style) if enabled.
-        when 'scripts', 'files'
-          search_module(module_path, module_file, fallback)
-        else
-          # If the path definitely isn't new-style, only look in the 'files/'
-          # directory.
-          search_module(module_path, module_file)
-        end
-      end
-
-      # This searches a module for files under 'files/' or 'scripts/',
-      # optionally falling back to the new style of file loading. It takes the
-      # absolute path to the module root, the relative path provided by the
-      # user, and whether to fall back to the new-style script loading if the
-      # file isn't found in 'files/'.
-      #
-      private def search_module(module_path, module_file, fallback = false)
+      def search_module(module_path, module_file)
         if File.exist?(File.join(module_path, 'files', module_file))
           File.join(module_path, 'files', module_file)
-        elsif File.exist?(File.join(module_path, module_file)) && fallback
+        elsif File.exist?(File.join(module_path, module_file))
           File.join(module_path, module_file)
         end
       end
+      alias find_file_in_module search_module
 
       # Copied directly from puppet/lib/puppet/parser/files.rb
       #

data/lib/bolt/validator.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'bolt/error'
+require_relative '../bolt/error'
 
 # This class validates config against a schema, raising an error that includes
 # details about any invalid configuration.

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.15.0'
+  VERSION = '3.18.0'
 end

data/lib/bolt.rb

@@ -4,5 +4,5 @@
 require 'logging_extensions/logging'
 
 module Bolt
-  require 'bolt/executor'
+  require_relative 'bolt/executor'
 end

data/lib/bolt_server/file_cache.rb

@@ -182,5 +182,17 @@ module BoltServer
         end
       end
     end
+
+    def get_cached_project_file(versioned_project, file_name)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.read(file_path) if File.exist?(file_path) }
+    end
+
+    def cache_project_file(versioned_project, file_name, data)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
+    end
   end
 end

data/lib/bolt_server/schemas/action-apply.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply request",
+  "description": "POST <transport>/apply request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "string",
+      "description": "Project from which to load code"
+    },
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "catalog": {
+          "type": "object",
+          "description": "Compiled catalog to apply"
+        },
+        "apply_options": {
+          "type": "object",
+          "description": "Options for application of a catalog"
+        }
+      }
+    },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    },
+    "target": { "$ref": "partial:target-any" }
+  },
+  "required": ["target", "versioned_project", "parameters", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/schemas/action-apply_prep.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply_prep request",
+  "description": "POST <transport>/apply_prep request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "String",
+      "description": "Project from which to load code"
+    },
+    "target": { "$ref": "partial:target-any" },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    }
+  },
+  "required": ["target", "versioned_project", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/transport_app.rb

@@ -43,6 +43,8 @@ module BoltServer
       transport-ssh
       transport-winrm
       connect-data
+      action-apply_prep
+      action-apply
     ].freeze
 
     # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
@@ -75,6 +77,12 @@ module BoltServer
       # This is needed until the PAL is threadsafe.
       @pal_mutex = Mutex.new
 
+      # Avoid redundant plugin tarbal construction
+      @plugin_mutex = Mutex.new
+
+      # Avoid redundant project_task metadata construction
+      @task_metadata_mutex = Mutex.new
+
       @logger = Bolt::Logger.logger(self)
 
       super(nil)
@@ -118,12 +126,7 @@ module BoltServer
       end
     end
 
-    def run_task(target, body)
-      validate_schema(@schemas["action-run_task"], body)
-
-      task_data = body['task']
-      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
-      parameters = body['parameters'] || {}
+    def task_helper(target, task, parameters)
       # Wrap parameters marked with '"sensitive": true' in the task metadata with a
       # Sensitive wrapper type. This way it's not shown in logs.
       if (param_spec = task.parameters)
@@ -142,6 +145,102 @@ module BoltServer
       end
     end
 
+    def run_task(target, body)
+      validate_schema(@schemas["action-run_task"], body)
+
+      task_data = body['task']
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'] || {})
+    end
+
+    def extract_install_task(target)
+      unless target.plugin_hooks['puppet_library']['task']
+        raise BoltServer::RequestError,
+              "Target must have 'task' plugin hook"
+      end
+      install_task = target.plugin_hooks['puppet_library']['task'].split('::', 2)
+      install_task << 'init' if install_task.count == 1
+      install_task
+    end
+
+    # This helper is responsible for computing or retrieving from the cache a plugin tarball. There are
+    # two supported plugin types 'fact_plugins', and 'all_plugins'. Note that this is cached based on
+    # versioned_project as there are no plugins in the "builtin content" directory
+    def plugin_tarball(versioned_project, tarball_type)
+      tarball_types = %w[fact_plugins all_plugins]
+      unless tarball_types.include?(tarball_type)
+        raise ArgumentError,
+              "tarball_type must be one of: #{tarball_types.join(', ')}"
+      end
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint
+      # the tarball computation only happens once (all the other targets will just need to read the cached data)
+      @plugin_mutex.synchronize do
+        if (tarball = @file_cache.get_cached_project_file(versioned_project, tarball_type))
+          tarball
+        else
+          new_tarball = build_project_plugins_tarball(versioned_project) do |mod|
+            search_dirs = []
+            search_dirs << mod.plugins if mod.plugins?
+            search_dirs << mod.pluginfacts if mod.pluginfacts?
+            if tarball_type == 'all_plugins'
+              search_dirs << mod.files if mod.files?
+              search_dirs << mod.scripts if mod.scripts?
+              type_files = "#{mod.path}/types"
+              search_dirs << type_files if File.exist?(type_files)
+            end
+            search_dirs
+          end
+          @file_cache.cache_project_file(versioned_project, tarball_type, new_tarball)
+          new_tarball
+        end
+      end
+    end
+
+    # This helper is responsible for computing or retrieving task metadata for a project.
+    # It expects task name in segments and uses the combination of task name and versioned_project
+    # as a unique identifier for caching in addition to the job_id. The job id is added to protect against
+    # a case where the buildtin content is update (where the apply_helpers would be managed)
+    def project_task_metadata(versioned_project, task_name_segments, job_id)
+      cached_file_name = "#{task_name_segments.join('_')}_#{job_id}"
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint the
+      # metadata computation will only be computed once, then the cache will be read.
+      @task_metadata_mutex.synchronize do
+        if (metadata = @file_cache.get_cached_project_file(versioned_project, cached_file_name))
+          JSON.parse(metadata)
+        else
+          new_metadata = in_bolt_project(versioned_project) do |context|
+            ps_parameters = {
+              'versioned_project' => versioned_project
+            }
+            pe_task_info(context[:pal], *task_name_segments, ps_parameters)
+          end
+          @file_cache.cache_project_file(versioned_project, cached_file_name, new_metadata.to_json)
+          new_metadata
+        end
+      end
+    end
+
+    def apply_prep(target, body)
+      validate_schema(@schemas["action-apply_prep"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'fact_plugins')
+      install_task_segments = extract_install_task(target.first)
+      task_data = project_task_metadata(body['versioned_project'], install_task_segments, body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      install_task_result = task_helper(target, task, target.first.plugin_hooks['puppet_library']['parameters'] || {})
+      return install_task_result unless install_task_result.ok
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers custom_facts], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, { 'plugins' => plugins_tarball })
+    end
+
+    def apply(target, body)
+      validate_schema(@schemas["action-apply"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'all_plugins')
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers apply_catalog], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'].merge({ 'plugins' => plugins_tarball }))
+    end
+
     def run_command(target, body)
       validate_schema(@schemas["action-run_command"], body)
       command = body['command']
@@ -370,13 +469,26 @@ module BoltServer
         pal.in_bolt_compiler do
           mod = Puppet.lookup(:current_environment).module(module_name)
           raise BoltServer::RequestError, "module_name: '#{module_name}' does not exist" unless mod
-          mod.file(file)
+          # First, look in the 'old' location <module>/files/<path>.
+          # If not found, and the path starts with `files` or `scripts`, munge
+          # the path and look inside that directory.
+          if (abs_path = mod.file(file))
+            abs_path
+          else
+            subdir, relative_path = file.split(File::SEPARATOR, 2)
+            case subdir
+            when 'files'
+              mod.file(relative_path)
+            when 'scripts'
+              mod.script(relative_path)
+            end
+          end
         end
       end
 
       unless abs_file_path
         raise BoltServer::RequestError,
-              "file: '#{file}' does not exist inside the module's 'files' directory"
+              "file: '#{file}' does not exist inside #{module_name} 'files' or 'scripts' directories"
       end
 
       fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
@@ -476,6 +588,8 @@ module BoltServer
       run_task
       run_script
       upload_file
+      apply
+      apply_prep
     ].freeze
 
     def make_ssh_target(target_hash)
@@ -499,7 +613,8 @@ module BoltServer
         'config' => {
           'transport' => 'ssh',
           'ssh' => opts.slice(*Bolt::Config::Transport::SSH.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -537,7 +652,8 @@ module BoltServer
         'config' => {
           'transport' => 'winrm',
           'winrm' => opts.slice(*Bolt::Config::Transport::WinRM.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -730,7 +846,7 @@ module BoltServer
           }
 
           connect_plugin = BoltServer::Plugin::PuppetConnectData.new(body['puppet_connect_data'])
-          plugins = Bolt::Plugin.setup(context[:config], context[:pal], load_plugins: false)
+          plugins = Bolt::Plugin.new(context[:config], context[:pal], load_plugins: false)
           plugins.add_plugin(connect_plugin)
           %w[aws_inventory azure_inventory gcloud_inventory].each do |plugin_name|
             plugins.add_module_plugin(plugin_name) if plugins.known_plugin?(plugin_name)
@@ -756,12 +872,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'fact_plugins')
 
       [200, plugins_tarball.to_json]
     end
@@ -773,15 +884,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs << mod.files if mod.files?
-        type_files = "#{mod.path}/types"
-        search_dirs << type_files if File.exist?(type_files)
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'all_plugins')
 
       [200, plugins_tarball.to_json]
     end