bolt 3.14.1 → 3.17.0

data/lib/bolt/project.rb

@@ -209,12 +209,5 @@ module Bolt
         Bolt::Logger.warn("missing_project_name", message)
       end
     end
-
-    def check_deprecated_file
-      if (@path + 'project.yaml').file?
-        msg = "Project configuration file 'project.yaml' is deprecated; use 'bolt-project.yaml' instead."
-        Bolt::Logger.warn("project_yaml", msg)
-      end
-    end
   end
 end

data/lib/bolt/result_set.rb

@@ -2,7 +2,8 @@
 
 module Bolt
   class ResultSet
-    attr_reader :results
+    attr_accessor :elapsed_time
+    attr_reader   :results
 
     include Enumerable
 

data/lib/bolt/transport/local/connection.rb

@@ -10,6 +10,8 @@ module Bolt
   module Transport
     class Local < Simple
       class Connection
+        RUBY_ENV_VARS = %w[GEM_PATH GEM_HOME RUBYLIB RUBYLIB_PREFIX RUBYOPT RUBYPATH RUBYSHELL].freeze
+
         attr_accessor :user, :logger, :target
 
         def initialize(target)
@@ -68,7 +70,21 @@ module Bolt
             end
           end
 
-          Open3.popen3(*command)
+          # Only do this if bundled-ruby is set to false, not nil
+          ruby_env_vars = if target.transport_config['bundled-ruby'] == false
+                            RUBY_ENV_VARS.each_with_object({}) do |e, acc|
+                              acc[e] = ENV["BOLT_ORIG_#{e}"] if ENV["BOLT_ORIG_#{e}"]
+                            end
+                          end
+
+          if target.transport_config['bundled-ruby'] == false &&
+             Gem.loaded_specs.keys.include?('bundler')
+            Bundler.with_unbundled_env do
+              Open3.popen3(ruby_env_vars || {}, *command)
+            end
+          else
+            Open3.popen3(ruby_env_vars || {}, *command)
+          end
         end
 
         # This is used by the Bash shell to decide whether to `cd` before

data/lib/bolt/transport/orch/connection.rb

@@ -36,7 +36,8 @@ module Bolt
           end
           logger.debug("Creating orchestrator client for #{client_opts}")
           @client = OrchestratorClient.new(client_opts, true)
-          @plan_job = start_plan(plan_context)
+          @plan_context = plan_context
+          @plan_job = start_plan(@plan_context)
           logger.debug("Started plan #{@plan_job}")
           @environment = opts["task-environment"]
         end
@@ -87,6 +88,17 @@ module Bolt
         def run_task(targets, task, arguments, options)
           body = build_request(targets, task, arguments, options[:description])
           @client.run_task(body)
+        rescue OrchestratorClient::ApiError => e
+          if e.data['kind'] == 'puppetlabs.orchestrator/plan-already-finished'
+            @logger.debug("Retrying the task")
+            # Instead of recursing, just retry once
+            @plan_job = start_plan(@plan_context)
+            # Rebuild the request with the new plan job ID
+            body = build_request(targets, task, arguments, options[:description])
+            @client.run_task(body)
+          else
+            raise e
+          end
         end
 
         def query_inventory(targets)

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.14.1'
+  VERSION = '3.17.0'
 end

data/lib/bolt_server/file_cache.rb

@@ -182,5 +182,17 @@ module BoltServer
         end
       end
     end
+
+    def get_cached_project_file(versioned_project, file_name)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.read(file_path) if File.exist?(file_path) }
+    end
+
+    def cache_project_file(versioned_project, file_name, data)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
+    end
   end
 end

data/lib/bolt_server/schemas/action-apply.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply request",
+  "description": "POST <transport>/apply request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "string",
+      "description": "Project from which to load code"
+    },
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "catalog": {
+          "type": "object",
+          "description": "Compiled catalog to apply"
+        },
+        "apply_options": {
+          "type": "object",
+          "description": "Options for application of a catalog"
+        }
+      }
+    },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    },
+    "target": { "$ref": "partial:target-any" }
+  },
+  "required": ["target", "versioned_project", "parameters", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/schemas/action-apply_prep.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply_prep request",
+  "description": "POST <transport>/apply_prep request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "String",
+      "description": "Project from which to load code"
+    },
+    "target": { "$ref": "partial:target-any" },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    }
+  },
+  "required": ["target", "versioned_project", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/transport_app.rb

@@ -43,6 +43,8 @@ module BoltServer
       transport-ssh
       transport-winrm
       connect-data
+      action-apply_prep
+      action-apply
     ].freeze
 
     # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
@@ -75,6 +77,12 @@ module BoltServer
       # This is needed until the PAL is threadsafe.
       @pal_mutex = Mutex.new
 
+      # Avoid redundant plugin tarbal construction
+      @plugin_mutex = Mutex.new
+
+      # Avoid redundant project_task metadata construction
+      @task_metadata_mutex = Mutex.new
+
       @logger = Bolt::Logger.logger(self)
 
       super(nil)
@@ -118,12 +126,7 @@ module BoltServer
       end
     end
 
-    def run_task(target, body)
-      validate_schema(@schemas["action-run_task"], body)
-
-      task_data = body['task']
-      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
-      parameters = body['parameters'] || {}
+    def task_helper(target, task, parameters)
       # Wrap parameters marked with '"sensitive": true' in the task metadata with a
       # Sensitive wrapper type. This way it's not shown in logs.
       if (param_spec = task.parameters)
@@ -142,6 +145,101 @@ module BoltServer
       end
     end
 
+    def run_task(target, body)
+      validate_schema(@schemas["action-run_task"], body)
+
+      task_data = body['task']
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'] || {})
+    end
+
+    def extract_install_task(target)
+      unless target.plugin_hooks['puppet_library']['task']
+        raise BoltServer::RequestError,
+              "Target must have 'task' plugin hook"
+      end
+      install_task = target.plugin_hooks['puppet_library']['task'].split('::', 2)
+      install_task << 'init' if install_task.count == 1
+      install_task
+    end
+
+    # This helper is responsible for computing or retrieving from the cache a plugin tarball. There are
+    # two supported plugin types 'fact_plugins', and 'all_plugins'. Note that this is cached based on
+    # versioned_project as there are no plugins in the "builtin content" directory
+    def plugin_tarball(versioned_project, tarball_type)
+      tarball_types = %w[fact_plugins all_plugins]
+      unless tarball_types.include?(tarball_type)
+        raise ArgumentError,
+              "tarball_type must be one of: #{tarball_types.join(', ')}"
+      end
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint
+      # the tarball computation only happens once (all the other targets will just need to read the cached data)
+      @plugin_mutex.synchronize do
+        if (tarball = @file_cache.get_cached_project_file(versioned_project, tarball_type))
+          tarball
+        else
+          new_tarball = build_project_plugins_tarball(versioned_project) do |mod|
+            search_dirs = []
+            search_dirs << mod.plugins if mod.plugins?
+            search_dirs << mod.pluginfacts if mod.pluginfacts?
+            if tarball_type == 'all_plugins'
+              search_dirs << mod.files if mod.files?
+              type_files = "#{mod.path}/types"
+              search_dirs << type_files if File.exist?(type_files)
+            end
+            search_dirs
+          end
+          @file_cache.cache_project_file(versioned_project, tarball_type, new_tarball)
+          new_tarball
+        end
+      end
+    end
+
+    # This helper is responsible for computing or retrieving task metadata for a project.
+    # It expects task name in segments and uses the combination of task name and versioned_project
+    # as a unique identifier for caching in addition to the job_id. The job id is added to protect against
+    # a case where the buildtin content is update (where the apply_helpers would be managed)
+    def project_task_metadata(versioned_project, task_name_segments, job_id)
+      cached_file_name = "#{task_name_segments.join('_')}_#{job_id}"
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint the
+      # metadata computation will only be computed once, then the cache will be read.
+      @task_metadata_mutex.synchronize do
+        if (metadata = @file_cache.get_cached_project_file(versioned_project, cached_file_name))
+          JSON.parse(metadata)
+        else
+          new_metadata = in_bolt_project(versioned_project) do |context|
+            ps_parameters = {
+              'versioned_project' => versioned_project
+            }
+            pe_task_info(context[:pal], *task_name_segments, ps_parameters)
+          end
+          @file_cache.cache_project_file(versioned_project, cached_file_name, new_metadata.to_json)
+          new_metadata
+        end
+      end
+    end
+
+    def apply_prep(target, body)
+      validate_schema(@schemas["action-apply_prep"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'fact_plugins')
+      install_task_segments = extract_install_task(target.first)
+      task_data = project_task_metadata(body['versioned_project'], install_task_segments, body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      install_task_result = task_helper(target, task, target.first.plugin_hooks['puppet_library']['parameters'] || {})
+      return install_task_result unless install_task_result.ok
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers custom_facts], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, { 'plugins' => plugins_tarball })
+    end
+
+    def apply(target, body)
+      validate_schema(@schemas["action-apply"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'all_plugins')
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers apply_catalog], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'].merge({ 'plugins' => plugins_tarball }))
+    end
+
     def run_command(target, body)
       validate_schema(@schemas["action-run_command"], body)
       command = body['command']
@@ -476,6 +574,8 @@ module BoltServer
       run_task
       run_script
       upload_file
+      apply
+      apply_prep
     ].freeze
 
     def make_ssh_target(target_hash)
@@ -499,7 +599,8 @@ module BoltServer
         'config' => {
           'transport' => 'ssh',
           'ssh' => opts.slice(*Bolt::Config::Transport::SSH.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -537,7 +638,8 @@ module BoltServer
         'config' => {
           'transport' => 'winrm',
           'winrm' => opts.slice(*Bolt::Config::Transport::WinRM.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -730,7 +832,7 @@ module BoltServer
           }
 
           connect_plugin = BoltServer::Plugin::PuppetConnectData.new(body['puppet_connect_data'])
-          plugins = Bolt::Plugin.setup(context[:config], context[:pal], load_plugins: false)
+          plugins = Bolt::Plugin.new(context[:config], context[:pal], load_plugins: false)
           plugins.add_plugin(connect_plugin)
           %w[aws_inventory azure_inventory gcloud_inventory].each do |plugin_name|
             plugins.add_module_plugin(plugin_name) if plugins.known_plugin?(plugin_name)
@@ -756,12 +858,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'fact_plugins')
 
       [200, plugins_tarball.to_json]
     end
@@ -773,15 +870,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs << mod.files if mod.files?
-        type_files = "#{mod.path}/types"
-        search_dirs << type_files if File.exist?(type_files)
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'all_plugins')
 
       [200, plugins_tarball.to_json]
     end

data/lib/bolt_spec/bolt_context.rb

@@ -154,7 +154,7 @@ module BoltSpec
     end
 
     def plugins
-      @plugins ||= Bolt::Plugin.setup(config, pal)
+      @plugins ||= Bolt::Plugin.new(config, pal)
     end
 
     def pal

data/lib/bolt_spec/run.rb

@@ -171,7 +171,7 @@ module BoltSpec
       end
 
       def plugins
-        @plugins ||= Bolt::Plugin.setup(config, pal)
+        @plugins ||= Bolt::Plugin.new(config, pal)
       end
 
       def puppetdb_client

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 3.14.1
+  version: 3.17.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-26 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -466,8 +466,18 @@ files:
 - bolt-modules/prompt/lib/puppet/functions/prompt/menu.rb
 - bolt-modules/system/lib/puppet/functions/system/env.rb
 - exe/bolt
+- guides/debugging.yaml
+- guides/inventory.yaml
+- guides/links.yaml
+- guides/logging.yaml
+- guides/module.yaml
+- guides/modulepath.yaml
+- guides/project.yaml
+- guides/targets.yaml
+- guides/transports.yaml
 - lib/bolt.rb
 - lib/bolt/analytics.rb
+- lib/bolt/application.rb
 - lib/bolt/applicator.rb
 - lib/bolt/apply_inventory.rb
 - lib/bolt/apply_result.rb
@@ -561,7 +571,6 @@ files:
 - lib/bolt/resource_instance.rb
 - lib/bolt/result.rb
 - lib/bolt/result_set.rb
-- lib/bolt/secret.rb
 - lib/bolt/shell.rb
 - lib/bolt/shell/bash.rb
 - lib/bolt/shell/bash/tmpdir.rb
@@ -601,6 +610,8 @@ files:
 - lib/bolt_server/plugin.rb
 - lib/bolt_server/plugin/puppet_connect_data.rb
 - lib/bolt_server/request_error.rb
+- lib/bolt_server/schemas/action-apply.json
+- lib/bolt_server/schemas/action-apply_prep.json
 - lib/bolt_server/schemas/action-check_node_connections.json
 - lib/bolt_server/schemas/action-run_command.json
 - lib/bolt_server/schemas/action-run_script.json

data/lib/bolt/secret.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require 'bolt/plugin'
-
-module Bolt
-  class Secret
-    KNOWN_KEYS = {
-      'createkeys' => %w[keysize private_key public_key],
-      'encrypt'    => %w[public_key],
-      'decrypt'    => %w[private_key public_key]
-    }.freeze
-
-    def self.execute(plugins, outputter, options)
-      name   = options[:plugin] || 'pkcs7'
-      plugin = plugins.by_name(name)
-
-      unless plugin
-        raise Bolt::Plugin::PluginError::Unknown, name
-      end
-
-      case options[:action]
-      when 'createkeys'
-        opts = { 'force' => options[:force] }.compact
-        result = plugins.get_hook(name, :secret_createkeys).call(opts)
-        outputter.print_message(result)
-      when 'encrypt'
-        encrypted = plugins.get_hook(name, :secret_encrypt).call('plaintext_value' => options[:object])
-        outputter.print_message(encrypted)
-      when 'decrypt'
-        decrypted = plugins.get_hook(name, :secret_decrypt).call('encrypted_value' => options[:object])
-        outputter.print_message(decrypted)
-      end
-
-      0
-    end
-  end
-end