bolt 3.16.0 → 3.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d61e436c7358897d1769da6db2544c2e5b3e23310bd5a6663d2b35e2afc1ab5
-  data.tar.gz: 1322c9a80154199cb658cf7807a88a96448b3b197ec5576ddaeabf76da52c29f
+  metadata.gz: 71d383b473f9356ff41f574bb0abe4aea9a9f8c3ac5055a459acffe0da290af1
+  data.tar.gz: 00d10803d865a9d6f0cb6eb7162e4a17f619b88e6702b559e0153b3796d732d0
 SHA512:
-  metadata.gz: 57bddbbc4d2cb78cb2aa6964e3ecff5a61c95c88e5f6b7524bbbc5b5ce3ca070098aae37d2ebb2ba3fcb569e3a082bf9ec4ebff2013bae31fc65eedadbde4b6c
-  data.tar.gz: 2c64c1607a21cdebe49550ad8207201fe853478c9f18cb4af04826410ddf507f8c505fbae5340faa64c070910e3f9e86372d4ae1de4b6108fad4eb95fc454316
+  metadata.gz: eaad9c12aab69e9d7c9f1d8b312623b8bcca4f26f983beb08b1d09c4568aafe0e430d639b88903df4e1a498a58c4c9d8add5d4b8b3930fe1dec3124bc15afad3
+  data.tar.gz: a05ec6c181fc6aa51c73ebfd928d73513d5506c1b0705bed4637bfa9490a6f1c010019c9209f7ed28c134f003a063ff6f4add490a55d658998f566d00c5ff017

data/Puppetfile

@@ -35,7 +35,7 @@ mod 'puppetlabs-stdlib', '7.1.0'
 mod 'puppetlabs-aws_inventory', '0.7.0'
 mod 'puppetlabs-azure_inventory', '0.5.0'
 mod 'puppetlabs-gcloud_inventory', '0.3.0'
-mod 'puppetlabs-http_request', '0.3.0'
+mod 'puppetlabs-http_request', '0.3.1'
 mod 'puppetlabs-pkcs7', '0.1.2'
 mod 'puppetlabs-secure_env_vars', '0.2.0'
 mod 'puppetlabs-terraform', '0.6.1'

data/lib/bolt/module_installer/puppetfile.rb

@@ -97,20 +97,34 @@ module Bolt
           end
         end
 
-        return if unsatisfied_specs.empty?
-
+        versionless_mods = @modules.select { |mod| mod.is_a?(ForgeModule) && mod.version.nil? }
         command = Bolt::Util.windows? ? 'Install-BoltModule -Force' : 'bolt module install --force'
 
-        message = <<~MESSAGE.chomp
-          Puppetfile does not include modules that satisfy the following specifications:
+        if unsatisfied_specs.any?
+          message = <<~MESSAGE.chomp
+            Puppetfile does not include modules that satisfy the following specifications:
+
+            #{unsatisfied_specs.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
+            
+            This Puppetfile might not be managed by Bolt. To forcibly overwrite the
+            Puppetfile, run '#{command}'.
+          MESSAGE
 
-          #{unsatisfied_specs.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
-          
-          This Puppetfile might not be managed by Bolt. To forcibly overwrite the
-          Puppetfile, run '#{command}'.
-        MESSAGE
+          raise Bolt::Error.new(message, 'bolt/missing-module-specs')
+        end
 
-        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
+        if versionless_mods.any?
+          message = <<~MESSAGE.chomp
+            Puppetfile includes Forge modules without a version requirement:
+            
+            #{versionless_mods.map(&:to_spec).join.chomp}
+            
+            This Puppetfile might not be managed by Bolt. To forcibly overwrite the
+            Puppetfile, run '#{command}'.
+          MESSAGE
+
+          raise Bolt::Error.new(message, 'bolt/missing-module-version-specs')
+        end
       end
     end
   end

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '3.16.0'
+  VERSION = '3.16.1'
 end

data/lib/bolt_server/file_cache.rb

@@ -182,5 +182,17 @@ module BoltServer
         end
       end
     end
+
+    def get_cached_project_file(versioned_project, file_name)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.read(file_path) if File.exist?(file_path) }
+    end
+
+    def cache_project_file(versioned_project, file_name, data)
+      file_dir = create_cache_dir(versioned_project)
+      file_path = File.join(file_dir, file_name)
+      serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
+    end
   end
 end

data/lib/bolt_server/schemas/action-apply.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply request",
+  "description": "POST <transport>/apply request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "string",
+      "description": "Project from which to load code"
+    },
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "catalog": {
+          "type": "object",
+          "description": "Compiled catalog to apply"
+        },
+        "apply_options": {
+          "type": "object",
+          "description": "Options for application of a catalog"
+        }
+      }
+    },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    },
+    "target": { "$ref": "partial:target-any" }
+  },
+  "required": ["target", "versioned_project", "parameters", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/schemas/action-apply_prep.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "apply_prep request",
+  "description": "POST <transport>/apply_prep request schema",
+  "type": "object",
+  "properties": {
+    "versioned_project": {
+      "type": "String",
+      "description": "Project from which to load code"
+    },
+    "target": { "$ref": "partial:target-any" },
+    "job_id": {
+      "type": "integer",
+      "description": "job-id associated with request"
+    }
+  },
+  "required": ["target", "versioned_project", "job_id"],
+  "additionalProperties": false
+}

data/lib/bolt_server/transport_app.rb

@@ -43,6 +43,8 @@ module BoltServer
       transport-ssh
       transport-winrm
       connect-data
+      action-apply_prep
+      action-apply
     ].freeze
 
     # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
@@ -75,6 +77,12 @@ module BoltServer
       # This is needed until the PAL is threadsafe.
       @pal_mutex = Mutex.new
 
+      # Avoid redundant plugin tarbal construction
+      @plugin_mutex = Mutex.new
+
+      # Avoid redundant project_task metadata construction
+      @task_metadata_mutex = Mutex.new
+
       @logger = Bolt::Logger.logger(self)
 
       super(nil)
@@ -118,12 +126,7 @@ module BoltServer
       end
     end
 
-    def run_task(target, body)
-      validate_schema(@schemas["action-run_task"], body)
-
-      task_data = body['task']
-      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
-      parameters = body['parameters'] || {}
+    def task_helper(target, task, parameters)
       # Wrap parameters marked with '"sensitive": true' in the task metadata with a
       # Sensitive wrapper type. This way it's not shown in logs.
       if (param_spec = task.parameters)
@@ -142,6 +145,101 @@ module BoltServer
       end
     end
 
+    def run_task(target, body)
+      validate_schema(@schemas["action-run_task"], body)
+
+      task_data = body['task']
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'] || {})
+    end
+
+    def extract_install_task(target)
+      unless target.plugin_hooks['puppet_library']['task']
+        raise BoltServer::RequestError,
+              "Target must have 'task' plugin hook"
+      end
+      install_task = target.plugin_hooks['puppet_library']['task'].split('::', 2)
+      install_task << 'init' if install_task.count == 1
+      install_task
+    end
+
+    # This helper is responsible for computing or retrieving from the cache a plugin tarball. There are
+    # two supported plugin types 'fact_plugins', and 'all_plugins'. Note that this is cached based on
+    # versioned_project as there are no plugins in the "builtin content" directory
+    def plugin_tarball(versioned_project, tarball_type)
+      tarball_types = %w[fact_plugins all_plugins]
+      unless tarball_types.include?(tarball_type)
+        raise ArgumentError,
+              "tarball_type must be one of: #{tarball_types.join(', ')}"
+      end
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint
+      # the tarball computation only happens once (all the other targets will just need to read the cached data)
+      @plugin_mutex.synchronize do
+        if (tarball = @file_cache.get_cached_project_file(versioned_project, tarball_type))
+          tarball
+        else
+          new_tarball = build_project_plugins_tarball(versioned_project) do |mod|
+            search_dirs = []
+            search_dirs << mod.plugins if mod.plugins?
+            search_dirs << mod.pluginfacts if mod.pluginfacts?
+            if tarball_type == 'all_plugins'
+              search_dirs << mod.files if mod.files?
+              type_files = "#{mod.path}/types"
+              search_dirs << type_files if File.exist?(type_files)
+            end
+            search_dirs
+          end
+          @file_cache.cache_project_file(versioned_project, tarball_type, new_tarball)
+          new_tarball
+        end
+      end
+    end
+
+    # This helper is responsible for computing or retrieving task metadata for a project.
+    # It expects task name in segments and uses the combination of task name and versioned_project
+    # as a unique identifier for caching in addition to the job_id. The job id is added to protect against
+    # a case where the buildtin content is update (where the apply_helpers would be managed)
+    def project_task_metadata(versioned_project, task_name_segments, job_id)
+      cached_file_name = "#{task_name_segments.join('_')}_#{job_id}"
+      # lock this so that in the case an apply/apply_prep with multiple targets hits this endpoint the
+      # metadata computation will only be computed once, then the cache will be read.
+      @task_metadata_mutex.synchronize do
+        if (metadata = @file_cache.get_cached_project_file(versioned_project, cached_file_name))
+          JSON.parse(metadata)
+        else
+          new_metadata = in_bolt_project(versioned_project) do |context|
+            ps_parameters = {
+              'versioned_project' => versioned_project
+            }
+            pe_task_info(context[:pal], *task_name_segments, ps_parameters)
+          end
+          @file_cache.cache_project_file(versioned_project, cached_file_name, new_metadata.to_json)
+          new_metadata
+        end
+      end
+    end
+
+    def apply_prep(target, body)
+      validate_schema(@schemas["action-apply_prep"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'fact_plugins')
+      install_task_segments = extract_install_task(target.first)
+      task_data = project_task_metadata(body['versioned_project'], install_task_segments, body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      install_task_result = task_helper(target, task, target.first.plugin_hooks['puppet_library']['parameters'] || {})
+      return install_task_result unless install_task_result.ok
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers custom_facts], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, { 'plugins' => plugins_tarball })
+    end
+
+    def apply(target, body)
+      validate_schema(@schemas["action-apply"], body)
+      plugins_tarball = plugin_tarball(body['versioned_project'], 'all_plugins')
+      task_data = project_task_metadata(body['versioned_project'], %w[apply_helpers apply_catalog], body["job_id"])
+      task = Bolt::Task::PuppetServer.new(task_data['name'], task_data['metadata'], task_data['files'], @file_cache)
+      task_helper(target, task, body['parameters'].merge({ 'plugins' => plugins_tarball }))
+    end
+
     def run_command(target, body)
       validate_schema(@schemas["action-run_command"], body)
       command = body['command']
@@ -476,6 +574,8 @@ module BoltServer
       run_task
       run_script
       upload_file
+      apply
+      apply_prep
     ].freeze
 
     def make_ssh_target(target_hash)
@@ -499,7 +599,8 @@ module BoltServer
         'config' => {
           'transport' => 'ssh',
           'ssh' => opts.slice(*Bolt::Config::Transport::SSH.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -537,7 +638,8 @@ module BoltServer
         'config' => {
           'transport' => 'winrm',
           'winrm' => opts.slice(*Bolt::Config::Transport::WinRM.options)
-        }
+        },
+        'plugin_hooks' => target_hash['plugin_hooks']
       }
 
       inventory = Bolt::Inventory.empty
@@ -756,12 +858,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'fact_plugins')
 
       [200, plugins_tarball.to_json]
     end
@@ -773,15 +870,7 @@ module BoltServer
       raise BoltServer::RequestError, "'versioned_project' is a required argument" if params['versioned_project'].nil?
       content_type :json
 
-      plugins_tarball = build_project_plugins_tarball(params['versioned_project']) do |mod|
-        search_dirs = []
-        search_dirs << mod.plugins if mod.plugins?
-        search_dirs << mod.pluginfacts if mod.pluginfacts?
-        search_dirs << mod.files if mod.files?
-        type_files = "#{mod.path}/types"
-        search_dirs << type_files if File.exist?(type_files)
-        search_dirs
-      end
+      plugins_tarball = plugin_tarball(params['versioned_project'], 'all_plugins')
 
       [200, plugins_tarball.to_json]
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 3.16.0
+  version: 3.16.1
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-09 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -610,6 +610,8 @@ files:
 - lib/bolt_server/plugin.rb
 - lib/bolt_server/plugin/puppet_connect_data.rb
 - lib/bolt_server/request_error.rb
+- lib/bolt_server/schemas/action-apply.json
+- lib/bolt_server/schemas/action-apply_prep.json
 - lib/bolt_server/schemas/action-check_node_connections.json
 - lib/bolt_server/schemas/action-run_command.json
 - lib/bolt_server/schemas/action-run_script.json