bolt 2.26.0 → 2.31.0

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '2.26.0'
+  VERSION = '2.31.0'
 end

data/lib/bolt_server/acl.rb

@@ -16,9 +16,9 @@ module BoltServer
       end
     end
 
-    def initialize(app, whitelist)
+    def initialize(app, allowlist)
       acls = []
-      whitelist.each do |entry|
+      allowlist.each do |entry|
         acls << {
           'resources' => [
             {

data/lib/bolt_server/base_config.rb

@@ -7,7 +7,7 @@ module BoltServer
   class BaseConfig
     def config_keys
       %w[host port ssl-cert ssl-key ssl-ca-cert
-         ssl-cipher-suites loglevel logfile whitelist]
+         ssl-cipher-suites loglevel logfile allowlist projects-dir]
     end
 
     def env_keys
@@ -98,8 +98,8 @@ module BoltServer
         raise Bolt::ValidationError, "Configured 'ssl-cipher-suites' must be an array of cipher suite names"
       end
 
-      unless @data['whitelist'].nil? || @data['whitelist'].is_a?(Array)
-        raise Bolt::ValidationError, "Configured 'whitelist' must be an array of names"
+      unless @data['allowlist'].nil? || @data['allowlist'].is_a?(Array)
+        raise Bolt::ValidationError, "Configured 'allowlist' must be an array of names"
       end
     end
 

data/lib/bolt_server/config.rb

@@ -7,7 +7,7 @@ require 'bolt/error'
 module BoltServer
   class Config < BoltServer::BaseConfig
     def config_keys
-      super + %w[concurrency cache-dir file-server-conn-timeout file-server-uri]
+      super + %w[concurrency cache-dir file-server-conn-timeout file-server-uri projects-dir]
     end
 
     def env_keys

data/lib/bolt_server/file_cache.rb

@@ -64,17 +64,17 @@ module BoltServer
 
     def client
       @client ||= begin
-                    uri = URI(@config['file-server-uri'])
-                    https = Net::HTTP.new(uri.host, uri.port)
-                    https.use_ssl = true
-                    https.ssl_version = :TLSv1_2
-                    https.ca_file = @config['ssl-ca-cert']
-                    https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
-                    https.key = OpenSSL::PKey::RSA.new(ssl_key)
-                    https.verify_mode = OpenSSL::SSL::VERIFY_PEER
-                    https.open_timeout = @config['file-server-conn-timeout']
-                    https
-                  end
+        uri = URI(@config['file-server-uri'])
+        https = Net::HTTP.new(uri.host, uri.port)
+        https.use_ssl = true
+        https.ssl_version = :TLSv1_2
+        https.ca_file = @config['ssl-ca-cert']
+        https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
+        https.key = OpenSSL::PKey::RSA.new(ssl_key)
+        https.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        https.open_timeout = @config['file-server-conn-timeout']
+        https
+      end
     end
 
     def request_file(path, params, file)

data/lib/bolt_server/transport_app.rb

@@ -5,6 +5,7 @@ require 'addressable/uri'
 require 'bolt'
 require 'bolt/error'
 require 'bolt/inventory'
+require 'bolt/project'
 require 'bolt/target'
 require 'bolt_server/file_cache'
 require 'bolt/task/puppet_server'
@@ -13,7 +14,9 @@ require 'json-schema'
 
 # These are only needed for the `/plans` endpoint.
 require 'puppet'
-require 'bolt_server/pe/pal'
+
+# Needed by the `/project_file_metadatas` endpoint
+require 'puppet/file_serving/fileset'
 
 module BoltServer
   class TransportApp < Sinatra::Base
@@ -34,6 +37,17 @@ module BoltServer
       transport-winrm
     ].freeze
 
+    # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
+    # in Bolt::PAL. Paths and variable names are similar to what exists in
+    # Bolt::PAL, but with a 'PE' prefix.
+    PE_BOLTLIB_PATH = '/opt/puppetlabs/server/apps/bolt-server/pe-bolt-modules'
+
+    # For now at least, we maintain an entirely separate codedir from
+    # puppetserver by default, so that filesync can work properly. If filesync
+    # is not used, this can instead match the usual puppetserver codedir.
+    # See the `orchestrator.bolt.codedir` tk config setting.
+    DEFAULT_BOLT_CODEDIR = '/opt/puppetlabs/server/data/orchestration-services/code'
+
     def initialize(config)
       @config = config
       @schemas = Hash[REQUEST_SCHEMAS.map do |basename|
@@ -190,21 +204,82 @@ module BoltServer
       [@executor.run_script(target, file_location, body['arguments'])]
     end
 
-    def in_pe_pal_env(environment)
-      if environment.nil?
-        [400, '`environment` is a required argument']
-      else
-        @pal_mutex.synchronize do
-          pal = BoltServer::PE::PAL.new({}, environment)
-          yield pal
-        rescue Puppet::Environments::EnvironmentNotFound
-          [400, {
-            "class" => 'bolt/unknown-environment',
-            "message" => "Environment #{environment} not found"
-          }.to_json]
-        rescue Bolt::Error => e
-          [400, e.to_json]
+    # This function is nearly identical to Bolt::Pal's `with_puppet_settings` with the
+    # one difference that we set the codedir to point to actual code, rather than the
+    # tmpdir. We only use this funtion inside the Modulepath initializer so that Puppet
+    # is correctly configured to pull environment configuration correctly. If we don't
+    # set codedir in this way: when we try to load and interpolate the modulepath it
+    # won't correctly load.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def with_pe_pal_init_settings(codedir, environmentpath, basemodulepath)
+      Dir.mktmpdir('pe-bolt') do |dir|
+        cli = []
+        Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
+          dir = setting == :codedir ? codedir : dir
+          cli << "--#{setting}" << dir
         end
+        cli << "--environmentpath" << environmentpath
+        cli << "--basemodulepath" << basemodulepath
+        Puppet.settings.send(:clear_everything_for_tests)
+        Puppet.initialize_settings(cli)
+        yield
+      end
+    end
+
+    # Use puppet to identify the modulepath from an environment.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def modulepath_from_environment(environment_name)
+      codedir = DEFAULT_BOLT_CODEDIR
+      environmentpath = "#{codedir}/environments"
+      basemodulepath = "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
+      modulepath_dirs = nil
+      with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
+        environment = Puppet.lookup(:environments).get!(environment_name)
+        modulepath_dirs = environment.modulepath
+      end
+      modulepath_dirs
+    end
+
+    def in_pe_pal_env(environment)
+      return [400, '`environment` is a required argument'] if environment.nil?
+      @pal_mutex.synchronize do
+        modulepath_obj = Bolt::Config::Modulepath.new(
+          modulepath_from_environment(environment),
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_obj, nil, nil)
+        yield pal
+      rescue Puppet::Environments::EnvironmentNotFound
+        [400, {
+          "class" => 'bolt/unknown-environment',
+          "message" => "Environment #{environment} not found"
+        }.to_json]
+      rescue Bolt::Error => e
+        [400, e.to_json]
+      end
+    end
+
+    def in_bolt_project(bolt_project)
+      return [400, '`project_ref` is a required argument'] if bolt_project.nil?
+      project_dir = File.join(@config['projects-dir'], bolt_project)
+      return [400, "`project_ref`: #{project_dir} does not exist"] unless Dir.exist?(project_dir)
+      @pal_mutex.synchronize do
+        project = Bolt::Project.create_project(project_dir)
+        bolt_config = Bolt::Config.from_project(project, { log: { 'bolt-debug.log' => 'disable' } })
+        modulepath_object = Bolt::Config::Modulepath.new(
+          bolt_config.modulepath,
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_object, nil, nil, nil, nil, nil, bolt_config.project)
+        context = {
+          pal: pal,
+          config: bolt_config
+        }
+        yield context
+      rescue Bolt::Error => e
+        [400, e.to_json]
       end
     end
 
@@ -221,14 +296,12 @@ module BoltServer
       plan_info
     end
 
-    def build_puppetserver_uri(file_identifier, module_name, environment)
+    def build_puppetserver_uri(file_identifier, module_name, parameters)
       segments = file_identifier.split('/', 3)
       if segments.size == 1
         {
           'path' => "/puppet/v3/file_content/tasks/#{module_name}/#{file_identifier}",
-          'params' => {
-            'environment' => environment
-          }
+          'params' => parameters
         }
       else
         module_segment, mount_segment, name_segment = *segments
@@ -241,14 +314,12 @@ module BoltServer
                     when 'lib'
                       "/puppet/v3/file_content/plugins/#{name_segment}"
                     end,
-          'params' => {
-            'environment' => environment
-          }
+          'params' => parameters
         }
       end
     end
 
-    def pe_task_info(pal, module_name, task_name, environment)
+    def pe_task_info(pal, module_name, task_name, parameters)
       # Handle case where task name is simply module name with special `init` task
       task_name = if task_name == 'init' || task_name.nil?
                     module_name
@@ -261,7 +332,7 @@ module BoltServer
           'filename' => file_hash['name'],
           'sha256' => Digest::SHA256.hexdigest(File.read(file_hash['path'])),
           'size_bytes' => File.size(file_hash['path']),
-          'uri' => build_puppetserver_uri(file_hash['name'], module_name, environment)
+          'uri' => build_puppetserver_uri(file_hash['name'], module_name, parameters)
         }
       end
       {
@@ -271,6 +342,38 @@ module BoltServer
       }
     end
 
+    def allowed_helper(metadata, allowlist)
+      allowed = allowlist.nil? || allowlist.include?(metadata['name']) ? true : false
+      metadata.merge({ 'allowed' => allowed })
+    end
+
+    def task_list(pal)
+      tasks = pal.list_tasks
+      tasks.map { |task_name, _description| { 'name' => task_name } }
+    end
+
+    def plan_list(pal)
+      plans = pal.list_plans.flatten
+      plans.map { |plan_name| { 'name' => plan_name } }
+    end
+
+    def file_metadatas(pal, module_name, file)
+      pal.in_bolt_compiler do
+        mod = Puppet.lookup(:current_environment).module(module_name)
+        raise ArgumentError, "`module_name`: #{module_name} does not exist" unless mod
+        abs_file_path = mod.file(file)
+        raise ArgumentError, "`file`: #{file} does not exist inside the module's 'files' directory" unless abs_file_path
+        fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
+        Puppet::FileServing::Fileset.merge(fileset).collect do |relative_file_path, base_path|
+          metadata = Puppet::FileServing::Metadata.new(base_path, relative_path: relative_file_path)
+          metadata.checksum_type = 'sha256'
+          metadata.links = 'follow'
+          metadata.collect
+          metadata.to_data_hash
+        end
+      end
+    end
+
     get '/' do
       200
     end
@@ -401,12 +504,40 @@ module BoltServer
       end
     end
 
+    # Fetches the metadata for a single plan
+    #
+    # @param project_ref [String] the project to fetch the plan from
+    get '/project_plans/:module_name/:plan_name' do
+      in_bolt_project(params['project_ref']) do |context|
+        plan_info = pe_plan_info(context[:pal], params[:module_name], params[:plan_name])
+        plan_info = allowed_helper(plan_info, context[:config].project.plans)
+        [200, plan_info.to_json]
+      end
+    end
+
     # Fetches the metadata for a single task
     #
     # @param environment [String] the environment to fetch the task from
     get '/tasks/:module_name/:task_name' do
       in_pe_pal_env(params['environment']) do |pal|
-        task_info = pe_task_info(pal, params[:module_name], params[:task_name], params['environment'])
+        ps_parameters = {
+          'environment' => params['environment']
+        }
+        task_info = pe_task_info(pal, params[:module_name], params[:task_name], ps_parameters)
+        [200, task_info.to_json]
+      end
+    end
+
+    # Fetches the metadata for a single task
+    #
+    # @param bolt_project_ref [String] the reference to the bolt-project directory to load task metadata from
+    get '/project_tasks/:module_name/:task_name' do
+      in_bolt_project(params['project_ref']) do |context|
+        ps_parameters = {
+          'project' => params['project_ref']
+        }
+        task_info = pe_task_info(context[:pal], params[:module_name], params[:task_name], ps_parameters)
+        task_info = allowed_helper(task_info, context[:config].project.tasks)
         [200, task_info.to_json]
       end
     end
@@ -435,13 +566,30 @@ module BoltServer
       end
     end
 
+    # Fetches the list of plans for a project
+    #
+    # @param project_ref [String] the project to fetch the list of plans from
+    get '/project_plans' do
+      in_bolt_project(params['project_ref']) do |context|
+        plans_response = plan_list(context[:pal])
+
+        # Dig in context for the allowlist of plans from project object
+        plans_response.map! { |metadata| allowed_helper(metadata, context[:config].project.plans) }
+
+        # We structure this array of plans to be an array of hashes so that it matches the structure
+        # returned by the puppetserver API that serves data like this. Structuring the output this way
+        # makes switching between puppetserver and bolt-server easier, which makes changes to switch
+        # to bolt-server smaller/simpler.
+        [200, plans_response.to_json]
+      end
+    end
+
     # Fetches the list of tasks for an environment
     #
     # @param environment [String] the environment to fetch the list of tasks from
     get '/tasks' do
       in_pe_pal_env(params['environment']) do |pal|
-        tasks = pal.list_tasks
-        tasks_response = tasks.map { |task_name, _description| { 'name' => task_name } }.to_json
+        tasks_response = task_list(pal).to_json
 
         # We structure this array of tasks to be an array of hashes so that it matches the structure
         # returned by the puppetserver API that serves data like this. Structuring the output this way
@@ -451,6 +599,37 @@ module BoltServer
       end
     end
 
+    # Fetches the list of tasks for a bolt-project
+    #
+    # @param project_ref [String] the project to fetch the list of tasks from
+    get '/project_tasks' do
+      in_bolt_project(params['project_ref']) do |context|
+        tasks_response = task_list(context[:pal])
+
+        # Dig in context for the allowlist of tasks from project object
+        tasks_response.map! { |metadata| allowed_helper(metadata, context[:config].project.tasks) }
+
+        # We structure this array of tasks to be an array of hashes so that it matches the structure
+        # returned by the puppetserver API that serves data like this. Structuring the output this way
+        # makes switching between puppetserver and bolt-server easier, which makes changes to switch
+        # to bolt-server smaller/simpler.
+        [200, tasks_response.to_json]
+      end
+    end
+
+    # Implements puppetserver's file_metadatas endpoint for projects.
+    #
+    # @param project_ref [String] the project_ref to fetch the file metadatas from
+    get '/project_file_metadatas/:module_name/*' do
+      in_bolt_project(params['project_ref']) do |context|
+        file = params[:splat].first
+        metadatas = file_metadatas(context[:pal], params[:module_name], file)
+        [200, metadatas.to_json]
+      end
+    rescue ArgumentError => e
+      [400, e.message]
+    end
+
     error 404 do
       err = Bolt::Error.new("Could not find route #{request.path}",
                             'boltserver/not-found')

data/lib/bolt_spec/bolt_context.rb

@@ -105,7 +105,7 @@ module BoltSpec
 
       # Set the things
       Puppet[:tasks] = true
-      RSpec.configuration.module_path = [modulepath, Bolt::PAL::BOLTLIB_PATH].join(File::PATH_SEPARATOR)
+      RSpec.configuration.module_path = [modulepath, Bolt::Config::Modulepath::BOLTLIB_PATH].join(File::PATH_SEPARATOR)
       opts = {
         bolt_executor: executor,
         bolt_inventory: inventory,
@@ -142,10 +142,10 @@ module BoltSpec
     # Override in your tests
     def config
       @config ||= begin
-                    conf = Bolt::Config.default
-                    conf.modulepath = [modulepath].flatten
-                    conf
-                  end
+        conf = Bolt::Config.default
+        conf.modulepath = [modulepath].flatten
+        conf
+      end
     end
 
     def plugins
@@ -153,7 +153,9 @@ module BoltSpec
     end
 
     def pal
-      @pal ||= Bolt::PAL.new(config.modulepath, config.hiera_config, config.project.resource_types)
+      @pal ||= Bolt::PAL.new(Bolt::Config::Modulepath.new(config.modulepath),
+                             config.hiera_config,
+                             config.project.resource_types)
     end
 
     BoltSpec::Plans::MOCKED_ACTIONS.each do |action|

data/lib/bolt_spec/plans.rb

@@ -224,7 +224,7 @@ module BoltSpec
 
     def run_plan(name, params)
       pal = Bolt::PAL.new(
-        config.modulepath,
+        Bolt::Config::Modulepath.new(config.modulepath),
         config.hiera_config,
         config.project.resource_types,
         config.compile_concurrency,

data/lib/bolt_spec/plans/mock_executor.rb

@@ -40,7 +40,7 @@ module BoltSpec
 
       def module_file_id(file)
         modpath = @modulepath.select { |path| file =~ /^#{path}/ }
-        raise "Could not identify module path containing #{file}: #{modpath}" unless modpath.size == 1
+        raise "Could not identify modulepath containing #{file}: #{modpath}" unless modpath.size == 1
 
         path = Pathname.new(file)
         relative = path.relative_path_from(Pathname.new(modpath.first))