bolt 2.30.0 → 2.34.0

data/lib/bolt_server/transport_app.rb

@@ -14,7 +14,9 @@ require 'json-schema'
 
 # These are only needed for the `/plans` endpoint.
 require 'puppet'
-require 'bolt_server/pe/pal'
+
+# Needed by the `/project_file_metadatas` endpoint
+require 'puppet/file_serving/fileset'
 
 module BoltServer
   class TransportApp < Sinatra::Base
@@ -35,6 +37,17 @@ module BoltServer
       transport-winrm
     ].freeze
 
+    # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
+    # in Bolt::PAL. Paths and variable names are similar to what exists in
+    # Bolt::PAL, but with a 'PE' prefix.
+    PE_BOLTLIB_PATH = '/opt/puppetlabs/server/apps/bolt-server/pe-bolt-modules'
+
+    # For now at least, we maintain an entirely separate codedir from
+    # puppetserver by default, so that filesync can work properly. If filesync
+    # is not used, this can instead match the usual puppetserver codedir.
+    # See the `orchestrator.bolt.codedir` tk config setting.
+    DEFAULT_BOLT_CODEDIR = '/opt/puppetlabs/server/data/orchestration-services/code'
+
     def initialize(config)
       @config = config
       @schemas = Hash[REQUEST_SCHEMAS.map do |basename|
@@ -191,10 +204,52 @@ module BoltServer
       [@executor.run_script(target, file_location, body['arguments'])]
     end
 
+    # This function is nearly identical to Bolt::Pal's `with_puppet_settings` with the
+    # one difference that we set the codedir to point to actual code, rather than the
+    # tmpdir. We only use this funtion inside the Modulepath initializer so that Puppet
+    # is correctly configured to pull environment configuration correctly. If we don't
+    # set codedir in this way: when we try to load and interpolate the modulepath it
+    # won't correctly load.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def with_pe_pal_init_settings(codedir, environmentpath, basemodulepath)
+      Dir.mktmpdir('pe-bolt') do |dir|
+        cli = []
+        Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
+          dir = setting == :codedir ? codedir : dir
+          cli << "--#{setting}" << dir
+        end
+        cli << "--environmentpath" << environmentpath
+        cli << "--basemodulepath" << basemodulepath
+        Puppet.settings.send(:clear_everything_for_tests)
+        Puppet.initialize_settings(cli)
+        yield
+      end
+    end
+
+    # Use puppet to identify the modulepath from an environment.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def modulepath_from_environment(environment_name)
+      codedir = DEFAULT_BOLT_CODEDIR
+      environmentpath = "#{codedir}/environments"
+      basemodulepath = "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
+      modulepath_dirs = nil
+      with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
+        environment = Puppet.lookup(:environments).get!(environment_name)
+        modulepath_dirs = environment.modulepath
+      end
+      modulepath_dirs
+    end
+
     def in_pe_pal_env(environment)
       return [400, '`environment` is a required argument'] if environment.nil?
       @pal_mutex.synchronize do
-        pal = BoltServer::PE::PAL.new({}, environment)
+        modulepath_obj = Bolt::Config::Modulepath.new(
+          modulepath_from_environment(environment),
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_obj, nil, nil)
         yield pal
       rescue Puppet::Environments::EnvironmentNotFound
         [400, {
@@ -213,16 +268,11 @@ module BoltServer
       @pal_mutex.synchronize do
         project = Bolt::Project.create_project(project_dir)
         bolt_config = Bolt::Config.from_project(project, { log: { 'bolt-debug.log' => 'disable' } })
-        pal = Bolt::PAL.new(bolt_config.modulepath, nil, nil, nil, nil, nil, bolt_config.project)
-        module_path = [
-          BoltServer::PE::PAL::PE_BOLTLIB_PATH,
-          Bolt::PAL::BOLTLIB_PATH,
-          *bolt_config.modulepath,
-          Bolt::PAL::MODULES_PATH
-        ]
-        # CODEREVIEW: I *think* this is the only thing we need to make different between bolt's PAL. Is it acceptable
-        # to hack this? Modulepath is currently a readable attribute, could we make it writeable?
-        pal.instance_variable_set(:@modulepath, module_path)
+        modulepath_object = Bolt::Config::Modulepath.new(
+          bolt_config.modulepath,
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_object, nil, nil, nil, nil, nil, bolt_config.project)
         context = {
           pal: pal,
           config: bolt_config
@@ -307,6 +357,23 @@ module BoltServer
       plans.map { |plan_name| { 'name' => plan_name } }
     end
 
+    def file_metadatas(pal, module_name, file)
+      pal.in_bolt_compiler do
+        mod = Puppet.lookup(:current_environment).module(module_name)
+        raise ArgumentError, "`module_name`: #{module_name} does not exist" unless mod
+        abs_file_path = mod.file(file)
+        raise ArgumentError, "`file`: #{file} does not exist inside the module's 'files' directory" unless abs_file_path
+        fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
+        Puppet::FileServing::Fileset.merge(fileset).collect do |relative_file_path, base_path|
+          metadata = Puppet::FileServing::Metadata.new(base_path, relative_path: relative_file_path)
+          metadata.checksum_type = 'sha256'
+          metadata.links = 'follow'
+          metadata.collect
+          metadata.to_data_hash
+        end
+      end
+    end
+
     get '/' do
       200
     end
@@ -550,6 +617,19 @@ module BoltServer
       end
     end
 
+    # Implements puppetserver's file_metadatas endpoint for projects.
+    #
+    # @param project_ref [String] the project_ref to fetch the file metadatas from
+    get '/project_file_metadatas/:module_name/*' do
+      in_bolt_project(params['project_ref']) do |context|
+        file = params[:splat].first
+        metadatas = file_metadatas(context[:pal], params[:module_name], file)
+        [200, metadatas.to_json]
+      end
+    rescue ArgumentError => e
+      [400, e.message]
+    end
+
     error 404 do
       err = Bolt::Error.new("Could not find route #{request.path}",
                             'boltserver/not-found')

data/lib/bolt_spec/bolt_context.rb

@@ -105,7 +105,7 @@ module BoltSpec
 
       # Set the things
       Puppet[:tasks] = true
-      RSpec.configuration.module_path = [modulepath, Bolt::PAL::BOLTLIB_PATH].join(File::PATH_SEPARATOR)
+      RSpec.configuration.module_path = [modulepath, Bolt::Config::Modulepath::BOLTLIB_PATH].join(File::PATH_SEPARATOR)
       opts = {
         bolt_executor: executor,
         bolt_inventory: inventory,
@@ -153,7 +153,9 @@ module BoltSpec
     end
 
     def pal
-      @pal ||= Bolt::PAL.new(config.modulepath, config.hiera_config, config.project.resource_types)
+      @pal ||= Bolt::PAL.new(Bolt::Config::Modulepath.new(config.modulepath),
+                             config.hiera_config,
+                             config.project.resource_types)
     end
 
     BoltSpec::Plans::MOCKED_ACTIONS.each do |action|

data/lib/bolt_spec/plans.rb

@@ -224,7 +224,7 @@ module BoltSpec
 
     def run_plan(name, params)
       pal = Bolt::PAL.new(
-        config.modulepath,
+        Bolt::Config::Modulepath.new(config.modulepath),
         config.hiera_config,
         config.project.resource_types,
         config.compile_concurrency,

data/lib/bolt_spec/plans/action_stubs/command_stub.rb

@@ -29,7 +29,7 @@ module BoltSpec
       end
 
       def result_for(target, stdout: '', stderr: '')
-        Bolt::Result.for_command(target, stdout, stderr, 0, 'command', '')
+        Bolt::Result.for_command(target, stdout, stderr, 0, 'command', '', [])
       end
 
       # Public methods

data/lib/bolt_spec/plans/action_stubs/script_stub.rb

@@ -35,7 +35,7 @@ module BoltSpec
       end
 
       def result_for(target, stdout: '', stderr: '')
-        Bolt::Result.for_command(target, stdout, stderr, 0, 'script', '')
+        Bolt::Result.for_command(target, stdout, stderr, 0, 'script', '', [])
       end
 
       # Public methods

data/lib/bolt_spec/plans/mock_executor.rb

@@ -48,7 +48,7 @@ module BoltSpec
         ([segments[0]] + segments[2..-1]).join('/')
       end
 
-      def run_command(targets, command, options = {})
+      def run_command(targets, command, options = {}, _position = [])
         result = nil
         if (doub = @command_doubles[command] || @command_doubles[:default])
           result = doub.process(targets, command, options)
@@ -61,7 +61,7 @@ module BoltSpec
         result
       end
 
-      def run_script(targets, script_path, arguments, options = {})
+      def run_script(targets, script_path, arguments, options = {}, _position = [])
         script = module_file_id(script_path)
         result = nil
         if (doub = @script_doubles[script] || @script_doubles[:default])
@@ -76,7 +76,7 @@ module BoltSpec
         result
       end
 
-      def run_task(targets, task, arguments, options = {})
+      def run_task(targets, task, arguments, options = {}, _position = [])
         result = nil
         if (doub = @task_doubles[task.name] || @task_doubles[:default])
           result = doub.process(targets, task.name, arguments, options)
@@ -90,7 +90,7 @@ module BoltSpec
         result
       end
 
-      def download_file(targets, source, destination, options = {})
+      def download_file(targets, source, destination, options = {}, _position = [])
         result = nil
         if (doub = @download_doubles[source] || @download_doubles[:default])
           result = doub.process(targets, source, destination, options)
@@ -103,7 +103,7 @@ module BoltSpec
         result
       end
 
-      def upload_file(targets, source_path, destination, options = {})
+      def upload_file(targets, source_path, destination, options = {}, _position = [])
         source = module_file_id(source_path)
         result = nil
         if (doub = @upload_doubles[source] || @upload_doubles[:default])

data/lib/bolt_spec/run.rb

@@ -179,7 +179,7 @@ module BoltSpec
       end
 
       def pal
-        @pal ||= Bolt::PAL.new(config.modulepath,
+        @pal ||= Bolt::PAL.new(Bolt::Config::Modulepath.new(config.modulepath),
                                config.hiera_config,
                                config.project.resource_types,
                                config.compile_concurrency)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 2.30.0
+  version: 2.34.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-30 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -182,16 +182,22 @@ dependencies:
   name: puppet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 6.18.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 6.18.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.20'
 - !ruby/object:Gem::Dependency
   name: puppetfile-resolver
   requirement: !ruby/object:Gem::Requirement
@@ -432,6 +438,9 @@ files:
 - bolt-modules/system/lib/puppet/functions/system/env.rb
 - exe/bolt
 - guides/inventory.txt
+- guides/logging.txt
+- guides/module.txt
+- guides/modulepath.txt
 - guides/project.txt
 - lib/bolt.rb
 - lib/bolt/analytics.rb
@@ -444,6 +453,7 @@ files:
 - lib/bolt/catalog/logging.rb
 - lib/bolt/cli.rb
 - lib/bolt/config.rb
+- lib/bolt/config/modulepath.rb
 - lib/bolt/config/options.rb
 - lib/bolt/config/transport/base.rb
 - lib/bolt/config/transport/docker.rb
@@ -462,6 +472,15 @@ files:
 - lib/bolt/logger.rb
 - lib/bolt/module.rb
 - lib/bolt/module_installer.rb
+- lib/bolt/module_installer/installer.rb
+- lib/bolt/module_installer/puppetfile.rb
+- lib/bolt/module_installer/puppetfile/forge_module.rb
+- lib/bolt/module_installer/puppetfile/git_module.rb
+- lib/bolt/module_installer/puppetfile/module.rb
+- lib/bolt/module_installer/resolver.rb
+- lib/bolt/module_installer/specs.rb
+- lib/bolt/module_installer/specs/forge_spec.rb
+- lib/bolt/module_installer/specs/git_spec.rb
 - lib/bolt/node/errors.rb
 - lib/bolt/node/output.rb
 - lib/bolt/outputter.rb
@@ -487,6 +506,7 @@ files:
 - lib/bolt/pal/yaml_plan/step/task.rb
 - lib/bolt/pal/yaml_plan/step/upload.rb
 - lib/bolt/pal/yaml_plan/transpiler.rb
+- lib/bolt/plan_creator.rb
 - lib/bolt/plan_result.rb
 - lib/bolt/plugin.rb
 - lib/bolt/plugin/env_var.rb
@@ -503,9 +523,6 @@ files:
 - lib/bolt/puppetdb.rb
 - lib/bolt/puppetdb/client.rb
 - lib/bolt/puppetdb/config.rb
-- lib/bolt/puppetfile.rb
-- lib/bolt/puppetfile/installer.rb
-- lib/bolt/puppetfile/module.rb
 - lib/bolt/r10k_log_proxy.rb
 - lib/bolt/rerun.rb
 - lib/bolt/resource_instance.rb
@@ -542,7 +559,6 @@ files:
 - lib/bolt_server/base_config.rb
 - lib/bolt_server/config.rb
 - lib/bolt_server/file_cache.rb
-- lib/bolt_server/pe/pal.rb
 - lib/bolt_server/schemas/action-check_node_connections.json
 - lib/bolt_server/schemas/action-run_command.json
 - lib/bolt_server/schemas/action-run_script.json
@@ -583,7 +599,6 @@ files:
 - modules/canary/lib/puppet/functions/canary/skip.rb
 - modules/canary/plans/init.pp
 - modules/puppetdb_fact/plans/init.pp
-- modules/secure_env_vars/plans/init.pp
 homepage: https://github.com/puppetlabs/bolt
 licenses:
 - Apache-2.0

data/lib/bolt/puppetfile.rb

@@ -1,142 +0,0 @@
-# frozen_string_literal: true
-
-require 'bolt/error'
-require 'bolt/puppetfile/module'
-
-# This class manages the logical contents of a Puppetfile. It includes methods
-# for parsing a Puppetfile and its modules, resolving module dependencies,
-# and writing a Puppetfile.
-#
-module Bolt
-  class Puppetfile
-    attr_reader :modules
-
-    def initialize(modules = [])
-      @modules = Set.new
-      add_modules(modules)
-    end
-
-    # Loads a Puppetfile and parses its module specifications, returning a
-    # Bolt::Puppetfile object with the modules set.
-    #
-    def self.parse(path, skip_unsupported_modules: false)
-      require 'puppetfile-resolver'
-      require 'puppetfile-resolver/puppetfile/parser/r10k_eval'
-
-      begin
-        parsed = ::PuppetfileResolver::Puppetfile::Parser::R10KEval.parse(File.read(path))
-      rescue StandardError => e
-        raise Bolt::Error.new(
-          "Unable to parse Puppetfile #{path}: #{e.message}",
-          'bolt/puppetfile-parsing'
-        )
-      end
-
-      unless parsed.valid?
-        # valid? Just checks if validation_errors is empty, so if we get here we know it's not.
-        raise Bolt::ValidationError, <<~MSG
-        Unable to parse Puppetfile #{path}:
-        #{parsed.validation_errors.join("\n\n")}.
-        This may not be a Puppetfile managed by Bolt.
-        MSG
-      end
-
-      modules = parsed.modules.each_with_object([]) do |mod, acc|
-        unless mod.instance_of? PuppetfileResolver::Puppetfile::ForgeModule
-          next if skip_unsupported_modules
-
-          raise Bolt::ValidationError,
-                "Module '#{mod.title}' is not a Puppet Forge module. Unable to "\
-                "parse Puppetfile #{path}."
-        end
-
-        acc << Bolt::Puppetfile::Module.new(mod.owner, mod.name, mod.version)
-      end
-
-      new(modules)
-    end
-
-    # Writes a Puppetfile that includes specifications for each of the
-    # modules.
-    #
-    def write(path, moduledir = nil)
-      File.open(path, 'w') do |file|
-        file.puts '# This Puppetfile is managed by Bolt. Do not edit.'
-        file.puts "moduledir '#{moduledir.basename}'" if moduledir
-        modules.each { |mod| file.puts mod.to_spec }
-      end
-    rescue SystemCallError => e
-      raise Bolt::FileError.new(
-        "#{e.message}: unable to write Puppetfile.",
-        path
-      )
-    end
-
-    # Resolves module dependencies using the puppetfile-resolver library. The
-    # resolver will return a document model including all module dependencies
-    # and the latest version that can be installed for each. The document model
-    # is parsed and turned into a Set of Bolt::Puppetfile::Module objects.
-    #
-    def resolve
-      require 'puppetfile-resolver'
-
-      # Build the document model from the modules.
-      model = PuppetfileResolver::Puppetfile::Document.new('')
-
-      @modules.each do |mod|
-        model.add_module(
-          PuppetfileResolver::Puppetfile::ForgeModule.new(mod.title).tap do |tap|
-            tap.version = mod.version || :latest
-          end
-        )
-      end
-
-      # Make sure the Puppetfile model is valid.
-      unless model.valid?
-        raise Bolt::ValidationError,
-              "Unable to resolve dependencies for modules: #{@modules.map(&:title).join(', ')}"
-      end
-
-      # Create the resolver using the Puppetfile model. nil disables Puppet
-      # version restrictions.
-      resolver = PuppetfileResolver::Resolver.new(model, nil)
-
-      # Configure and resolve the dependency graph, catching any errors
-      # raised by puppetfile-resolver and re-raising them as Bolt errors.
-      begin
-        result = resolver.resolve(
-          cache:                 nil,
-          ui:                    nil,
-          module_paths:          [],
-          allow_missing_modules: false
-        )
-      rescue StandardError => e
-        raise Bolt::Error.new(e.message, 'bolt/puppetfile-resolver-error')
-      end
-
-      # Turn specifications into module objects. This will skip over anything that is not
-      # a module specification (i.e. a Puppet version specification).
-      @modules = result.specifications.each_with_object(Set.new) do |(_name, spec), acc|
-        next unless spec.instance_of? PuppetfileResolver::Models::ModuleSpecification
-        acc << Bolt::Puppetfile::Module.new(spec.owner, spec.name, spec.version.to_s)
-      end
-    end
-
-    # Adds to the set of modules.
-    #
-    def add_modules(modules)
-      Array(modules).each do |mod|
-        case mod
-        when Bolt::Puppetfile::Module
-          @modules << mod
-        when Hash
-          @modules << Bolt::Puppetfile::Module.from_hash(mod)
-        else
-          raise Bolt::ValidationError, "Module must be a Bolt::Puppetfile::Module or Hash."
-        end
-      end
-
-      @modules
-    end
-  end
-end