bolt 2.27.0 → 2.32.0

data/lib/bolt/project_migrator/modules.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require 'bolt/project_migrator/base'
+
+module Bolt
+  class ProjectMigrator
+    class Modules < Base
+      def migrate(project, configured_modulepath)
+        return true unless project.modules.nil?
+
+        @outputter.print_message "Migrating project modules\n\n"
+
+        config            = project.project_file
+        puppetfile        = project.puppetfile
+        managed_moduledir = project.managed_moduledir
+        modulepath        = [(project.path + 'modules').to_s,
+                             (project.path + 'site-modules').to_s,
+                             (project.path + 'site').to_s]
+
+        # Notify user to manually migrate modules if using non-default modulepath
+        if configured_modulepath != modulepath
+          @outputter.print_action_step(
+            "Project has a non-default configured modulepath, unable to automatically "\
+            "migrate project modules. To migrate project modules manually, see "\
+            "http://pup.pt/bolt-modules"
+          )
+          true
+        # Migrate modules from Puppetfile
+        elsif File.exist?(puppetfile)
+          migrate_modules_from_puppetfile(config, puppetfile, managed_moduledir, modulepath)
+        # Migrate modules to updated modulepath
+        else
+          consolidate_modules(modulepath)
+          update_project_config([], config)
+        end
+      end
+
+      # Migrates modules by reading a Puppetfile and prompting the user for
+      # which ones are direct dependencies for the project. Once the user has
+      # selected the direct dependencies, this will resolve the modules, write a
+      # new Puppetfile, install the modules, and then move any remaining modules
+      # to the new moduledir.
+      #
+      private def migrate_modules_from_puppetfile(config, puppetfile_path, managed_moduledir, modulepath)
+        require 'bolt/module_installer/installer'
+        require 'bolt/module_installer/puppetfile'
+        require 'bolt/module_installer/resolver'
+        require 'bolt/module_installer/specs'
+
+        begin
+          @outputter.print_action_step("Parsing Puppetfile at #{puppetfile_path}")
+          puppetfile = Bolt::ModuleInstaller::Puppetfile.parse(puppetfile_path, skip_unsupported_modules: true)
+        rescue Bolt::Error => e
+          @outputter.print_action_error("#{e.message}\nSkipping module migration.")
+          return false
+        end
+
+        # Prompt for direct dependencies
+        modules = select_modules(puppetfile.modules)
+
+        # Create specs to resolve from
+        specs = Bolt::ModuleInstaller::Specs.new(modules.map(&:to_hash))
+
+        # Attempt to resolve dependencies
+        begin
+          @outputter.print_message('')
+          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          puppetfile = Bolt::ModuleInstaller::Resolver.new.resolve(specs)
+        rescue Bolt::Error => e
+          @outputter.print_action_error("#{e.message}\nSkipping module migration.")
+          return false
+        end
+
+        migrate_managed_modules(puppetfile, puppetfile_path, managed_moduledir)
+
+        # Move remaining modules to 'modules'
+        consolidate_modules(modulepath)
+
+        # Delete old modules that are now managed
+        delete_modules(modulepath.first, puppetfile.modules)
+
+        # Add modules to project
+        update_project_config(modules.map(&:to_hash), config)
+      end
+
+      # Migrates the managed modules. If modules were selected to be managed,
+      # the Puppetfile is rewritten and modules are installed. If no modules
+      # were selected, the Puppetfile is deleted.
+      #
+      private def migrate_managed_modules(puppetfile, puppetfile_path, managed_moduledir)
+        if puppetfile.modules.any?
+          # Show the new Puppetfile content
+          message  = "Generated new Puppetfile content:\n\n"
+          message += puppetfile.modules.map(&:to_spec).join("\n").to_s
+          @outputter.print_action_step(message)
+
+          # Write Puppetfile
+          @outputter.print_action_step("Updating Puppetfile at #{puppetfile_path}")
+          puppetfile.write(puppetfile_path, managed_moduledir)
+
+          # Install Puppetfile
+          @outputter.print_action_step("Syncing modules from #{puppetfile_path} to #{managed_moduledir}")
+          Bolt::ModuleInstaller::Installer.new.install(puppetfile_path, managed_moduledir)
+        else
+          @outputter.print_action_step(
+            "Project does not include any managed modules, deleting Puppetfile "\
+            "at #{puppetfile_path}"
+          )
+          FileUtils.rm(puppetfile_path)
+        end
+      end
+
+      # Prompts the user to select modules, returning a list of
+      # the selected modules.
+      #
+      private def select_modules(modules)
+        @outputter.print_action_step(
+          "Select modules that are direct dependencies of your project. Bolt will "\
+          "automatically manage dependencies for each module selected, so do not "\
+          "select a module's dependencies unless you use content from it directly "\
+          "in your project."
+        )
+
+        all = Bolt::Util.prompt_yes_no("Select all modules?", @outputter)
+        return modules if all
+
+        modules.select do |mod|
+          Bolt::Util.prompt_yes_no("Select #{mod.full_name}?", @outputter)
+        end
+      end
+
+      # Consolidates all modules on the modulepath to 'modules'.
+      #
+      private def consolidate_modules(modulepath)
+        moduledir, *sources = modulepath
+
+        sources.select! { |source| Dir.exist?(source) }
+
+        if sources.any?
+          @outputter.print_action_step(
+            "Moving modules from #{sources.join(', ')} to #{moduledir}"
+          )
+
+          FileUtils.mkdir_p(moduledir)
+          move_modules(moduledir, sources)
+        end
+      end
+
+      # Moves modules from a list of source directories to the specified
+      # moduledir, deleting the source directory after it's done.
+      #
+      private def move_modules(moduledir, sources)
+        moduledir = Pathname.new(moduledir)
+
+        sources.each do |source|
+          source = Pathname.new(source)
+
+          source.each_child do |mod|
+            next unless mod.directory?
+            next if (moduledir + mod.basename).directory?
+            FileUtils.mv(mod, moduledir)
+          end
+
+          FileUtils.rm_r(source)
+        end
+      end
+
+      # Deletes modules from a specified directory.
+      #
+      private def delete_modules(moduledir, modules)
+        @outputter.print_action_step("Cleaning up #{moduledir}")
+        moduledir = Pathname.new(moduledir)
+
+        modules.each do |mod|
+          path = moduledir + mod.name
+          FileUtils.rm_r(path) if path.directory?
+        end
+      end
+
+      # Adds a list of modules to the project configuration file.
+      #
+      private def update_project_config(modules, config_file)
+        @outputter.print_action_step("Updating project configuration at #{config_file}")
+        data = Bolt::Util.read_optional_yaml_hash(config_file, 'project')
+        data.merge!('modules' => modules)
+        data.delete('modulepath')
+
+        begin
+          File.write(config_file, data.to_yaml)
+          true
+        rescue StandardError => e
+          raise Bolt::FileError.new(
+            "Unable to write to #{config_file}: #{e.message}",
+            config_file
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bolt/shell/bash.rb

@@ -143,7 +143,7 @@ module Bolt
 
             execute_options[:stdin] = stdin
             execute_options[:sudoable] = true if run_as
-            output = execute(remote_task_path, execute_options)
+            output = execute(remote_task_path, **execute_options)
           end
           Bolt::Result.for_task(target, output.stdout.string,
                                 output.stderr.string,
@@ -202,13 +202,14 @@ module Bolt
       end
 
       def handle_sudo_errors(err)
-        if err =~ /^#{conn.user} is not in the sudoers file\./
+        case err
+        when /^#{conn.user} is not in the sudoers file\./
           @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "User #{conn.user} does not have sudo permission on #{target}",
             'SUDO_DENIED'
           )
-        elsif err =~ /^Sorry, try again\./
+        when /^Sorry, try again\./
           @logger.trace { err }
           raise Bolt::Node::EscalateError.new(
             "Sudo password for user #{conn.user} not recognized on #{target}",

data/lib/bolt/transport/base.rb

@@ -47,10 +47,10 @@ module Bolt
         callback&.call(type: :node_start, target: target)
 
         result = begin
-                   yield
-                 rescue StandardError, NotImplementedError => e
-                   Bolt::Result.from_exception(target, e, action: action)
-                 end
+          yield
+        rescue StandardError, NotImplementedError => e
+          Bolt::Result.from_exception(target, e, action: action)
+        end
 
         callback&.call(type: :node_result, result: result)
         result

data/lib/bolt/transport/ssh/connection.rb

@@ -30,7 +30,7 @@ module Bolt
           @transport_logger = transport_logger
           @logger.trace("Initializing ssh connection to #{@target.safe_name}")
 
-          if target.options['private-key']&.instance_of?(String)
+          if target.options['private-key'].instance_of?(String)
             begin
               Bolt::Util.validate_file('ssh key', target.options['private-key'])
             rescue Bolt::FileError => e

data/lib/bolt/util.rb

@@ -3,6 +3,25 @@
 module Bolt
   module Util
     class << self
+      # Gets input for an argument.
+      def get_arg_input(value)
+        if value.start_with?('@')
+          file = value.sub(/^@/, '')
+          read_arg_file(file)
+        elsif value == '-'
+          $stdin.read
+        else
+          value
+        end
+      end
+
+      # Reads a file passed as an argument to a command.
+      def read_arg_file(file)
+        File.read(File.expand_path(file))
+      rescue StandardError => e
+        raise Bolt::FileError.new("Error attempting to read #{file}: #{e}", file)
+      end
+
       def read_yaml_hash(path, file_name)
         require 'yaml'
 
@@ -179,16 +198,16 @@ module Bolt
           # object was frozen
           frozen = obj.frozen?
           cl = begin
-                 obj.clone(freeze: false)
-                 # Some datatypes, such as FalseClass, can't be unfrozen. These
-                 # aren't the types we recurse on, so we can leave them frozen
-               rescue ArgumentError => e
-                 if e.message =~ /can't unfreeze/
-                   obj.clone
-                 else
-                   raise e
-                 end
-               end
+            obj.clone(freeze: false)
+          # Some datatypes, such as FalseClass, can't be unfrozen. These
+          # aren't the types we recurse on, so we can leave them frozen
+          rescue ArgumentError => e
+            if e.message =~ /can't unfreeze/
+              obj.clone
+            else
+              raise e
+            end
+          end
         rescue *error_types
           cloned[obj.object_id] = obj
           obj
@@ -280,6 +299,28 @@ module Bolt
         raise Bolt::ValidationError, "path must be a String, received #{path.class} #{path}" unless path.is_a?(String)
         path.split(%r{[/\\]}).last
       end
+
+      # Prompts yes or no, returning true for yes and false for no.
+      #
+      def prompt_yes_no(prompt, outputter)
+        choices = {
+          'y'   => true,
+          'yes' => true,
+          'n'   => false,
+          'no'  => false
+        }
+
+        loop do
+          outputter.print_prompt("#{prompt} ([y]es/[n]o) ")
+          response = $stdin.gets.to_s.downcase.chomp
+
+          if choices.key?(response)
+            return choices[response]
+          else
+            outputter.print_prompt_error("Invalid response, must pick [y]es or [n]o")
+          end
+        end
+      end
     end
   end
 end

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '2.27.0'
+  VERSION = '2.32.0'
 end

data/lib/bolt_server/acl.rb

@@ -16,9 +16,9 @@ module BoltServer
       end
     end
 
-    def initialize(app, whitelist)
+    def initialize(app, allowlist)
       acls = []
-      whitelist.each do |entry|
+      allowlist.each do |entry|
         acls << {
           'resources' => [
             {

data/lib/bolt_server/base_config.rb

@@ -7,7 +7,7 @@ module BoltServer
   class BaseConfig
     def config_keys
       %w[host port ssl-cert ssl-key ssl-ca-cert
-         ssl-cipher-suites loglevel logfile whitelist projects-dir]
+         ssl-cipher-suites loglevel logfile allowlist projects-dir]
     end
 
     def env_keys
@@ -98,8 +98,8 @@ module BoltServer
         raise Bolt::ValidationError, "Configured 'ssl-cipher-suites' must be an array of cipher suite names"
       end
 
-      unless @data['whitelist'].nil? || @data['whitelist'].is_a?(Array)
-        raise Bolt::ValidationError, "Configured 'whitelist' must be an array of names"
+      unless @data['allowlist'].nil? || @data['allowlist'].is_a?(Array)
+        raise Bolt::ValidationError, "Configured 'allowlist' must be an array of names"
       end
     end
 

data/lib/bolt_server/file_cache.rb

@@ -64,17 +64,17 @@ module BoltServer
 
     def client
       @client ||= begin
-                    uri = URI(@config['file-server-uri'])
-                    https = Net::HTTP.new(uri.host, uri.port)
-                    https.use_ssl = true
-                    https.ssl_version = :TLSv1_2
-                    https.ca_file = @config['ssl-ca-cert']
-                    https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
-                    https.key = OpenSSL::PKey::RSA.new(ssl_key)
-                    https.verify_mode = OpenSSL::SSL::VERIFY_PEER
-                    https.open_timeout = @config['file-server-conn-timeout']
-                    https
-                  end
+        uri = URI(@config['file-server-uri'])
+        https = Net::HTTP.new(uri.host, uri.port)
+        https.use_ssl = true
+        https.ssl_version = :TLSv1_2
+        https.ca_file = @config['ssl-ca-cert']
+        https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
+        https.key = OpenSSL::PKey::RSA.new(ssl_key)
+        https.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        https.open_timeout = @config['file-server-conn-timeout']
+        https
+      end
     end
 
     def request_file(path, params, file)

data/lib/bolt_server/schemas/partials/task.json

@@ -63,9 +63,24 @@
                   "environment": {
                     "description": "Environment the task is in",
                     "type": "string"
+                  },
+                  "project": {
+                    "description": "Project the task is in",
+                    "type": "string"
                   }
                 },
-                "required": ["environment"],
+                "oneOf": [
+                  {
+                    "required": [
+                      "environment"
+                    ]
+                  },
+                  {
+                    "required": [
+                      "project"
+                    ]
+                  }
+                ],              
                 "additionalProperties": true
               }
             },
@@ -90,5 +105,5 @@
     }
   },
   "required": ["name", "files"],
-  "additionalProperties": false
+  "additionalProperties": true
 }

data/lib/bolt_server/transport_app.rb

@@ -14,7 +14,9 @@ require 'json-schema'
 
 # These are only needed for the `/plans` endpoint.
 require 'puppet'
-require 'bolt_server/pe/pal'
+
+# Needed by the `/project_file_metadatas` endpoint
+require 'puppet/file_serving/fileset'
 
 module BoltServer
   class TransportApp < Sinatra::Base
@@ -35,6 +37,17 @@ module BoltServer
       transport-winrm
     ].freeze
 
+    # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
+    # in Bolt::PAL. Paths and variable names are similar to what exists in
+    # Bolt::PAL, but with a 'PE' prefix.
+    PE_BOLTLIB_PATH = '/opt/puppetlabs/server/apps/bolt-server/pe-bolt-modules'
+
+    # For now at least, we maintain an entirely separate codedir from
+    # puppetserver by default, so that filesync can work properly. If filesync
+    # is not used, this can instead match the usual puppetserver codedir.
+    # See the `orchestrator.bolt.codedir` tk config setting.
+    DEFAULT_BOLT_CODEDIR = '/opt/puppetlabs/server/data/orchestration-services/code'
+
     def initialize(config)
       @config = config
       @schemas = Hash[REQUEST_SCHEMAS.map do |basename|
@@ -191,10 +204,52 @@ module BoltServer
       [@executor.run_script(target, file_location, body['arguments'])]
     end
 
+    # This function is nearly identical to Bolt::Pal's `with_puppet_settings` with the
+    # one difference that we set the codedir to point to actual code, rather than the
+    # tmpdir. We only use this funtion inside the Modulepath initializer so that Puppet
+    # is correctly configured to pull environment configuration correctly. If we don't
+    # set codedir in this way: when we try to load and interpolate the modulepath it
+    # won't correctly load.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def with_pe_pal_init_settings(codedir, environmentpath, basemodulepath)
+      Dir.mktmpdir('pe-bolt') do |dir|
+        cli = []
+        Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
+          dir = setting == :codedir ? codedir : dir
+          cli << "--#{setting}" << dir
+        end
+        cli << "--environmentpath" << environmentpath
+        cli << "--basemodulepath" << basemodulepath
+        Puppet.settings.send(:clear_everything_for_tests)
+        Puppet.initialize_settings(cli)
+        yield
+      end
+    end
+
+    # Use puppet to identify the modulepath from an environment.
+    #
+    # WARNING: THIS FUNCTION SHOULD ONLY BE CALLED INSIDE A SYNCHRONIZED PAL MUTEX
+    def modulepath_from_environment(environment_name)
+      codedir = DEFAULT_BOLT_CODEDIR
+      environmentpath = "#{codedir}/environments"
+      basemodulepath = "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
+      modulepath_dirs = nil
+      with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
+        environment = Puppet.lookup(:environments).get!(environment_name)
+        modulepath_dirs = environment.modulepath
+      end
+      modulepath_dirs
+    end
+
     def in_pe_pal_env(environment)
       return [400, '`environment` is a required argument'] if environment.nil?
       @pal_mutex.synchronize do
-        pal = BoltServer::PE::PAL.new({}, environment)
+        modulepath_obj = Bolt::Config::Modulepath.new(
+          modulepath_from_environment(environment),
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_obj, nil, nil)
         yield pal
       rescue Puppet::Environments::EnvironmentNotFound
         [400, {
@@ -212,17 +267,12 @@ module BoltServer
       return [400, "`project_ref`: #{project_dir} does not exist"] unless Dir.exist?(project_dir)
       @pal_mutex.synchronize do
         project = Bolt::Project.create_project(project_dir)
-        bolt_config = Bolt::Config.from_project(project, {})
-        pal = Bolt::PAL.new(bolt_config.modulepath, nil, nil, nil, nil, nil, bolt_config.project)
-        module_path = [
-          BoltServer::PE::PAL::PE_BOLTLIB_PATH,
-          Bolt::PAL::BOLTLIB_PATH,
-          *bolt_config.modulepath,
-          Bolt::PAL::MODULES_PATH
-        ]
-        # CODEREVIEW: I *think* this is the only thing we need to make different between bolt's PAL. Is it acceptable
-        # to hack this? Modulepath is currently a readable attribute, could we make it writeable?
-        pal.instance_variable_set(:@modulepath, module_path)
+        bolt_config = Bolt::Config.from_project(project, { log: { 'bolt-debug.log' => 'disable' } })
+        modulepath_object = Bolt::Config::Modulepath.new(
+          bolt_config.modulepath,
+          boltlib_path: [PE_BOLTLIB_PATH, Bolt::Config::Modulepath::BOLTLIB_PATH]
+        )
+        pal = Bolt::PAL.new(modulepath_object, nil, nil, nil, nil, nil, bolt_config.project)
         context = {
           pal: pal,
           config: bolt_config
@@ -307,6 +357,23 @@ module BoltServer
       plans.map { |plan_name| { 'name' => plan_name } }
     end
 
+    def file_metadatas(pal, module_name, file)
+      pal.in_bolt_compiler do
+        mod = Puppet.lookup(:current_environment).module(module_name)
+        raise ArgumentError, "`module_name`: #{module_name} does not exist" unless mod
+        abs_file_path = mod.file(file)
+        raise ArgumentError, "`file`: #{file} does not exist inside the module's 'files' directory" unless abs_file_path
+        fileset = Puppet::FileServing::Fileset.new(abs_file_path, 'recurse' => 'yes')
+        Puppet::FileServing::Fileset.merge(fileset).collect do |relative_file_path, base_path|
+          metadata = Puppet::FileServing::Metadata.new(base_path, relative_path: relative_file_path)
+          metadata.checksum_type = 'sha256'
+          metadata.links = 'follow'
+          metadata.collect
+          metadata.to_data_hash
+        end
+      end
+    end
+
     get '/' do
       200
     end
@@ -550,6 +617,19 @@ module BoltServer
       end
     end
 
+    # Implements puppetserver's file_metadatas endpoint for projects.
+    #
+    # @param project_ref [String] the project_ref to fetch the file metadatas from
+    get '/project_file_metadatas/:module_name/*' do
+      in_bolt_project(params['project_ref']) do |context|
+        file = params[:splat].first
+        metadatas = file_metadatas(context[:pal], params[:module_name], file)
+        [200, metadatas.to_json]
+      end
+    rescue ArgumentError => e
+      [400, e.message]
+    end
+
     error 404 do
       err = Bolt::Error.new("Could not find route #{request.path}",
                             'boltserver/not-found')