openbolt 5.0.0.rc1

data/lib/bolt/project_manager/inventory_migrator.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../bolt/project_manager/migrator'
+
+module Bolt
+  class ProjectManager
+    class InventoryMigrator < Migrator
+      def migrate(inventory_file, backup_dir)
+        inventory1to2(inventory_file, backup_dir)
+      end
+
+      # Migrates an inventory v1 file to inventory v2.
+      #
+      private def inventory1to2(inventory_file, backup_dir)
+        unless File.exist?(inventory_file)
+          return true
+        end
+
+        data = Bolt::Util.read_yaml_hash(inventory_file, 'inventory')
+        data.delete('version') if data['version'] != 2
+        migrated = migrate_group(data)
+
+        return true unless migrated
+
+        @outputter.print_message "Migrating inventory\n\n"
+
+        backup_file(inventory_file, backup_dir)
+
+        begin
+          File.write(inventory_file, data.to_yaml)
+          @outputter.print_action_step(
+            "Successfully migrated Bolt inventory to the latest version."
+          )
+          true
+        rescue StandardError => e
+          raise Bolt::FileError.new(
+            "Unable to write to #{inventory_file}: #{e.message}. See "\
+            "http://pup.pt/bolt-inventory to manually update.",
+            inventory_file
+          )
+        end
+      end
+
+      # Walks an inventory hash and replaces all 'nodes' keys with 'targets'
+      # keys and all 'name' keys nested in a 'targets' hash with 'uri' keys.
+      # Data is modified in place.
+      #
+      private def migrate_group(group)
+        migrated = false
+        if group.key?('nodes')
+          migrated = true
+          targets = group['nodes'].map do |target|
+            target['uri'] = target.delete('name') if target.is_a?(Hash)
+            target
+          end
+          group.delete('nodes')
+          group['targets'] = targets
+        end
+        (group['groups'] || []).each do |subgroup|
+          migrated_group = migrate_group(subgroup)
+          migrated ||= migrated_group
+        end
+        migrated
+      end
+    end
+  end
+end

data/lib/bolt/project_manager/migrator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require_relative '../../bolt/error'
+
+module Bolt
+  class ProjectManager
+    class Migrator
+      def initialize(outputter)
+        @outputter = outputter
+      end
+
+      protected def backup_file(origin_path, backup_dir)
+        unless File.exist?(origin_path)
+          @outputter.print_action_step(
+            "Could not find file #{origin_path}, skipping backup."
+          )
+          return
+        end
+
+        date = Time.new.strftime("%Y%m%d_%H%M%S%L")
+        FileUtils.mkdir_p(backup_dir)
+
+        filename = File.basename(origin_path)
+        backup_path = File.join(backup_dir, "#{filename}.#{date}.bak")
+
+        @outputter.print_action_step(
+          "Backing up #{filename} from #{origin_path} to #{backup_path}"
+        )
+
+        begin
+          FileUtils.cp(origin_path, backup_path)
+        rescue StandardError => e
+          raise Bolt::FileError.new("#{e.message}; unable to create backup of #{filename}.", origin_path)
+        end
+      end
+    end
+  end
+end

data/lib/bolt/project_manager/module_migrator.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require_relative '../../bolt/project_manager/migrator'
+
+module Bolt
+  class ProjectManager
+    class ModuleMigrator < Migrator
+      def migrate(project, configured_modulepath)
+        return true if project.managed_moduledir.exist?
+
+        @outputter.print_message "Migrating project modules\n\n"
+
+        config            = project.project_file
+        puppetfile        = project.puppetfile
+        managed_moduledir = project.managed_moduledir
+        new_modulepath    = [(project.path + 'modules').to_s]
+        old_modulepath    = [(project.path + 'modules').to_s,
+                             (project.path + 'site-modules').to_s,
+                             (project.path + 'site').to_s]
+
+        # Notify user to manually migrate modules if using non-default modulepath
+        if configured_modulepath != new_modulepath && configured_modulepath != old_modulepath
+          @outputter.print_action_step(
+            "Project has a non-default configured modulepath, unable to automatically "\
+            "migrate project modules. To migrate project modules manually, see "\
+            "http://pup.pt/bolt-modules"
+          )
+          true
+        # Migrate modules from Puppetfile
+        elsif File.exist?(puppetfile)
+          migrate_modules_from_puppetfile(config, puppetfile, managed_moduledir, old_modulepath)
+        # Migrate modules to updated modulepath
+        else
+          consolidate_modules(old_modulepath)
+          update_project_config([], config)
+        end
+      end
+
+      # Migrates modules by reading a Puppetfile and prompting the user for
+      # which ones are direct dependencies for the project. Once the user has
+      # selected the direct dependencies, this will resolve the modules, write a
+      # new Puppetfile, install the modules, and then move any remaining modules
+      # to the new moduledir.
+      #
+      private def migrate_modules_from_puppetfile(config, puppetfile_path, managed_moduledir, modulepath)
+        require_relative '../../bolt/module_installer/installer'
+        require_relative '../../bolt/module_installer/puppetfile'
+        require_relative '../../bolt/module_installer/resolver'
+        require_relative '../../bolt/module_installer/specs'
+
+        begin
+          @outputter.print_action_step("Parsing Puppetfile at #{puppetfile_path}")
+          puppetfile = Bolt::ModuleInstaller::Puppetfile.parse(puppetfile_path, skip_unsupported_modules: true)
+        rescue Bolt::Error => e
+          @outputter.print_action_error("#{e.message}\nSkipping module migration.")
+          return false
+        end
+
+        # Prompt for direct dependencies
+        modules = select_modules(puppetfile.modules)
+
+        # Create specs to resolve from
+        specs = Bolt::ModuleInstaller::Specs.new(modules.map(&:to_hash))
+
+        @outputter.start_spin
+        # Attempt to resolve dependencies
+        begin
+          @outputter.print_message('')
+          @outputter.print_action_step("Resolving module dependencies, this might take a moment")
+          puppetfile = Bolt::ModuleInstaller::Resolver.new.resolve(specs)
+        rescue Bolt::Error => e
+          @outputter.print_action_error("#{e.message}\nSkipping module migration.")
+          return false
+        end
+
+        migrate_managed_modules(puppetfile, puppetfile_path, managed_moduledir)
+        @outputter.stop_spin
+
+        # Move remaining modules to 'modules'
+        consolidate_modules(modulepath)
+
+        # Delete old modules that are now managed
+        delete_modules(modulepath.first, puppetfile.modules)
+
+        # Add modules to project
+        update_project_config(modules.map(&:to_hash), config)
+      end
+
+      # Migrates the managed modules. If modules were selected to be managed,
+      # the Puppetfile is rewritten and modules are installed. If no modules
+      # were selected, the Puppetfile is deleted.
+      #
+      private def migrate_managed_modules(puppetfile, puppetfile_path, managed_moduledir)
+        if puppetfile.modules.any?
+          # Show the new Puppetfile content
+          message  = "Generated new Puppetfile content:\n\n"
+          message += puppetfile.modules.map(&:to_spec).join("\n").to_s
+          @outputter.print_action_step(message)
+
+          # Write Puppetfile
+          @outputter.print_action_step("Updating Puppetfile at #{puppetfile_path}")
+          puppetfile.write(puppetfile_path, managed_moduledir)
+
+          # Install Puppetfile
+          @outputter.print_action_step("Syncing modules from #{puppetfile_path} to #{managed_moduledir}")
+          Bolt::ModuleInstaller::Installer.new.install(puppetfile_path, managed_moduledir)
+        else
+          @outputter.print_action_step(
+            "Project does not include any managed modules, deleting Puppetfile "\
+            "at #{puppetfile_path}"
+          )
+          FileUtils.rm(puppetfile_path)
+        end
+      end
+
+      # Prompts the user to select modules, returning a list of
+      # the selected modules.
+      #
+      private def select_modules(modules)
+        @outputter.print_action_step(
+          "Select modules that are direct dependencies of your project. Bolt will "\
+          "automatically manage dependencies for each module selected, so do not "\
+          "select a module's dependencies unless you use content from it directly "\
+          "in your project."
+        )
+
+        all = Bolt::Util.prompt_yes_no("Select all modules?", @outputter)
+        return modules if all
+
+        modules.select do |mod|
+          Bolt::Util.prompt_yes_no("Select #{mod.full_name}?", @outputter)
+        end
+      end
+
+      # Consolidates all modules on the modulepath to 'modules'.
+      #
+      private def consolidate_modules(modulepath)
+        moduledir, *sources = modulepath
+
+        sources.select! { |source| Dir.exist?(source) }
+
+        if sources.any?
+          @outputter.print_action_step(
+            "Moving modules from #{sources.join(', ')} to #{moduledir}"
+          )
+
+          FileUtils.mkdir_p(moduledir)
+          move_modules(moduledir, sources)
+        end
+      end
+
+      # Moves modules from a list of source directories to the specified
+      # moduledir, deleting the source directory after it's done.
+      #
+      private def move_modules(moduledir, sources)
+        moduledir = Pathname.new(moduledir)
+
+        sources.each do |source|
+          source = Pathname.new(source)
+
+          source.each_child do |mod|
+            next unless mod.directory?
+            next if (moduledir + mod.basename).directory?
+            FileUtils.mv(mod, moduledir)
+          end
+
+          FileUtils.rm_r(source)
+        end
+      end
+
+      # Deletes modules from a specified directory.
+      #
+      private def delete_modules(moduledir, modules)
+        @outputter.print_action_step("Cleaning up #{moduledir}")
+        moduledir = Pathname.new(moduledir)
+
+        modules.each do |mod|
+          path = moduledir + mod.name
+          FileUtils.rm_r(path) if path.directory?
+        end
+      end
+
+      # Adds a list of modules to the project configuration file.
+      #
+      private def update_project_config(modules, config_file)
+        @outputter.print_action_step("Updating project configuration at #{config_file}")
+        data = Bolt::Util.read_optional_yaml_hash(config_file, 'project')
+        data.merge!('modules' => modules)
+        data.delete('modulepath')
+
+        begin
+          File.write(config_file, data.to_yaml)
+          true
+        rescue StandardError => e
+          raise Bolt::FileError.new(
+            "Unable to write to #{config_file}: #{e.message}",
+            config_file
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bolt/project_manager.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require_relative 'project_manager/config_migrator'
+require_relative 'project_manager/inventory_migrator'
+require_relative 'project_manager/module_migrator'
+
+module Bolt
+  class ProjectManager
+    INVENTORY_TEMPLATE = <<~INVENTORY
+      # This is an example inventory.yaml
+      # To read more about inventory files, see https://pup.pt/bolt-inventory
+      #
+      # groups:
+      #   - name: linux
+      #     targets:
+      #       - target1.example.com
+      #       - target2.example.com
+      #     config:
+      #       transport: ssh
+      #       ssh:
+      #         private-key: /path/to/private_key.pem
+      #   - name: windows
+      #     targets:
+      #       - name: win1
+      #         uri: target3.example.com
+      #       - name: win2
+      #         uri: target4.example.com
+      #     config:
+      #       transport: winrm
+      # config:
+      #   ssh:
+      #     host-key-check: false
+      #   winrm:
+      #     user: Administrator
+      #     password: Bolt!
+      #     ssl: false
+    INVENTORY
+
+    GITIGNORE_CONTENT = <<~GITIGNORE
+      .modules/
+      .resource_types/
+      bolt-debug.log
+      .plan_cache.json
+      .plugin_cache.json
+      .task_cache.json
+      .rerun.json
+    GITIGNORE
+
+    def initialize(config, outputter, pal)
+      @config    = config
+      @outputter = outputter
+      @pal       = pal
+    end
+
+    # Creates a new project at the specified directory.
+    #
+    def create(path, name, modules)
+      require_relative '../bolt/module_installer'
+
+      project       = Pathname.new(File.expand_path(path))
+      old_config    = project + 'bolt.yaml'
+      config        = project + 'bolt-project.yaml'
+      puppetfile    = project + 'Puppetfile'
+      moduledir     = project + '.modules'
+      inventoryfile = project + 'inventory.yaml'
+      gitignore     = project + '.gitignore'
+      project_name  = name || File.basename(project)
+
+      if config.exist?
+        if modules
+          command = Bolt::Util.powershell? ? 'Add-BoltModule -Module' : 'bolt module add'
+          raise Bolt::Error.new(
+            "Found existing project directory with #{config.basename} at #{project}, "\
+            "unable to initialize project with modules. To add modules to the project, "\
+            "run '#{command} <module>' instead.",
+            'bolt/existing-project-error'
+          )
+        else
+          raise Bolt::Error.new(
+            "Found existing project directory with #{config.basename} at #{project}, "\
+            "unable to initialize project.",
+            'bolt/existing-project-error'
+          )
+        end
+      elsif old_config.exist?
+        command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
+        raise Bolt::Error.new(
+          "Found existing project directory with #{old_config.basename} at #{project}, "\
+          "unable to initialize project. #{old_config.basename} is deprecated. To "\
+          "update the project to current best practices, run '#{command}'.",
+          'bolt/existing-project-error'
+        )
+      elsif modules && puppetfile.exist?
+        raise Bolt::Error.new(
+          "Found existing Puppetfile at #{puppetfile}, unable to initialize project "\
+          "with modules.",
+          'bolt/existing-puppetfile-error'
+        )
+      elsif project_name !~ Bolt::Module::MODULE_NAME_REGEX
+        if name
+          raise Bolt::ValidationError,
+                "The provided project name '#{project_name}' is invalid; project name must "\
+                "begin with a lowercase letter and can include lowercase letters, "\
+                "numbers, and underscores."
+        else
+          command = Bolt::Util.powershell? ? 'New-BoltProject -Name' : 'bolt project init'
+          raise Bolt::ValidationError,
+                "The current directory name '#{project_name}' is an invalid project name. "\
+                "Please specify a name using '#{command} <name>'."
+        end
+      end
+
+      # If modules were specified, resolve and install first. We want to error
+      # early here and not initialize the project if the modules cannot be
+      # resolved and installed.
+      if modules
+        @outputter.start_spin
+        Bolt::ModuleInstaller.new(@outputter, @pal).install(modules, puppetfile, moduledir)
+        @outputter.stop_spin
+      end
+
+      data = { 'name' => project_name }
+      data['modules'] = modules || []
+
+      begin
+        File.write(config.to_path, data.to_yaml)
+      rescue StandardError => e
+        raise Bolt::FileError.new("Could not create bolt-project.yaml at #{project}: #{e.message}", nil)
+      end
+
+      unless inventoryfile.exist?
+        begin
+          File.write(inventoryfile.to_path, INVENTORY_TEMPLATE)
+        rescue StandardError => e
+          raise Bolt::FileError.new("Could not create inventory.yaml at #{project}: #{e.message}", nil)
+        end
+      end
+
+      unless gitignore.exist?
+        begin
+          File.write(gitignore.to_path, GITIGNORE_CONTENT)
+        rescue StandardError => e
+          raise Bolt::FileError.new("Could not create .gitignore at #{project}: #{e.message}", nil)
+        end
+      end
+
+      @outputter.print_message("Successfully created Bolt project at #{project}")
+
+      0
+    end
+
+    # Migrates a project to use the latest file versions and best practices.
+    #
+    def migrate
+      unless $stdin.tty?
+        raise Bolt::Error.new(
+          "stdin is not a tty, unable to migrate project",
+          'bolt/stdin-not-a-tty-error'
+        )
+      end
+
+      @outputter.print_message("Migrating project #{@config.project.path}\n\n")
+
+      @outputter.print_action_step(
+        "Migrating a Bolt project might make irreversible changes to the project's "\
+        "configuration and inventory files. Before continuing, make sure the "\
+        "project has a backup or uses a version control system."
+      )
+
+      return 0 unless Bolt::Util.prompt_yes_no("Continue with project migration?", @outputter)
+
+      @outputter.print_message('')
+
+      ok = migrate_inventory && migrate_config && migrate_modules
+
+      if ok
+        @outputter.print_message("Project successfully migrated")
+      else
+        @outputter.print_error("Project could not be migrated completely")
+      end
+
+      ok ? 0 : 1
+    end
+
+    # Migrates the project-level configuration file to the latest version.
+    #
+    private def migrate_config
+      migrator   = ConfigMigrator.new(@outputter)
+      configfile = @config.project.path + 'bolt.yaml'
+
+      migrator.migrate(
+        configfile,
+        @config.project.project_file,
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the inventory file to the latest version.
+    #
+    private def migrate_inventory
+      migrator = InventoryMigrator.new(@outputter)
+
+      migrator.migrate(
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the project's modules to use current best practices.
+    #
+    private def migrate_modules
+      migrator = ModuleMigrator.new(@outputter)
+
+      migrator.migrate(
+        @config.project,
+        @config.modulepath[0...-1]
+      )
+    end
+  end
+end

data/lib/bolt/puppetdb/client.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'logging'
+require_relative '../../bolt/puppetdb/instance'
+
+module Bolt
+  module PuppetDB
+    class Client
+      # @param config [Hash] A map of default PuppetDB configuration.
+      # @param instances [Hash] A map of configuration for named PuppetDB instances.
+      # @param default [String] The name of PuppetDB instance to use as the default.
+      # @param project [String] The path to the Bolt project.
+      #
+      def initialize(config:, instances: {}, default: nil, project: nil)
+        @logger = Bolt::Logger.logger(self)
+
+        @instances = instances.transform_values do |instance_config|
+          Bolt::PuppetDB::Instance.new(config: instance_config, project: project)
+        end
+
+        @default_instance = if default
+                              validate_instance(default)
+                              @instances[default]
+                            else
+                              Bolt::PuppetDB::Instance.new(config: config, project: project, load_defaults: true)
+                            end
+      end
+
+      # Checks whether a given named instance is configured, erroring if not.
+      #
+      # @param name [String] The name of the PuppetDB instance.
+      #
+      private def validate_instance(name)
+        unless @instances[name]
+          raise Bolt::PuppetDBError, "PuppetDB instance '#{name}' has not been configured, unable to connect"
+        end
+      end
+
+      # Yields the PuppetDB instance to connect to.
+      #
+      # @param name [String] The name of the PuppetDB instance.
+      # @yield [Bolt::PuppetDB::Instance]
+      #
+      private def with_instance(name = nil)
+        yield instance(name)
+      end
+
+      # Selects the PuppetDB instance to connect to. If an instance is not specified,
+      # the default instance is used.
+      #
+      # @param name [String] The name of the PuppetDB instance.
+      # @return [Bolt::PuppetDB::Instance]
+      #
+      def instance(name = nil)
+        if name
+          validate_instance(name)
+          @instances[name]
+        else
+          @default_instance
+        end
+      end
+
+      # Queries certnames from the PuppetDB instance.
+      #
+      # @param query [String] The PDB query.
+      # @param instance [String] The name of the PuppetDB instance.
+      #
+      def query_certnames(query, instance = nil)
+        return [] unless query
+
+        @logger.debug("Querying certnames")
+        results = make_query(query, nil, instance)
+
+        if results&.first && !results.first&.key?('certname')
+          fields = results.first&.keys
+          raise Bolt::PuppetDBError, "Query results did not contain a 'certname' field: got #{fields.join(', ')}"
+        end
+
+        results&.map { |result| result['certname'] }&.uniq
+      end
+
+      # Retrieve facts from PuppetDB for a list of nodes.
+      #
+      # @param certnames [Array] The list of certnames to retrieve facts for.
+      # @param instance [String] The name of the PuppetDB instance.
+      #
+      def facts_for_node(certnames, instance = nil)
+        return {} if certnames.empty? || certnames.nil?
+
+        certnames.uniq!
+        name_query = certnames.map { |c| ["=", "certname", c] }
+        name_query.insert(0, "or")
+
+        @logger.debug("Querying certnames")
+        result = make_query(name_query, 'inventory', instance)
+
+        result&.each_with_object({}) do |node, coll|
+          coll[node['certname']] = node['facts']
+        end
+      end
+
+      # Retrive fact values for a list of nodes.
+      #
+      # @param certnames [Array] The list of certnames to retrieve fact values for.
+      # @param facts [Array] The list of facts to retrive.
+      # @param instance [String] The name of the PuppetDB instance.
+      #
+      def fact_values(certnames = [], facts = [], instance = nil)
+        return {} if certnames.empty? || facts.empty?
+
+        certnames.uniq!
+        name_query = certnames.each_slice(100).map { |slice| ["in", "certname", ["array", slice]] }
+        name_query.unshift("or")
+
+        facts_query = facts.map { |f| ["=", "path", f] }
+        facts_query.unshift("or")
+
+        query = ['and', name_query, facts_query]
+
+        @logger.debug("Querying certnames")
+        result = make_query(query, 'fact-contents', instance)
+        result.map! { |h| h.delete_if { |k, _v| %w[environment name].include?(k) } }
+        result.group_by { |c| c['certname'] }
+      end
+
+      # Sends a command to PuppetDB using the commands API.
+      #
+      # @param command [String] The command to invoke.
+      # @param version [Integer] The version of the command to invoke.
+      # @param payload [Hash] The payload to send with the command.
+      # @param instance [String] The name of the PuppetDB instance.
+      #
+      def send_command(command, version, payload, instance = nil)
+        with_instance(instance) do |pdb|
+          pdb.send_command(command, version, payload)
+        end
+      end
+
+      # Sends a query to PuppetDB.
+      #
+      # @param query [String] The query to send to PuppetDB.
+      # @param path [String] The API path to append to the query URL.
+      # @param instance [String] The name of the PuppetDB instance.
+      #
+      def make_query(query, path = nil, instance = nil)
+        with_instance(instance) do |pdb|
+          pdb.make_query(query, path)
+        end
+      end
+    end
+  end
+end