openbolt 5.0.0.rc1

data/lib/bolt/applicator.rb

@@ -0,0 +1,368 @@
+# frozen_string_literal: true
+
+require 'base64'
+require_relative '../bolt/apply_result'
+require_relative '../bolt/apply_target'
+require_relative '../bolt/config'
+require_relative '../bolt/error'
+require_relative '../bolt/task'
+require_relative '../bolt/util/puppet_log_level'
+require 'find'
+require 'json'
+require 'logging'
+require 'open3'
+
+module Bolt
+  class Applicator
+    def initialize(inventory, executor, modulepath, plugin_dirs, project,
+                   pdb_client, hiera_config, max_compiles, apply_settings)
+      # lazy-load expensive gem code
+      require 'concurrent'
+      @inventory = inventory
+      @executor = executor
+      @modulepath = modulepath || []
+      @plugin_dirs = plugin_dirs
+      @project = project
+      @pdb_client = pdb_client
+      @hiera_config = hiera_config ? validate_hiera_config(hiera_config) : nil
+      @apply_settings = apply_settings || {}
+
+      @pool = Concurrent::ThreadPoolExecutor.new(name: 'apply', max_threads: max_compiles)
+      @logger = Bolt::Logger.logger(self)
+    end
+
+    private def libexec
+      @libexec ||= File.join(Gem::Specification.find_by_name('openbolt').gem_dir, 'libexec')
+    end
+
+    def custom_facts_task
+      @custom_facts_task ||= begin
+        path = File.join(libexec, 'custom_facts.rb')
+        file = { 'name' => 'custom_facts.rb', 'path' => path }
+        metadata = { 'supports_noop' => true, 'input_method' => 'stdin',
+                     'implementations' => [
+                       { 'name' => 'custom_facts.rb' },
+                       { 'name' => 'custom_facts.rb', 'remote' => true }
+                     ] }
+        Bolt::Task.new('apply_helpers::custom_facts', metadata, [file])
+      end
+    end
+
+    def catalog_apply_task
+      @catalog_apply_task ||= begin
+        path = File.join(libexec, 'apply_catalog.rb')
+        file = { 'name' => 'apply_catalog.rb', 'path' => path }
+        metadata = { 'supports_noop' => true, 'input_method' => 'stdin',
+                     'implementations' => [
+                       { 'name' => 'apply_catalog.rb' },
+                       { 'name' => 'apply_catalog.rb', 'remote' => true }
+                     ] }
+        Bolt::Task.new('apply_helpers::apply_catalog', metadata, [file])
+      end
+    end
+
+    def query_resources_task
+      @query_resources_task ||= begin
+        path = File.join(libexec, 'query_resources.rb')
+        file = { 'name' => 'query_resources.rb', 'path' => path }
+        metadata = { 'supports_noop' => true, 'input_method' => 'stdin',
+                     'implementations' => [
+                       { 'name' => 'query_resources.rb' },
+                       { 'name' => 'query_resources.rb', 'remote' => true }
+                     ] }
+
+        Bolt::Task.new('apply_helpers::query_resources', metadata, [file])
+      end
+    end
+
+    def compile(target, scope)
+      # This simplified Puppet node object is what .local uses to determine the
+      # certname of the target
+      node = Puppet::Node.from_data_hash('name' => target.name,
+                                         'parameters' => { 'clientcert' => target.name })
+      trusted = Puppet::Context::TrustedInformation.local(node)
+      target_data = {
+        name: target.name,
+        facts: @inventory.facts(target).merge('bolt' => true),
+        variables: @inventory.vars(target),
+        trusted: trusted.to_h
+      }
+      catalog_request = scope.merge(target: target_data).merge(future: @executor.future || {})
+
+      bolt_catalog_exe = File.join(libexec, 'bolt_catalog')
+      old_path = ENV['PATH']
+      ENV['PATH'] = "#{RbConfig::CONFIG['bindir']}#{File::PATH_SEPARATOR}#{old_path}"
+      out, err, stat = Open3.capture3('ruby', bolt_catalog_exe, 'compile', stdin_data: catalog_request.to_json)
+      ENV['PATH'] = old_path
+
+      # If bolt_catalog does not return valid JSON, we should print stderr to
+      # see what happened
+      print_logs = stat.success?
+      result = begin
+        JSON.parse(out)
+      rescue JSON::ParserError
+        print_logs = true
+        { 'message' => "Something's gone terribly wrong! STDERR is logged." }
+      end
+
+      # Any messages logged by Puppet will be on stderr as JSON hashes, so we
+      # parse those and store them here. Any message on stderr that is not
+      # properly JSON formatted is assumed to be an error message.  If
+      # compilation was successful, we print the logs as they may include
+      # important warnings. If compilation failed, we don't print the logs as
+      # they are likely redundant with the error that caused the failure, which
+      # will be handled separately.
+      logs = err.lines.map do |line|
+        JSON.parse(line)
+      rescue JSON::ParserError
+        { 'level' => 'err', 'message' => line }
+      end
+
+      if print_logs
+        logs.each do |log|
+          bolt_level = Bolt::Util::PuppetLogLevel::MAPPING[log['level'].to_sym]
+          message = log['message'].chomp
+
+          case bolt_level
+          when :warn
+            handle_warning(target, message)
+          else
+            @logger.send(bolt_level, "#{target.name}: #{message}")
+          end
+        end
+      end
+
+      unless stat.success?
+        message = if @apply_settings['trace'] && result['backtrace']
+                    ([result['message']] + result['backtrace']).join("\n  ")
+                  else
+                    result['message']
+                  end
+        raise ApplyError.new(target.name, message)
+      end
+
+      result
+    end
+
+    # Handles logging Puppet warnings, some of which are suppressable.
+    #
+    # @param target [Bolt::Target] The target the apply ran on.
+    # @param message [String] The log message.
+    #
+    private def handle_warning(target, message)
+      # Messages about exported resource declaration and collection, which are
+      # not supported in manifest blocks.
+      if message.include?(Puppet::Pops::Issues::RT_NO_STORECONFIGS_EXPORT.format) ||
+         message.include?(Puppet::Pops::Issues::RT_NO_STORECONFIGS.format)
+        Bolt::Logger.warn('exported_resources', "#{target.name}: #{message}")
+      else
+        @logger.send(:warn, "#{target.name}: #{message}")
+      end
+    end
+
+    def validate_hiera_config(hiera_config)
+      if File.exist?(File.path(hiera_config))
+        data = File.open(File.path(hiera_config), "r:UTF-8") do |f|
+          if Psych.method(:safe_load).parameters.rassoc(:permitted_classes)
+            YAML.safe_load(f.read, permitted_classes: [Symbol])
+          else
+            YAML.safe_load(f.read, [Symbol])
+          end
+        end
+        if data.nil?
+          return nil
+        elsif data['version'] != 5
+          raise Bolt::ParseError, "Hiera v5 is required, found v#{data['version'] || 3} in #{hiera_config}"
+        end
+        hiera_config
+      end
+    end
+
+    def apply(args, apply_body, scope)
+      raise(ArgumentError, 'apply requires a TargetSpec') if args.empty?
+      raise(ArgumentError, 'apply requires at least one statement in the apply block') if apply_body.nil?
+      type0 = Puppet.lookup(:pal_script_compiler).type('TargetSpec')
+      Puppet::Pal.assert_type(type0, args[0], 'apply targets')
+
+      @executor.report_function_call('apply')
+
+      options = {}
+      if args.count > 1
+        type1 = Puppet.lookup(:pal_script_compiler).type('Hash[String, Data]')
+        Puppet::Pal.assert_type(type1, args[1], 'apply options')
+        options = args[1].transform_keys { |k| k.sub(/^_/, '').to_sym }
+      end
+
+      plan_vars = scope.to_hash(true, true)
+
+      targets = @inventory.get_targets(args[0])
+
+      apply_ast(apply_body, targets, options, plan_vars)
+    end
+
+    # Count the number of top-level statements in the AST.
+    def count_statements(ast)
+      case ast
+      when Puppet::Pops::Model::Program
+        count_statements(ast.body)
+      when Puppet::Pops::Model::BlockExpression
+        ast.statements.count
+      else
+        1
+      end
+    end
+
+    def apply_ast(raw_ast, targets, options, plan_vars = {})
+      ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
+      # Serialize as pcore for *Result* objects
+      plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
+                                                                       rich_data: true,
+                                                                       symbol_as_string: true,
+                                                                       type_by_reference: true,
+                                                                       local_reference: true)
+
+      scope = {
+        code_ast: ast,
+        modulepath: @modulepath,
+        project: @project.to_h,
+        pdb_config: @pdb_client.instance(options[:puppetdb]).config.to_hash,
+        hiera_config: @hiera_config,
+        plan_vars: plan_vars,
+        # This data isn't available on the target config hash
+        config: @inventory.transport_data_get
+      }.freeze
+      description = options[:description] || 'apply catalog'
+
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          search_dirs << mod.scripts if mod.scripts?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
+      r = @executor.log_action(description, targets) do
+        futures = targets.map do |target|
+          Concurrent::Future.execute(executor: @pool) do
+            Thread.current[:name] ||= Thread.current.name
+            @executor.with_node_logging("Compiling manifest block", [target], :trace) do
+              compile(target, scope)
+            end
+          end
+        end
+
+        result_promises = targets.zip(futures).flat_map do |target, future|
+          @executor.queue_execute([target]) do |transport, batch|
+            @executor.with_node_logging("Applying manifest block", batch) do
+              catalog = future.value
+              if future.rejected?
+                batch.map do |batch_target|
+                  # If an unhandled exception occurred, wrap it in an ApplyError
+                  error = if future.reason.is_a?(Bolt::ApplyError)
+                            future.reason
+                          else
+                            Bolt::ApplyError.new(batch_target, future.reason.message)
+                          end
+
+                  result = Bolt::ApplyResult.new(batch_target, error: error.to_h)
+                  @executor.publish_event(type: :node_result, result: result)
+                  result
+                end
+              else
+                arguments = {
+                  'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
+                  'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),
+                  'apply_settings' => @apply_settings,
+                  # This should just be boltlib and modules dirs shipped with bolt packages
+                  # The apply_catalog task uses them to load core types if they exist
+                  'bolt_builtin_content' => @modulepath - @plugin_dirs,
+                  '_task' => catalog_apply_task.name,
+                  '_noop' => options[:noop]
+                }
+
+                callback = proc do |event|
+                  if event[:type] == :node_result
+                    event = event.merge(result: ApplyResult.from_task_result(event[:result], catalog))
+                  end
+                  @executor.publish_event(event)
+                end
+                # Respect the run_as default set on the executor
+                options[:run_as] = @executor.run_as if @executor.run_as && !options.key?(:run_as)
+
+                results = transport.batch_task(batch, catalog_apply_task, arguments, options, &callback)
+                Array(results).map { |result| ApplyResult.from_task_result(result, catalog) }
+              end
+            end
+          end
+        end
+
+        @executor.await_results(result_promises)
+      end
+
+      # Allow for report to exclude event metrics (apply_result doesn't require it to be present)
+      resource_counts = r.ok_set.map { |result| result.event_metrics&.fetch('total') }.compact
+      @executor.report_apply(count_statements(raw_ast), resource_counts)
+
+      if !r.ok && !options[:catch_errors]
+        raise Bolt::ApplyFailure, r
+      end
+      r
+    end
+
+    def plugins
+      @plugin_tarball.value ||
+        raise(Bolt::Error.new("Failed to pack module plugins: #{@plugin_tarball.reason}", 'bolt/plugin-error'))
+    end
+
+    def build_plugin_tarball
+      # lazy-load expensive gem code
+      require 'minitar'
+      require 'zlib'
+
+      start_time = Time.now
+      sio = StringIO.new
+      output = Minitar::Output.new(Zlib::GzipWriter.new(sio))
+
+      Puppet.lookup(:current_environment).override_with(modulepath: @plugin_dirs).modules.each do |mod|
+        search_dirs = yield mod
+
+        tar_dir = Pathname.new(mod.name) # goes great with fish
+        mod_dir = Pathname.new(mod.path)
+        files = Find.find(*search_dirs).select { |file| File.file?(file) }
+
+        files.each do |file|
+          tar_path = tar_dir + Pathname.new(file).relative_path_from(mod_dir)
+          @logger.trace("Packing plugin #{file} to #{tar_path}")
+          stat = File.stat(file)
+          content = File.binread(file)
+          output.tar.add_file_simple(
+            tar_path.to_s,
+            data: content,
+            size: content.size,
+            mode: stat.mode & 0o777,
+            mtime: stat.mtime
+          )
+        end
+      end
+
+      duration = Time.now - start_time
+      @logger.trace("Packed plugins in #{duration * 1000} ms")
+
+      output.close
+      Base64.encode64(sio.string)
+    ensure
+      output&.close
+    end
+  end
+end

data/lib/bolt/apply_inventory.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative '../bolt/project'
+require_relative '../bolt/config'
+require_relative '../bolt/error'
+
+module Bolt
+  class ApplyInventory
+    class InvalidFunctionCall < Bolt::Error
+      def initialize(function)
+        super("The function '#{function}' is not callable within an apply block",
+              'bolt.inventory/invalid-function-call')
+      end
+    end
+
+    attr_reader :config_hash
+
+    def initialize(config_hash = {})
+      @config_hash = config_hash
+      @targets = {}
+    end
+
+    def create_apply_target(target)
+      @targets[target.name] = target
+    end
+
+    def validate
+      @groups.validate
+    end
+
+    def version
+      2
+    end
+
+    def target_implementation_class
+      Bolt::ApplyTarget
+    end
+
+    def get_targets(*_params)
+      raise InvalidFunctionCall, 'get_targets'
+    end
+
+    def get_target(*_params)
+      raise InvalidFunctionCall, 'get_target'
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+    def set_var(*_params)
+      raise InvalidFunctionCall, 'set_var'
+    end
+
+    def set_feature(*_params)
+      raise InvalidFunctionCall, 'set_feature'
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
+    def vars(target)
+      @targets[target.name].vars
+    end
+
+    def add_facts(*_params)
+      raise InvalidFunctionCall, 'add_facts'
+    end
+
+    def facts(target)
+      @targets[target.name].facts
+    end
+
+    def features(target)
+      @targets[target.name].features
+    end
+
+    def resource(target, type, title)
+      @targets[target.name].resource(type, title)
+    end
+
+    def add_to_group(*_params)
+      raise InvalidFunctionCall, 'add_to_group'
+    end
+
+    def plugin_hooks(target)
+      @targets[target.name].plugin_hooks
+    end
+
+    def set_config(_target, _key_or_key_path, _value)
+      raise InvalidFunctionCall, 'set_config'
+    end
+
+    def target_config(target)
+      @targets[target.name].config
+    end
+  end
+end

data/lib/bolt/apply_result.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../bolt/error'
+require_relative '../bolt/result'
+
+module Bolt
+  class ApplyResult < Result
+    def self.puppet_missing_error(result)
+      error_hash = result.error_hash
+      exit_code = error_hash['details']['exit_code'] if error_hash && error_hash['details']
+      # If we get exit code 126 or 127 back, it means the shebang command wasn't found; Puppet isn't present
+      if [126, 127].include?(exit_code)
+        {
+          'msg' => "Puppet is not installed on the target, please install it to enable 'apply'",
+          'kind' => 'bolt/apply-error'
+        }
+      elsif exit_code == 1 &&
+            (error_hash['msg'] =~ /Could not find executable 'ruby.exe'/ ||
+             error_hash['msg'] =~ /The term 'ruby.exe' is not recognized as the name of a cmdlet/)
+        # Windows does not have Ruby present
+        {
+          'msg' => "Puppet was not found on the target or in $env:ProgramFiles, please install it to enable 'apply'",
+          'kind' => 'bolt/apply-error'
+        }
+      elsif exit_code == 1 && error_hash['msg'] =~ /cannot load such file -- puppet \(LoadError\)/
+        # Windows uses a Ruby that doesn't have Puppet installed
+        # TODO: fix so we don't find other Rubies, or point to a known issues URL for more info
+        { 'msg' => 'Found a Ruby without Puppet present, please install Puppet ' \
+                   "or remove Ruby from $env:Path to enable 'apply'",
+          'kind' => 'bolt/apply-error' }
+      end
+    end
+
+    def self.resource_error(result)
+      if result.value['status'] == 'failed'
+        resources = result.value['resource_statuses']
+        failed = resources.select { |_, r| r['failed'] }.flat_map do |key, resource|
+          resource['events'].select { |e| e['status'] == 'failure' }.map do |event|
+            "\n  #{key}: #{event['message']}"
+          end
+        end
+
+        { 'msg' => "Resources failed to apply for #{result.target.name}#{failed.join}",
+          'kind' => 'bolt/resource-failure' }
+      end
+    end
+
+    def self.invalid_report_error(result)
+      # These are the keys ApplyResult methods rely on.
+      expected_report_keys = %w[metrics resource_statuses status]
+      missing_keys = expected_report_keys.reject { |k| result.value.include?(k) }
+
+      unless missing_keys.empty?
+        if result['_output']
+          # rubocop:disable Layout/LineLength
+          msg = "Report result contains an '_output' key. Catalog application might have printed extraneous output to stdout: #{result['_output']}"
+          # rubocop:enable Layout/LineLength
+        else
+          msg = "Report did not contain all expected keys missing: #{missing_keys.join(', ')}"
+        end
+
+        { 'msg' => msg,
+          'kind' => 'bolt/invalid-report' }
+      end
+    end
+
+    def self.from_task_result(result, catalog = nil)
+      if (puppet_missing = puppet_missing_error(result))
+        new(result.target,
+            error: puppet_missing,
+            report: result.value.reject { |k| k == '_error' },
+            catalog: catalog)
+      elsif !result.ok?
+        new(result.target,
+            error: result.error_hash,
+            catalog: catalog)
+      elsif (invalid_report = invalid_report_error(result))
+        new(result.target,
+            error: invalid_report,
+            report: result.value.reject { |k| %w[_error _output].include?(k) },
+            catalog: catalog)
+      elsif (resource_error = resource_error(result))
+        new(result.target,
+            error: resource_error,
+            report: result.value.reject { |k| k == '_error' },
+            catalog: catalog)
+      else
+        new(result.target,
+            report: result.value,
+            catalog: catalog)
+      end
+    end
+
+    # Other pcore methods are inherited from Result
+    def _pcore_init_hash
+      { 'target' => @target,
+        'error' => value['_error'],
+        'report' => value['report'],
+        'catalog' => catalog }
+    end
+
+    def initialize(target, error: nil, report: nil, catalog: nil)
+      @target = target
+      @value = {}
+      @action = 'apply'
+      @value['report'] = report if report
+      @value['_error'] = error if error
+      @value['_output'] = metrics_message if metrics_message
+
+      if catalog
+        @value['_sensitive'] = Puppet::Pops::Types::PSensitiveType::Sensitive.new({ 'catalog' => catalog })
+      end
+    end
+
+    def event_metrics
+      if (events = value.dig('report', 'metrics', 'resources', 'values'))
+        events.each_with_object({}) { |ev, h| h[ev[0]] = ev[2] }
+      end
+    end
+
+    def logs
+      value.dig('report', 'logs') || []
+    end
+
+    # Return only log messages associated with resources
+    def resource_logs
+      logs.reject { |log| log['source'] == 'Puppet' }
+    end
+
+    def metrics_message
+      if (metrics = event_metrics)
+        changed = metrics['changed']
+        failed = metrics['failed']
+        skipped = metrics['skipped']
+        unchanged = metrics['total'] - changed - failed - skipped
+        noop = metrics['out_of_sync'] - changed - failed
+        "changed: #{changed}, failed: #{failed}, unchanged: #{unchanged} skipped: #{skipped}, noop: #{noop}"
+      end
+    end
+
+    def report
+      @value['report']
+    end
+
+    def catalog
+      sensitive.unwrap['catalog'] if sensitive
+    end
+
+    def generic_value
+      {}
+    end
+  end
+end