openbolt 5.0.0.rc1

data/lib/bolt/config/transport/winrm.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../../../bolt/error'
+require_relative '../../../bolt/config/transport/base'
+
+module Bolt
+  class Config
+    module Transport
+      class WinRM < Base
+        OPTIONS = %w[
+          basic-auth-only
+          cacert
+          cleanup
+          connect-timeout
+          extensions
+          file-protocol
+          host
+          interpreters
+          password
+          port
+          realm
+          smb-port
+          ssl
+          ssl-verify
+          tmpdir
+          user
+        ].freeze
+
+        DEFAULTS = {
+          "basic-auth-only" => false,
+          "cleanup"         => true,
+          "connect-timeout" => 10,
+          "ssl"             => true,
+          "ssl-verify"      => true,
+          "file-protocol"   => "winrm"
+        }.freeze
+
+        private def validate
+          super
+
+          if @config['ssl']
+            if @config['file-protocol'] == 'smb'
+              raise Bolt::ValidationError, "SMB file transfers are not allowed with SSL enabled"
+            end
+
+            if @config['cacert']
+              @config['cacert'] = File.expand_path(@config['cacert'], @project)
+              Bolt::Util.validate_file('cacert', @config['cacert'])
+            end
+          end
+
+          if !@config['ssl'] && @config['basic-auth-only']
+            raise Bolt::ValidationError, "Basic auth is only allowed when using SSL"
+          end
+
+          if @config['interpreters']
+            @config['interpreters'] = normalize_interpreters(@config['interpreters'])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/config.rb

@@ -0,0 +1,515 @@
+# frozen_string_literal: true
+
+require 'etc'
+require 'logging'
+require 'pathname'
+require_relative '../bolt/project'
+require_relative '../bolt/logger'
+require_relative '../bolt/util'
+require_relative 'config/options'
+require_relative '../bolt/validator'
+
+module Bolt
+  class UnknownTransportError < Bolt::Error
+    def initialize(transport, uri = nil)
+      msg = uri.nil? ? "Unknown transport #{transport}" : "Unknown transport #{transport} found for #{uri}"
+      super(msg, 'bolt/unknown-transport')
+    end
+  end
+
+  class Config
+    include Bolt::Config::Options
+
+    attr_reader :config_files, :data, :transports, :project, :modified_concurrency
+
+    DEFAULTS_NAME = 'bolt-defaults.yaml'
+
+    # The default concurrency value that is used when the ulimit is not low (i.e. < 700)
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
+    def self.default
+      new(Bolt::Project.default_project, {})
+    end
+
+    def self.from_project(project, overrides = {})
+      data = load_defaults.push(
+        filepath: project.project_file,
+        data: project.data
+      )
+
+      new(project, data, overrides)
+    end
+
+    # Builds a hash of definitions for transport configuration.
+    #
+    def self.transport_definitions
+      INVENTORY_OPTIONS.each_with_object({}) do |(option, definition), acc|
+        acc[option] = TRANSPORT_CONFIG.key?(option) ? definition.merge(TRANSPORT_CONFIG[option].schema) : definition
+      end
+    end
+
+    # Builds the schema for bolt-defaults.yaml used by the validator.
+    #
+    def self.defaults_schema
+      schema = {
+        type:        Hash,
+        properties:  DEFAULTS_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: OPTIONS.merge(transport_definitions)
+      }
+
+      schema[:definitions]['inventory-config'][:properties] = transport_definitions
+
+      schema
+    end
+
+    def self.system_path
+      if Bolt::Util.windows?
+        Pathname.new(File.join(ENV['ALLUSERSPROFILE'], 'PuppetLabs', 'bolt', 'etc'))
+      else
+        Pathname.new(File.join('/etc', 'puppetlabs', 'bolt'))
+      end
+    end
+
+    def self.user_path
+      Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt')))
+    rescue StandardError
+      nil
+    end
+
+    # Loads a 'bolt-defaults.yaml' file, which contains default configuration that applies to all
+    # projects. This file does not allow project-specific configuration such as 'hiera-config'
+    # and nests all default inventory configuration under an 'inventory-config' key.
+    def self.load_bolt_defaults_yaml(dir)
+      filepath     = dir + DEFAULTS_NAME
+      data         = Bolt::Util.read_yaml_hash(filepath, 'config')
+
+      Bolt::Logger.debug("Loaded configuration from #{filepath}")
+
+      # Validate the config against the schema. This will raise a single error
+      # with all validation errors.
+      Bolt::Validator.new.tap do |validator|
+        validator.validate(data, defaults_schema, filepath)
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
+        validator.deprecations.each { |dep| Bolt::Logger.deprecate(dep[:id], dep[:msg]) }
+      end
+
+      # Remove project-specific config such as hiera-config, etc.
+      project_config = data.slice(*(PROJECT_OPTIONS - DEFAULTS_OPTIONS))
+
+      if project_config.any?
+        data.reject! { |key, _| project_config.include?(key) }
+
+        Bolt::Logger.warn(
+          "unsupported_project_config",
+          "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
+          "Project configuration should be set in 'bolt-project.yaml'."
+        )
+      end
+
+      # Remove top-level transport config such as transport, ssh, etc.
+      transport_config = data.slice(*INVENTORY_OPTIONS.keys)
+
+      if transport_config.any?
+        data.reject! { |key, _| transport_config.include?(key) }
+
+        Bolt::Logger.warn(
+          "unsupported_inventory_config",
+          "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
+          "Transport configuration should be set under the 'inventory-config' option or "\
+          "in 'inventory.yaml'."
+        )
+      end
+
+      # Move data under inventory-config to top-level so it can be easily merged with
+      # config from other sources. Error early if inventory-config is not a hash or
+      # has a plugin reference.
+      if data.key?('inventory-config')
+        unless data['inventory-config'].is_a?(Hash)
+          raise Bolt::ValidationError,
+                "Option 'inventory-config' must be of type Hash, received #{data['inventory-config']} "\
+                "#{data['inventory-config']} (file: #{filepath})"
+        end
+
+        if data['inventory-config'].key?('_plugin')
+          raise Bolt::ValidationError,
+                "Found unsupported key '_plugin' for option 'inventory-config'; supported keys are "\
+                "'#{INVENTORY_OPTIONS.keys.join("', '")}' (file: #{filepath})"
+        end
+
+        data = data.merge(data.delete('inventory-config'))
+      end
+
+      { filepath: filepath, data: data }
+    end
+
+    def self.load_defaults
+      confs = []
+
+      # Load system-level config.
+      if File.exist?(system_path + DEFAULTS_NAME)
+        confs << load_bolt_defaults_yaml(system_path)
+      end
+
+      # Load user-level config if there is a homedir.
+      if user_path && File.exist?(user_path + DEFAULTS_NAME)
+        confs << load_bolt_defaults_yaml(user_path)
+      end
+
+      confs
+    end
+
+    def initialize(project, config_data, overrides = {})
+      unless config_data.is_a?(Array)
+        config_data = [{ filepath: project.project_file, data: config_data }]
+      end
+
+      @logger       = Bolt::Logger.logger(self)
+      @project      = project
+      @transports   = {}
+      @config_files = []
+
+      default_data = {
+        'analytics'           => false,
+        'apply-settings'      => {},
+        'color'               => true,
+        'compile-concurrency' => Etc.nprocessors,
+        'concurrency'         => default_concurrency,
+        'disable-warnings'    => [],
+        'format'              => 'human',
+        'log'                 => { 'console' => {} },
+        'module-install'      => {},
+        'plugin-hooks'        => {},
+        'plugins'             => {},
+        'puppetdb'            => {},
+        'puppetdb-instances'  => {},
+        'save-rerun'          => true,
+        'spinner'             => true,
+        'transport'           => 'ssh'
+      }
+
+      if project.path.directory?
+        default_data['log']['bolt-debug.log'] = {
+          'level' => 'debug',
+          'append' => false
+        }
+      end
+
+      loaded_data = config_data.each_with_object([]) do |data, acc|
+        if data[:data].any?
+          @config_files.push(data[:filepath])
+          acc.push(data[:data])
+        end
+      end
+
+      override_data = normalize_overrides(overrides)
+
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      @modified_concurrency = default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+                              !ld_concurrency &&
+                              !override_data.key?('concurrency')
+
+      @data = merge_config_layers(default_data, *loaded_data, override_data)
+
+      TRANSPORT_CONFIG.each do |transport, config|
+        @transports[transport] = config.new(@data.delete(transport), @project.path)
+      end
+
+      finalize_data
+      validate
+    end
+
+    # Transforms CLI options into a config hash that can be merged with
+    # default and loaded config.
+    def normalize_overrides(options)
+      opts = options.transform_keys(&:to_s)
+
+      # Pull out config options. We need to add 'transport' and 'inventoryfile' as they're
+      # not part of the OPTIONS hash but are valid options that can be set with CLI options
+      overrides = opts.slice(*OPTIONS.keys, 'inventoryfile', 'transport', 'default_puppetdb')
+
+      # Pull out transport config options
+      TRANSPORT_CONFIG.each do |transport, config|
+        overrides[transport] = opts.slice(*config.options)
+      end
+
+      overrides['trace'] = opts['trace'] if opts.key?('trace')
+
+      # Validate the overrides that can have arbitrary values
+      schema = {
+        type:        Hash,
+        properties:  CLI_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: OPTIONS.merge(INVENTORY_OPTIONS)
+      }
+
+      Bolt::Validator.new.validate(overrides.slice(*CLI_OPTIONS), schema, 'command line')
+
+      overrides
+    end
+
+    # Merge configuration from all sources into a single hash. Precedence from lowest to highest:
+    # defaults, system-wide, user-level, project-level, CLI overrides
+    def merge_config_layers(*config_data)
+      config_data.inject({}) do |acc, config|
+        acc.merge(config) do |key, val1, val2|
+          case key
+          # Shallow merge config for each plugin
+          when 'plugins'
+            val1.merge(val2) { |_, v1, v2| v1.merge(v2) }
+          # Transports are deep merged
+          when *TRANSPORT_CONFIG.keys
+            Bolt::Util.deep_merge(val1, val2)
+          # Hash values are shallow merged
+          when 'apply-settings', 'log', 'plugin-hooks', 'puppetdb', 'puppetdb-instances'
+            val1.merge(val2)
+          # Disabled warnings are concatenated
+          when 'disable-warnings'
+            val1.concat(val2)
+          when 'analytics'
+            val1 && val2
+          # All other values are overwritten
+          else
+            val2
+          end
+        end
+      end
+    end
+
+    def deep_clone
+      Bolt::Util.deep_clone(self)
+    end
+
+    private def finalize_data
+      if @data['log'].is_a?(Hash)
+        @data['log'] = update_logs(@data['log'])
+      end
+
+      # Expand paths relative to the project. Any settings that came from the
+      # CLI will already be absolute, so the expand will be skipped.
+      if @data.key?('modulepath')
+        moduledirs = if data['modulepath'].is_a?(String)
+                       data['modulepath'].split(File::PATH_SEPARATOR)
+                     else
+                       data['modulepath']
+                     end
+        @data['modulepath'] = moduledirs.map do |moduledir|
+          File.expand_path(moduledir, @project.path)
+        end
+      end
+
+      %w[hiera-config inventoryfile trusted-external-command].each do |opt|
+        @data[opt] = File.expand_path(@data[opt], @project.path) if @data.key?(opt)
+      end
+
+      # Filter hashes to only include valid options
+      %w[apply-settings module-install].each do |opt|
+        @data[opt] = @data[opt].slice(*OPTIONS.dig(opt, :properties).keys)
+      end
+    end
+
+    private def normalize_log(target)
+      return target if target == 'console'
+      target = target[5..-1] if target.start_with?('file:')
+      'file:' + File.expand_path(target, @project.path)
+    end
+
+    private def update_logs(logs)
+      begin
+        if logs['bolt-debug.log'] && logs['bolt-debug.log'] != 'disable'
+          FileUtils.touch(File.expand_path('bolt-debug.log', @project.path))
+        end
+      rescue StandardError
+        logs.delete('bolt-debug.log')
+      end
+
+      logs.each_with_object({}) do |(key, val), acc|
+        # Remove any disabled logs
+        next if val == 'disable'
+
+        name = normalize_log(key)
+        acc[name] = val.slice('append', 'level').transform_keys(&:to_sym)
+      end
+    end
+
+    def validate
+      if @data['modulepath']&.include?(@project.managed_moduledir.to_s)
+        raise Bolt::ValidationError,
+              "Found invalid path in modulepath: #{@project.managed_moduledir}. This path "\
+              "is automatically appended to the modulepath and cannot be configured."
+      end
+
+      compile_limit = 2 * Etc.nprocessors
+      unless compile_concurrency < compile_limit
+        raise Bolt::ValidationError, "Compilation is CPU-intensive, set concurrency less than #{compile_limit}"
+      end
+
+      %w[hiera-config trusted-external-command inventoryfile].each do |opt|
+        Bolt::Util.validate_file(opt, @data[opt]) if @data[opt]
+      end
+
+      if File.exist?(default_inventoryfile)
+        Bolt::Util.validate_file('inventory file', default_inventoryfile)
+      end
+    end
+
+    def default_inventoryfile
+      @project.inventory_file
+    end
+
+    def rerunfile
+      @project.rerunfile
+    end
+
+    def hiera_config
+      @data['hiera-config'] || @project.hiera_config
+    end
+
+    def puppetfile
+      @project.puppetfile
+    end
+
+    def modulepath
+      (@data['modulepath'] || @project.modulepath) + [@project.managed_moduledir.to_s]
+    end
+
+    def modulepath=(value)
+      @data['modulepath'] = Array(value)
+    end
+
+    def plugin_cache
+      @project.plugin_cache || @data['plugin-cache'] || {}
+    end
+
+    def concurrency
+      @data['concurrency']
+    end
+
+    def format
+      @data['format']
+    end
+
+    def format=(value)
+      @data['format'] = value
+    end
+
+    def future
+      @data['future']
+    end
+
+    def trace
+      @data['trace']
+    end
+
+    def log
+      @data['log']
+    end
+
+    def puppetdb
+      @data['puppetdb']
+    end
+
+    def puppetdb_instances
+      @data['puppetdb-instances']
+    end
+
+    def default_puppetdb
+      @data['default_puppetdb']
+    end
+
+    def color
+      @data['color']
+    end
+
+    def save_rerun
+      @data['save-rerun']
+    end
+
+    def spinner
+      @data['spinner']
+    end
+
+    def stream
+      @data['stream']
+    end
+
+    def inventoryfile
+      @data['inventoryfile']
+    end
+
+    def compile_concurrency
+      @data['compile-concurrency']
+    end
+
+    def plugins
+      @data['plugins']
+    end
+
+    def plugin_hooks
+      @data['plugin-hooks']
+    end
+
+    def policies
+      @data['policies']
+    end
+
+    def trusted_external
+      @data['trusted-external-command']
+    end
+
+    def apply_settings
+      @data['apply-settings']
+    end
+
+    def transport
+      @data['transport']
+    end
+
+    def module_install
+      @project.module_install || @data['module-install']
+    end
+
+    def disable_warnings
+      Set.new(@project.disable_warnings + @data['disable-warnings'])
+    end
+
+    def analytics
+      @data['analytics']
+    end
+
+    # Check if there is a case-insensitive match to the path
+    def check_path_case(type, paths)
+      return if paths.nil?
+      matches = matching_paths(paths)
+
+      if matches.any?
+        msg = "WARNING: Bolt is case sensitive when specifying a #{type}. Did you mean:\n"
+        matches.each { |path| msg += "         #{path}\n" }
+        Bolt::Logger.warn("path_case", msg)
+      end
+    end
+
+    def matching_paths(paths)
+      Array(paths).map { |p| Dir.glob([p, casefold(p)]) }.flatten.uniq.reject { |p| Array(paths).include?(p) }
+    end
+
+    private def casefold(path)
+      path.chars.map do |l|
+        l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
+      end.join
+    end
+
+    # Etc::SC_OPEN_MAX is meaningless on windows, not defined in PE Jruby and not available
+    # on some platforms. This method holds the logic to decide whether or not to even consider it.
+    def sc_open_max_available?
+      !Bolt::Util.windows? && defined?(Etc::SC_OPEN_MAX) && Etc.sysconf(Etc::SC_OPEN_MAX)
+    end
+
+    def default_concurrency
+      @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
+                                 DEFAULT_DEFAULT_CONCURRENCY
+                               else
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 7
+                               end
+    end
+  end
+end

data/lib/bolt/container_result.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../bolt/error'
+require_relative '../bolt/result'
+
+module Bolt
+  class ContainerResult
+    attr_reader :value, :object
+
+    def self.from_exception(exception, exit_code, image, position: [])
+      details = Bolt::Result.create_details(position)
+      error = {
+        'kind' => 'puppetlabs.tasks/container-error',
+        'issue_code' => 'CONTAINER_ERROR',
+        'msg' => "Error running container '#{image}': #{exception}",
+        'details' => details
+      }
+      error['details']['exit_code'] = exit_code
+      ContainerResult.new({ '_error' => error }, object: image)
+    end
+
+    def _pcore_init_hash
+      { 'value' => @value,
+        'object' => @image }
+    end
+
+    # First argument can't be named given the way that Puppet deserializes variables
+    def initialize(value = nil, object: nil)
+      @value = value || {}
+      @object = object
+    end
+
+    def eql?(other)
+      self.class == other.class &&
+        value == other.value
+    end
+    alias == eql?
+
+    def [](key)
+      value[key]
+    end
+
+    def to_json(opts = nil)
+      to_data.to_json(opts)
+    end
+    alias to_s to_json
+
+    # This is the value with all non-UTF-8 characters removed, suitable for
+    # printing or converting to JSON. It *should* only be possible to have
+    # non-UTF-8 characters in stdout/stderr keys as they are not allowed from
+    # tasks but we scrub the whole thing just in case.
+    def safe_value
+      Bolt::Util.walk_vals(value) do |val|
+        if val.is_a?(String)
+          # Replace invalid bytes with hex codes, ie. \xDE\xAD\xBE\xEF
+          val.scrub { |c| c.bytes.map { |b| "\\x" + b.to_s(16).upcase }.join }
+        else
+          val
+        end
+      end
+    end
+
+    def stdout
+      value['stdout']
+    end
+
+    def stderr
+      value['stderr']
+    end
+
+    def to_data
+      {
+        "object" => object,
+        "status" => status,
+        "value" => safe_value
+      }
+    end
+
+    def status
+      ok? ? 'success' : 'failure'
+    end
+
+    def ok?
+      error_hash.nil?
+    end
+    alias ok ok?
+    alias success? ok?
+
+    # This allows access to errors outside puppet compilation
+    # it should be prefered over error in bolt code
+    def error_hash
+      value['_error']
+    end
+
+    # Warning: This will fail outside of a compilation.
+    # Use error_hash inside bolt.
+    # Is it crazy for this to behave differently outside a compiler?
+    def error
+      if error_hash
+        Puppet::DataTypes::Error.from_asserted_hash(error_hash)
+      end
+    end
+  end
+end