bolt 1.34.0 → 1.35.0

data/lib/bolt/module.rb

@@ -8,10 +8,10 @@ module Bolt
     def self.discover(modulepath)
       modulepath.each_with_object({}) do |path, mods|
         next unless File.exist?(path) && File.directory?(path)
-        Dir.children(path)
-           .map { |dir| File.join(path, dir) }
-           .select { |dir| File.directory?(dir) }
-           .each do |dir|
+        (Dir.entries(path) - %w[. ..])
+          .map { |dir| File.join(path, dir) }
+          .select { |dir| File.directory?(dir) }
+          .each do |dir|
           module_name = File.basename(dir)
           if module_name =~ MODULE_NAME_REGEX
             # Puppet will load some objects from shadowed modules but this won't

data/lib/bolt/plugin.rb

@@ -123,13 +123,10 @@ module Bolt
       # PDB is special do we want to expose the default client to the context?
       plugins.add_plugin(Bolt::Plugin::Puppetdb.new(pdb_client))
 
-      plugins.add_ruby_plugin('Bolt::Plugin::AwsInventory')
       plugins.add_ruby_plugin('Bolt::Plugin::InstallAgent')
       plugins.add_ruby_plugin('Bolt::Plugin::Task')
-      plugins.add_ruby_plugin('Bolt::Plugin::Terraform')
       plugins.add_ruby_plugin('Bolt::Plugin::Pkcs7')
       plugins.add_ruby_plugin('Bolt::Plugin::Prompt')
-      plugins.add_ruby_plugin('Bolt::Plugin::Vault')
 
       plugins
     end

data/lib/bolt/plugin/module.rb

@@ -35,9 +35,18 @@ module Bolt
       # This method interacts with the module on disk so it's separate from initialize
       def setup
         @data = load_data
-        @config_schema = process_schema(@data['config'] || {})
-
         @hook_map = find_hooks(@data['hooks'] || {})
+        # If there is a config section in bolt_plugin.json, validate against that and send
+        # validated values nested under `_config` key. Otherwise validate againsts the intersection
+        # of all task schemas.
+        # TODO: remove @send_config when deprecated
+        schema = if @data['config']
+                   @send_config = true
+                   @data['config']
+                 else
+                   extract_task_parameter_schema
+                 end
+        @config_schema = process_schema(schema)
 
         validate_config(@config, @config_schema)
       end
@@ -150,15 +159,45 @@ module Bolt
         # handled previously. That may not always be the case so filter them
         # out now.
         _meta, params = opts.partition { |key, _val| key.start_with?('_') }.map(&:to_h)
-
         metaparams = {}
-        metaparams['_config'] = config if config?
+        # Send config with `_config` when config is defined in bolt_plugin.json
+        # Otherwise, merge config with params
+        # TODO: remove @send_config when deprecated
+        if @send_config
+          metaparams['_config'] = config if config?
+        else
+          params = @config ? config.merge(params) : params
+        end
         metaparams['_boltdir'] = @context.boltdir
 
         validate_params(task, params)
         [params, metaparams]
       end
 
+      def extract_task_parameter_schema
+        # Get the intersection of expected types (using Set)
+        type_set = @hook_map.each_with_object({}) do |(_hook, task), acc|
+          next unless (schema = task['task'].metadata['parameters'])
+          schema.each do |param, scheme|
+            next unless scheme['type'].is_a?(String)
+            scheme['type'] = Set.new([scheme['type']])
+            if acc.dig(param, 'type').is_a?(Set)
+              scheme['type'].merge(acc[param]['type'])
+            end
+          end
+          acc.merge!(schema)
+        end
+        # Convert Set to string
+        type_set.each do |_param, schema|
+          next unless schema['type']
+          schema['type'] = if schema['type'].size > 1
+                             "Optional[Variant[#{schema['type'].to_a.join(', ')}]]"
+                           else
+                             "Optional[#{schema['type'].to_a.first}]"
+                           end
+        end
+      end
+
       def run_task(task, opts)
         params, metaparams = process_params(task, opts)
         params = params.merge(metaparams)
@@ -191,7 +230,15 @@ module Bolt
       end
 
       def validate_resolve_reference(opts)
-        params = opts.reject { |k, _v| k.start_with?('_') }
+        # Send config with `_config` when config is defined in bolt_plugin.json
+        # Otherwise, merge config with params
+        # TODO: remove @send_config when deprecated
+        if @send_config
+          params = opts.reject { |k, _v| k.start_with?('_') }
+        else
+          merged = @config.merge(opts)
+          params = merged.reject { |k, _v| k.start_with?('_') }
+        end
         sig = @hook_map[:resolve_reference]['task']
         if sig
           validate_params(sig, params)

data/lib/bolt/plugin/task.rb

@@ -48,7 +48,11 @@ module Bolt
 
       def puppet_library(opts, target, apply_prep)
         params = opts['parameters'] || {}
-        task = apply_prep.get_task(opts['task'], params)
+        begin
+          task = apply_prep.get_task(opts['task'], params)
+        rescue Bolt::Error => e
+          raise Bolt::Plugin::PluginError::ExecutionError.new(e.message, name, 'puppet_library')
+        end
         proc do
           apply_prep.run_task([target], task, params).first
         end

data/lib/bolt/target.rb

@@ -10,10 +10,12 @@ module Bolt
     # Target.new from a plan initialized with a hash
     def self.from_asserted_hash(hash)
       inventory = Puppet.lookup(:bolt_inventory)
-      inventory.create_target_from_plan(hash)
+      target = inventory.create_target_from_plan(hash)
+      new(target.name, inventory)
     end
 
-    # Target.new from a plan with just a uri
+    # Target.new from a plan with just a uri. Puppet requires the arguments to
+    # this method to match (by name) the attributes defined on the datatype.
     # rubocop:disable Lint/UnusedMethodArgument
     def self.from_asserted_args(uri = nil,
                                 name = nil,
@@ -23,37 +25,15 @@ module Bolt
                                 vars = nil,
                                 features = nil,
                                 plugin_hooks = nil)
-      inventory = Puppet.lookup(:bolt_inventory)
-      inventory.create_target_from_plan('uri' => uri)
+      from_asserted_hash('uri' => uri)
     end
 
-    # URI can be passes as nil
-    def initialize(uri = nil,
-                   name = nil,
-                   target_alias = nil,
-                   config = nil,
-                   facts = nil,
-                   vars = nil,
-                   features = nil,
-                   plugin_hooks = nil)
+    def initialize(name, inventory = nil)
       @name = name
+      @inventory = inventory
     end
     # rubocop:enable Lint/UnusedMethodArgument
 
-    # Used for munging target + group data
-    def target_data_hash
-      {
-        'config' => @inventory.targets[@name]['config'],
-        'vars' => @inventory.targets[@name]['vars'],
-        'facts' => @inventory.targets[@name]['facts'],
-        'features' => @inventory.targets[@name]['features'].to_a,
-        'plugin_hooks' => @inventory.targets[@name]['plugin_hooks'],
-        'name' => @inventory.targets[@name]['name'],
-        'uri' => @inventory.targets[@name]['uri'],
-        'alias' => @inventory.targets[@name]['target_alias']
-      }
-    end
-
     # features returns an array to be compatible with plans
     def features
       @inventory.features(self).to_a
@@ -77,15 +57,15 @@ module Bolt
     end
 
     def config
-      @inventory.target_config(self)
+      inventory_target.config
     end
 
     def safe_name
-      @inventory.targets[@name]['safe_name']
+      inventory_target.safe_name
     end
 
     def target_alias
-      @inventory.targets[@name]['target_alias']
+      inventory_target.target_alias
     end
 
     def to_h
@@ -100,53 +80,51 @@ module Bolt
       )
     end
 
+    def inventory_target
+      @inventory.targets[@name]
+    end
+
     def host
-      @inventory.targets[@name]['uri_obj']&.hostname || @inventory.targets[@name]['host']
+      inventory_target.host
     end
 
     attr_reader :name
 
     def uri
-      @inventory.targets[@name]['uri']
+      inventory_target.uri
     end
 
     def remote?
-      @inventory.targets[@name]['uri_obj']&.scheme == 'remote' || @inventory.targets[@name]['protocol'] == 'remote'
+      protocol == 'remote'
     end
 
     def port
-      @inventory.targets[@name]['uri_obj']&.port || @inventory.targets[@name]['port']
+      inventory_target.port
     end
 
-    # transport is separate from protocol for remote targets.
     def transport
-      remote? ? 'remote' : protocol
+      inventory_target.protocol
     end
 
     def protocol
-      @inventory.targets[@name]['uri_obj']&.scheme || @inventory.targets[@name]['protocol']
+      inventory_target.protocol
     end
 
     def user
-      unencode(@inventory.targets[@name]['uri_obj']&.user) || @inventory.targets[@name]['user']
+      inventory_target.user
     end
 
     def password
-      unencode(@inventory.targets[@name]['uri_obj']&.password) || @inventory.targets[@name]['password']
+      inventory_target.password
     end
 
     def options
-      @inventory.targets[@name]['options']
+      inventory_target.options
     end
 
     def plugin_hooks
-      @inventory.targets[@name]['cached_state']['plugin_hooks']
-    end
-
-    def unencode(component)
-      Addressable::URI.unencode_component(component)
+      inventory_target.plugin_hooks
     end
-    private :unencode
 
     def eql?(other)
       self.class.equal?(other.class) && @name == other.name

data/lib/bolt/util.rb

@@ -147,6 +147,24 @@ module Bolt
         end
       end
 
+      # Accepts a Data object and returns a copy with all hash and array values
+      # modified by the given block. Descendants are modified before their
+      # parents.
+      def postwalk_vals(data, skip_top = false, &block)
+        new_data = if data.is_a? Hash
+                     map_vals(data) { |v| postwalk_vals(v, &block) }
+                   elsif data.is_a? Array
+                     data.map { |v| postwalk_vals(v, &block) }
+                   else
+                     data
+                   end
+        if skip_top
+          new_data
+        else
+          yield(new_data)
+        end
+      end
+
       # Performs a deep_clone, using an identical copy if the cloned structure contains multiple
       # references to the same object and prevents endless recursion.
       # Credit to Jan Molic via https://github.com/rubyworks/facets/blob/master/LICENSE.txt

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '1.34.0'
+  VERSION = '1.35.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 1.34.0
+  version: 1.35.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-17 00:00:00.000000000 Z
+date: 2019-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -373,6 +373,7 @@ files:
 - lib/bolt/inventory/group.rb
 - lib/bolt/inventory/group2.rb
 - lib/bolt/inventory/inventory2.rb
+- lib/bolt/inventory/target.rb
 - lib/bolt/logger.rb
 - lib/bolt/module.rb
 - lib/bolt/node/errors.rb
@@ -399,15 +400,12 @@ files:
 - lib/bolt/pal/yaml_plan/transpiler.rb
 - lib/bolt/plan_result.rb
 - lib/bolt/plugin.rb
-- lib/bolt/plugin/aws_inventory.rb
 - lib/bolt/plugin/install_agent.rb
 - lib/bolt/plugin/module.rb
 - lib/bolt/plugin/pkcs7.rb
 - lib/bolt/plugin/prompt.rb
 - lib/bolt/plugin/puppetdb.rb
 - lib/bolt/plugin/task.rb
-- lib/bolt/plugin/terraform.rb
-- lib/bolt/plugin/vault.rb
 - lib/bolt/puppetdb.rb
 - lib/bolt/puppetdb/client.rb
 - lib/bolt/puppetdb/config.rb

data/lib/bolt/plugin/aws_inventory.rb

@@ -1,102 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-module Bolt
-  class Plugin
-    class AwsInventory
-      attr_accessor :client
-      attr_reader :config
-
-      def initialize(config:, **_kwargs)
-        require 'aws-sdk-ec2'
-        @config = config
-        @logger = Logging.logger[self]
-      end
-
-      def name
-        'aws_inventory'
-      end
-
-      def hooks
-        [:resolve_reference]
-      end
-
-      def config_client(opts)
-        return client if client
-
-        options = {}
-
-        if opts.key?('region')
-          options[:region] = opts['region']
-        end
-        if opts.key?('profile')
-          options[:profile] = opts['profile']
-        end
-
-        if config['credentials']
-          creds = File.expand_path(config['credentials'])
-          if File.exist?(creds)
-            options[:credentials] = ::Aws::SharedCredentials.new(path: creds)
-          else
-            raise Bolt::ValidationError, "Cannot load credentials file #{config['credentials']}"
-          end
-        end
-
-        ::Aws::EC2::Client.new(options)
-      end
-
-      def resolve_reference(opts)
-        client = config_client(opts)
-        resource = ::Aws::EC2::Resource.new(client: client)
-
-        # Retrieve a list of EC2 instances and create a list of targets
-        # Note: It doesn't seem possible to filter stubbed responses...
-        resource.instances(filters: opts['filters']).map do |instance|
-          next unless instance.state.name == 'running'
-          target = {}
-
-          if opts.key?('uri')
-            uri = lookup(instance, opts['uri'])
-            target['uri'] = uri if uri
-          end
-          if opts.key?('name')
-            real_name = lookup(instance, opts['name'])
-            target['name'] = real_name if real_name
-          end
-          if opts.key?('config')
-            target['config'] = resolve_config(instance, opts['config'])
-          end
-
-          target if target['uri'] || target['name']
-        end.compact
-      end
-
-      # Look for an instance attribute specified in the inventory file
-      def lookup(instance, attribute)
-        value = instance.data.respond_to?(attribute) ? instance.data[attribute] : nil
-        unless value
-          warn_missing_attribute(instance, attribute)
-        end
-        value
-      end
-
-      def warn_missing_attribute(instance, attribute)
-        @logger.warn("Could not find attribute #{attribute} of instance #{instance.instance_id}")
-      end
-
-      # Walk the "template" config mapping provided in the plugin config and
-      # replace all values with the corresponding value from the resource
-      # parameters.
-      def resolve_config(name, config_template)
-        Bolt::Util.walk_vals(config_template) do |value|
-          if value.is_a?(String)
-            lookup(name, value)
-          else
-            value
-          end
-        end
-      end
-    end
-  end
-end

data/lib/bolt/plugin/terraform.rb

@@ -1,175 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-module Bolt
-  class Plugin
-    class Terraform
-      KNOWN_KEYS = Set['_plugin', 'dir', 'resource_type', 'uri', 'name', 'statefile',
-                       'config', 'backend']
-      REQ_KEYS = Set['dir', 'resource_type']
-
-      def initialize(*_args)
-        @logger = Logging.logger[self]
-      end
-
-      def name
-        'terraform'
-      end
-
-      def hooks
-        [:resolve_reference]
-      end
-
-      def warn_missing_property(name, property)
-        @logger.warn("Could not find property #{property} of terraform resource #{name}")
-      end
-
-      # Make sure no unexpected keys are in the inventory config and
-      # that required keys are present
-      def validate_options(opts)
-        opt_keys = opts.keys.to_set
-
-        unless KNOWN_KEYS.superset?(opt_keys)
-          keys = opt_keys - KNOWN_KEYS
-          raise Bolt::ValidationError, "Unexpected key(s) in inventory config: #{keys.to_a.inspect}"
-        end
-
-        unless opt_keys.superset?(REQ_KEYS)
-          keys = REQ_KEYS - opt_keys
-          raise Bolt::ValidationError, "Expected key(s) in inventory config: #{keys.to_a.inspect}"
-        end
-      end
-
-      def resolve_reference(opts)
-        validate_options(opts)
-
-        state = load_statefile(opts)
-
-        resources = extract_resources(state)
-
-        regex = Regexp.new(opts['resource_type'])
-
-        resources.select do |name, _resource|
-          name.match?(regex)
-        end.map do |name, resource|
-          target = {}
-
-          if opts.key?('uri')
-            uri = lookup(name, resource, opts['uri'])
-            target['uri'] = uri if uri
-          end
-          if opts.key?('name')
-            real_name = lookup(name, resource, opts['name'])
-            target['name'] = real_name if real_name
-          end
-          if opts.key?('config')
-            target['config'] = resolve_config(name, resource, opts['config'])
-          end
-          target
-        end.compact
-      end
-
-      def load_statefile(opts)
-        statefile = if opts['backend'] == 'remote'
-                      load_remote_statefile(opts)
-                    else
-                      load_local_statefile(opts)
-                    end
-
-        JSON.parse(statefile)
-      end
-
-      # Uses the Terraform CLI to pull remote state files
-      def load_remote_statefile(opts)
-        dir = File.expand_path(opts['dir'])
-
-        begin
-          stdout_str, stderr_str, status = Open3.capture3('terraform state pull', chdir: dir)
-        rescue Errno::ENOENT
-          reason = if File.directory?(dir)
-                     "Could not find executable 'terraform'"
-                   else
-                     "Could not find directory '#{dir}'"
-                   end
-          raise Bolt::Error.new(reason, 'FILE_ERROR')
-        end
-
-        unless status.success?
-          err = stdout_str + stderr_str
-          msg = "Could not pull Terraform remote state file for #{opts['dir']}:\n#{err}"
-          raise Bolt::Error.new(msg, 'bolt/terraform-state-error')
-        end
-
-        stdout_str
-      end
-
-      def load_local_statefile(opts)
-        dir = opts['dir']
-        filename = opts.fetch('statefile', 'terraform.tfstate')
-        File.read(File.expand_path(File.join(dir, filename)))
-      rescue StandardError => e
-        raise Bolt::FileError.new("Could not load Terraform state file #{filename}: #{e}", filename)
-      end
-
-      # Format the list of resources into a list of [name, attribute map]
-      # pairs. This method handles both version 4 and earlier statefiles, doing
-      # the appropriate munging based on the shape of the data.
-      def extract_resources(state)
-        if state['version'] >= 4
-          state.fetch('resources', []).flat_map do |resource_set|
-            prefix = "#{resource_set['type']}.#{resource_set['name']}"
-            resource_set['instances'].map do |resource|
-              instance_name = prefix
-              instance_name += ".#{resource['index_key']}" if resource['index_key']
-              # When using `terraform state pull` with terraform >= 0.12 version 3 statefiles
-              # Will be converted to version 4. When converted attributes is converted to attributes_flat
-              attributes = resource['attributes'] || resource['attributes_flat']
-              [instance_name, attributes]
-            end
-          end
-        else
-          state.fetch('modules', {}).flat_map do |mod|
-            mod.fetch('resources', {}).map do |name, resource|
-              [name, resource.dig('primary', 'attributes')]
-            end
-          end
-        end
-      end
-
-      # Look up a nested value from the resource attributes. The key is of the
-      # form `foo.bar.0.baz`. For terraform statefile version 3, this will
-      # exactly correspond to a key in the resource. In version 4, it will
-      # correspond to a nested hash entry at {foo: {bar: [{baz: <value>}]}}
-      # For simplicity's sake, we just check both.
-      def lookup(name, resource, path)
-        segments = path.split('.').map do |segment|
-          begin
-            Integer(segment)
-          rescue ArgumentError
-            segment
-          end
-        end
-
-        value = resource[path] || resource.dig(*segments)
-        unless value
-          warn_missing_property(name, path)
-        end
-        value
-      end
-
-      # Walk the "template" config mapping provided in the plugin config and
-      # replace all values with the corresponding value from the resource
-      # parameters.
-      def resolve_config(name, resource, config_template)
-        Bolt::Util.walk_vals(config_template) do |value|
-          if value.is_a?(String)
-            lookup(name, resource, value)
-          else
-            value
-          end
-        end
-      end
-    end
-  end
-end