bolt 1.14.0 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a29df5e9c84d323a9d9b739bff71705573816f077ef68848cf7deb0fba373879
-  data.tar.gz: add9bd7f64f5cc1a61b237421f9973f2d9afe52dcb2fa3f89db1adf94270a93f
+  metadata.gz: 271a7fea2ecaf024054a9ae2266bc352b19a9969300c882f9622341582585d48
+  data.tar.gz: d10c047f108f62efdfe1e6ac1a35bc7b3303332f0ca0bd50c55635857d41648e
 SHA512:
-  metadata.gz: 58002395a520aa67a681c85efe894f98d7fb172eaebf11ddc15719c918d7479335d16ad181873d5ad9cd15a1151eb59551752e4d6f6ecdd5bd1e15335ab7ef83
-  data.tar.gz: 92fbc2af2713bc301e476de777067bca1f9d391c6a538e40e0907a7d7adb821cd8a0b5ba0a9da666b4799a4392bd783d7abcb846993c7f51c1baf923e18a80b8
+  metadata.gz: 7d6355d7246bfa1305c02e13edf5fd7873584b70306e505f48ee1470babf6aa282b9b5f141ee5af6703b60a589f937c7997110ad21a2d6705926835a8b42b415
+  data.tar.gz: 4a1e5f3bf454ce08b084c0310b98c71119305f3c34fbeca552dffe3f6029cc16e7c410e2490605a985552e9603d7a9e653509b8a71679f3b2bb5e0a56e13951d

data/bolt-modules/boltlib/lib/puppet/functions/run_plan.rb

@@ -59,6 +59,11 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
       executor.run_as = run_as
     end
 
+    closure = func.class.dispatcher.dispatchers[0]
+    if closure.model.is_a?(Bolt::PAL::YamlPlan)
+      executor.report_yaml_plan(closure.model.body)
+    end
+
     # wrap plan execution in logging messages
     executor.log_plan(plan_name) do
       result = nil
@@ -66,7 +71,9 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
         # If the plan does not throw :return by calling the return function it's result is
         # undef/nil
         result = catch(:return) do
-          func.class.dispatcher.dispatchers[0].call_by_name_with_scope(scope, params, true)
+          scope.with_global_scope do |global_scope|
+            closure.call_by_name_with_scope(global_scope, params, true)
+          end
           nil
         end&.value
         # Validate the result is a PlanResult

data/bolt-modules/system/lib/puppet/functions/system/env.rb

@@ -11,6 +11,6 @@ Puppet::Functions.create_function(:'system::env') do
   end
 
   def env(name)
-    ENV[name].freeze
+    ENV[name]
   end
 end

data/lib/bolt/analytics.rb

@@ -21,7 +21,9 @@ module Bolt
       target_nodes: :cd4,
       output_format: :cd5,
       statement_count: :cd6,
-      resource_mean: :cd7
+      resource_mean: :cd7,
+      plan_steps: :cd8,
+      return_type: :cd9
     }.freeze
 
     def self.build_client
@@ -62,6 +64,10 @@ module Bolt
       attr_reader :user_id
 
       def initialize(user_id)
+        # lazy-load expensive gem code
+        require 'concurrent/configuration'
+        require 'concurrent/future'
+
         @logger = Logging.logger[self]
         @http = HTTPClient.new
         @user_id = user_id

data/lib/bolt/applicator.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'base64'
-require 'concurrent'
 require 'find'
 require 'json'
 require 'logging'
@@ -15,6 +14,9 @@ require 'bolt/util/puppet_log_level'
 module Bolt
   class Applicator
     def initialize(inventory, executor, modulepath, plugin_dirs, pdb_client, hiera_config, max_compiles)
+      # lazy-load expensive gem code
+      require 'concurrent'
+
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath

data/lib/bolt/apply_result.rb

@@ -65,6 +65,7 @@ module Bolt
     def initialize(target, error: nil, report: nil)
       @target = target
       @value = {}
+      @type = 'apply'
       value['report'] = report if report
       value['_error'] = error if error
       value['_output'] = metrics_message if metrics_message

data/lib/bolt/config.rb

@@ -2,7 +2,8 @@
 
 require 'yaml'
 require 'logging'
-require 'concurrent'
+# limit the loaded portion of concurrent as its expensive
+require 'concurrent/utility/processor_counter'
 require 'pathname'
 require 'bolt/boltdir'
 require 'bolt/transport/ssh'

data/lib/bolt/executor.rb

@@ -3,7 +3,6 @@
 # Used for $ERROR_INFO. This *must* be capitalized!
 require 'English'
 require 'json'
-require 'concurrent'
 require 'logging'
 require 'set'
 require 'bolt/analytics'
@@ -25,10 +24,13 @@ module Bolt
                    noop = nil,
                    bundled_content: nil,
                    load_config: true)
+
+      # lazy-load expensive gem code
+      require 'concurrent'
+
       @analytics = analytics
       @bundled_content = bundled_content
       @logger = Logging.logger[self]
-      @plan_logging = false
       @load_config = load_config
 
       @transports = Bolt::TRANSPORTS.each_with_object({}) do |(key, val), coll|
@@ -199,6 +201,22 @@ module Bolt
       @analytics&.event('Apply', 'ast', data)
     end
 
+    def report_yaml_plan(plan)
+      steps = plan.steps.count
+      return_type = case plan.return
+                    when Bolt::PAL::YamlPlan::EvaluableString
+                      'expression'
+                    when nil
+                      nil
+                    else
+                      'value'
+                    end
+
+      @analytics&.event('Plan', 'yaml', plan_steps: steps, return_type: return_type)
+    rescue StandardError => e
+      @logger.debug { "Failed to submit analytics event: #{e.message}" }
+    end
+
     def with_node_logging(description, batch)
       @logger.info("#{description} on #{batch.map(&:uri)}")
       result = yield

data/lib/bolt/inventory/group.rb

@@ -10,6 +10,11 @@ module Bolt
       # Regex used to validate group names and target aliases.
       NAME_REGEX = /\A[a-z0-9_][a-z0-9_-]*\Z/.freeze
 
+      DATA_KEYS = %w[name config facts vars features].freeze
+      NODE_KEYS = DATA_KEYS + ['alias']
+      GROUP_KEYS = DATA_KEYS + %w[groups nodes]
+      CONFIG_KEYS = Bolt::TRANSPORTS.keys.map(&:to_s) + ['transport']
+
       def initialize(data)
         @logger = Logging.logger[self]
 
@@ -20,11 +25,21 @@ module Bolt
         raise ValidationError.new("Group name must be a String, not #{@name.inspect}", nil) unless @name.is_a?(String)
         raise ValidationError.new("Invalid group name #{@name}", @name) unless @name =~ NAME_REGEX
 
+        unless (unexpected_keys = data.keys - GROUP_KEYS).empty?
+          msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in group #{@name}"
+          @logger.warn(msg)
+        end
+
         @vars = fetch_value(data, 'vars', Hash)
         @facts = fetch_value(data, 'facts', Hash)
         @features = fetch_value(data, 'features', Array)
         @config = fetch_value(data, 'config', Hash)
 
+        unless (unexpected_keys = @config.keys - CONFIG_KEYS).empty?
+          msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in config for group #{@name}"
+          @logger.warn(msg)
+        end
+
         nodes = fetch_value(data, 'nodes', Array)
         groups = fetch_value(data, 'groups', Array)
 
@@ -43,6 +58,16 @@ module Bolt
           raise ValidationError.new("Node #{node} does not have a name", @name) unless node['name']
           @nodes[node['name']] = node
 
+          unless (unexpected_keys = node.keys - NODE_KEYS).empty?
+            msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in node #{node['name']}"
+            @logger.warn(msg)
+          end
+          config_keys = node['config']&.keys || []
+          unless (unexpected_keys = config_keys - CONFIG_KEYS).empty?
+            msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in config for node #{node['name']}"
+            @logger.warn(msg)
+          end
+
           next unless node.include?('alias')
 
           aliases = node['alias']
@@ -67,9 +92,6 @@ module Bolt
         @name_or_alias = nodes.select { |node| node.is_a?(String) }
 
         @groups = groups.map { |g| Group.new(g) }
-
-        # this allows arbitrary info for the top level
-        @rest = data.reject { |k, _| %w[name nodes config groups vars facts features].include? k }
       end
 
       private def fetch_value(data, key, type)

data/lib/bolt/notifier.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
-require 'concurrent'
-
 module Bolt
   class Notifier
-    def initialize(executor = Concurrent::SingleThreadExecutor.new)
-      @executor = executor
+    def initialize(executor = nil)
+      # lazy-load expensive gem code
+      require 'concurrent'
+
+      @executor = executor || Concurrent::SingleThreadExecutor.new
     end
 
     def notify(callback, event)

data/lib/bolt/pal.rb

@@ -81,6 +81,7 @@ module Bolt
 
       require 'bolt/pal/logging'
       require 'bolt/pal/issues'
+      require 'bolt/pal/yaml_plan/loader'
 
       # Now that puppet is loaded we can include puppet mixins in data types
       Bolt::ResultSet.include_iterable
@@ -103,7 +104,9 @@ module Bolt
         pal.with_script_compiler do |compiler|
           alias_types(compiler)
           begin
-            yield compiler
+            Puppet.override(yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
+              yield compiler
+            end
           rescue Bolt::Error => err
             err
           rescue Puppet::PreformattedError => err
@@ -247,7 +250,12 @@ module Bolt
 
     def list_plans
       in_bolt_compiler do |compiler|
-        compiler.list_plans.map { |plan| [plan.name] }.sort
+        errors = []
+        plans = compiler.list_plans(nil, errors).map { |plan| [plan.name] }.sort
+        errors.each do |error|
+          @logger.warn(error.details['original_error'])
+        end
+        plans
       end
     end
 

data/lib/bolt/pal/yaml_plan.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Bolt
+  class PAL
+    class YamlPlan
+      Parameter = Struct.new(:name, :value, :type_expr) do
+        def captures_rest
+          false
+        end
+      end
+
+      attr_reader :name, :parameters, :steps, :return
+
+      def initialize(name, plan)
+        # Top-level plan keys aren't allowed to be Puppet code, so force them
+        # all to strings.
+        plan = Bolt::Util.walk_keys(plan) { |key| stringify(key) }
+
+        @name = name.freeze
+
+        # Nothing in parameters is allowed to be code, since no variables are defined yet
+        params_hash = stringify(plan.fetch('parameters', {}))
+
+        # Munge parameters into an array of Parameter objects, which is what
+        # the Puppet API expects
+        @parameters = params_hash.map do |param, definition|
+          definition ||= {}
+          type = Puppet::Pops::Types::TypeParser.singleton.parse(definition['type']) if definition.key?('type')
+          Parameter.new(param, definition['default'], type)
+        end.freeze
+
+        @steps = plan['steps']&.map do |step|
+          # Step keys also aren't allowed to be code and neither is the value of "name"
+          stringified_step = Bolt::Util.walk_keys(step) { |key| stringify(key) }
+          stringified_step['name'] = stringify(stringified_step['name']) if stringified_step.key?('name')
+          stringified_step
+        end.freeze
+
+        @return = plan['return']
+
+        validate
+      end
+
+      VAR_NAME_PATTERN = /\A[a-z_][a-z0-9_]*\z/.freeze
+
+      def validate
+        unless @steps.is_a?(Array)
+          raise Bolt::Error.new("Plan must specify an array of steps", "bolt/invalid-plan")
+        end
+
+        used_names = Set.new
+
+        # Parameters come in a hash, so they must be unique
+        @parameters.each do |param|
+          unless param.name.is_a?(String) && param.name.match?(VAR_NAME_PATTERN)
+            raise Bolt::Error.new("Invalid parameter name #{param.name.inspect}", "bolt/invalid-plan")
+          end
+
+          used_names << param.name
+        end
+
+        @steps.each do |step|
+          next unless step.key?('name')
+
+          unless step['name'].is_a?(String) && step['name'].match?(VAR_NAME_PATTERN)
+            raise Bolt::Error.new("Invalid step name #{step['name'].inspect}", "bolt/invalid-plan")
+          end
+
+          if used_names.include?(step['name'])
+            msg = "Step name #{step['name'].inspect} matches an existing parameter or step name"
+            raise Bolt::Error.new(msg, "bolt/invalid-plan")
+          end
+
+          used_names << step['name']
+        end
+      end
+
+      def body
+        self
+      end
+
+      # Turn all "potential" strings in the object into actual strings.
+      # Because we interpret bare strings as potential Puppet code, even in
+      # places where Puppet code isn't allowed (like some hash keys), we need
+      # to be able to force them back into regular strings, as if we had
+      # parsed them normally.
+      def stringify(value)
+        case value
+        when Array
+          value.map { |element| stringify(element) }
+        when Hash
+          value.each_with_object({}) do |(k, v), o|
+            o[stringify(k)] = stringify(v)
+          end
+        when EvaluableString
+          value.value
+        else
+          value
+        end
+      end
+
+      def return_type
+        Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult')
+      end
+
+      # This class wraps a value parsed from YAML which may be Puppet code.
+      # That includes double-quoted strings and string literals, each of which
+      # subclasses this parent class in order to implement its own evaluation
+      # logic.
+      class EvaluableString
+        attr_reader :value
+        def initialize(value)
+          @value = value
+        end
+      end
+
+      # This class represents a double-quoted YAML string, which is interpreted
+      # as though it were a double-quoted Puppet string (with associated
+      # variable interpolations)
+      class DoubleQuotedString < EvaluableString
+        def evaluate(scope, evaluator)
+          # "inspect" allows us to get back a double-quoted string literal with
+          # special characters escaped. This is based on the assumption that
+          # YAML, Ruby and Puppet all support similar escape sequences.
+          parse_result = evaluator.parse_string(@value.inspect)
+
+          scope.with_local_scope({}) do
+            evaluator.evaluate(scope, parse_result)
+          end
+        end
+      end
+
+      # This represents a literal snippet of Puppet code
+      class CodeLiteral < EvaluableString
+        def evaluate(scope, evaluator)
+          parse_result = evaluator.parse_string(@value)
+
+          scope.with_local_scope({}) do
+            evaluator.evaluate(scope, parse_result)
+          end
+        end
+      end
+
+      # This class stores a bare YAML string, which is fuzzily interpreted as
+      # either Puppet code or a literal string, depending on whether it starts
+      # with a variable reference.
+      class BareString < EvaluableString
+        def evaluate(scope, evaluator)
+          if @value.start_with?('$')
+            # Try to parse the string as Puppet code. If it's invalid code,
+            # return the original string.
+            parse_result = evaluator.parse_string(@value)
+            scope.with_local_scope({}) do
+              evaluator.evaluate(scope, parse_result)
+            end
+          else
+            @value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/pal/yaml_plan/evaluator.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'bolt/pal/yaml_plan'
+
+module Bolt
+  class PAL
+    class YamlPlan
+      class Evaluator
+        def initialize(analytics = Bolt::Analytics::NoopClient.new)
+          @logger = Logging.logger[self]
+          @analytics = analytics
+          @evaluator = Puppet::Pops::Parser::EvaluatingParser.new
+        end
+
+        STEP_KEYS = %w[task command eval script source plan].freeze
+
+        def dispatch_step(scope, step)
+          step = evaluate_code_blocks(scope, step)
+
+          step_type, *extra_keys = STEP_KEYS.select { |key| step.key?(key) }
+          if !step_type || extra_keys.any?
+            unsupported_step(scope, step)
+          end
+
+          case step_type
+          when 'task'
+            task_step(scope, step)
+          when 'command'
+            command_step(scope, step)
+          when 'plan'
+            plan_step(scope, step)
+          when 'script'
+            script_step(scope, step)
+          when 'source'
+            upload_file_step(scope, step)
+          when 'eval'
+            eval_step(scope, step)
+          else
+            # This shouldn't be able to happen since this case statement should
+            # match the STEP_KEYS list, but raise an error *just in case*,
+            # instead of silently skipping the step.
+            unsupported_step(scope, step)
+          end
+        end
+
+        def task_step(scope, step)
+          task = step['task']
+          target = step['target']
+          description = step['description']
+          params = step['parameters'] || {}
+          raise "Can't run a task without specifying a target" unless target
+
+          args = if description
+                   [task, target, description, params]
+                 else
+                   [task, target, params]
+                 end
+
+          scope.call_function('run_task', args)
+        end
+
+        def plan_step(scope, step)
+          plan = step['plan']
+          parameters = step['parameters'] || {}
+
+          args = [plan, parameters]
+
+          scope.call_function('run_plan', args)
+        end
+
+        def script_step(scope, step)
+          script = step['script']
+          target = step['target']
+          description = step['description']
+          arguments = step['arguments'] || []
+          raise "Can't run a script without specifying a target" unless target
+
+          options = { 'arguments' => arguments }
+          args = if description
+                   [script, target, description, options]
+                 else
+                   [script, target, options]
+                 end
+
+          scope.call_function('run_script', args)
+        end
+
+        def command_step(scope, step)
+          command = step['command']
+          target = step['target']
+          description = step['description']
+          raise "Can't run a command without specifying a target" unless target
+
+          args = [command, target]
+          args << description if description
+          scope.call_function('run_command', args)
+        end
+
+        def upload_file_step(scope, step)
+          source = step['source']
+          destination = step['destination']
+          target = step['target']
+          description = step['description']
+          raise "Can't upload a file without specifying a target" unless target
+          raise "Can't upload a file without specifying a destination" unless destination
+
+          args = [source, destination, target]
+          args << description if description
+          scope.call_function('upload_file', args)
+        end
+
+        def eval_step(_scope, step)
+          step['eval']
+        end
+
+        def unsupported_step(_scope, step)
+          raise Bolt::Error.new("Unsupported plan step", "bolt/unsupported-step", step: step)
+        end
+
+        # This is the method that Puppet calls to evaluate the plan. The name
+        # makes more sense for .pp plans.
+        def evaluate_block_with_bindings(closure_scope, args_hash, plan)
+          plan_result = closure_scope.with_local_scope(args_hash) do |scope|
+            plan.steps.each do |step|
+              step_result = dispatch_step(scope, step)
+
+              scope.setvar(step['name'], step_result) if step.key?('name')
+            end
+
+            evaluate_code_blocks(scope, plan.return)
+          end
+
+          throw :return, Puppet::Pops::Evaluator::Return.new(plan_result, nil, nil)
+        end
+
+        # Recursively evaluate any EvaluableString instances in the object.
+        def evaluate_code_blocks(scope, value)
+          # XXX We should establish a local scope here probably
+          case value
+          when Array
+            value.map { |element| evaluate_code_blocks(scope, element) }
+          when Hash
+            value.each_with_object({}) do |(k, v), o|
+              key = k.is_a?(EvaluableString) ? k.value : k
+              o[key] = evaluate_code_blocks(scope, v)
+            end
+          when EvaluableString
+            value.evaluate(scope, @evaluator)
+          else
+            value
+          end
+        end
+
+        # Occasionally the Closure will ask us to evaluate what it assumes are
+        # AST objects. Because we've sidestepped the AST, they aren't, so just
+        # return the values as already evaluated.
+        def evaluate(value, _scope)
+          value
+        end
+      end
+    end
+  end
+end