bolt 0.16.1 → 0.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c145190f4fb70b998c8fd501a368940c2d2db3c
-  data.tar.gz: 45949da83f3903d369a71f88730e8ef7413cdff7
+  metadata.gz: ee1bb84782f71c0faea509d91746ce03e9749432
+  data.tar.gz: 636871a24286b9ca256ffd8c3e31c3224a20ea97
 SHA512:
-  metadata.gz: 4597004b2108d16ab6aaf401e0b5160ef711ce27fe487b1996457f3b007ffc522c438b6df3a4b540b7e7566b9bc3794ecee4474400dbf21393b132b5abf99134
-  data.tar.gz: fe62dba570d0e11ed31e2be2e490a8eaa347485edf9adca4798e6c731ae892f48ad767f6f3493f1f54f42df7468b091c9a8db057e6bc65b2037eb607817b3152
+  metadata.gz: bede0c32f199f41154a2223f74e83c9c7e62a6ea32edadd0850db0bc6faa655603126750e949b1ee2a9192186c527ba3a848205688568bbfcf1bdcb7eab45129
+  data.tar.gz: b008ffcdd129e7b785a0d93cdb4c37b128d3ab28be930fd7f9e8e5fa40ac5b4250c094be60a2ba12117806c9194029298b6d89df638cc6e945585d9b9b523e0d

data/lib/bolt.rb

@@ -1,4 +1,3 @@
 module Bolt
   require 'bolt/executor'
-  require 'bolt/node'
 end

data/lib/bolt/cli.rb

@@ -9,7 +9,6 @@ require 'bolt/error'
 require 'bolt/executor'
 require 'bolt/inventory'
 require 'bolt/logger'
-require 'bolt/node'
 require 'bolt/outputter'
 require 'bolt/pal'
 require 'bolt/target'
@@ -19,7 +18,6 @@ module Bolt
   class CLIError < Bolt::Error
     def initialize(msg)
       super(msg, "bolt/cli-error")
-      @error_code = error_code if error_code
     end
   end
 
@@ -136,7 +134,7 @@ HELP
                   '* protocol is `ssh` by default, may be `ssh` or `winrm`',
                   '* port defaults to `22` for SSH',
                   '* port defaults to `5985` or `5986` for WinRM, based on the --[no-]ssl setting') do |nodes|
-            results[:nodes] << get_arg_input(nodes).split(/[[:space:],]+/).reject(&:empty?)
+            results[:nodes] << get_arg_input(nodes)
           end
         end
         opts.on('-u', '--user USER',
@@ -259,6 +257,12 @@ HELP
       parser
     end
 
+    # Only call after @config has been initialized.
+    def inventory
+      Bolt::Inventory.from_config(@config)
+    end
+    private :inventory
+
     def parse
       if @argv.empty?
         options[:help] = true
@@ -309,8 +313,11 @@ HELP
 
       validate(options)
 
+      # After validation, initialize inventory and targets. Errors here are better to catch early.
+      options[:targets] = inventory.get_targets(options[:nodes]) if options[:nodes]
+
       options
-    rescue Bolt::CLIError => e
+    rescue Bolt::Error => e
       warn e.message
       raise e
     end
@@ -434,7 +441,6 @@ HELP
         return 0
       end
 
-      inventory = Bolt::Inventory.from_config(@config)
       message = 'There may be processes left executing on some nodes.'
 
       if options[:mode] == 'plan'
@@ -445,7 +451,7 @@ HELP
         code = 0
       else
         executor = Bolt::Executor.new(@config, options[:noop])
-        targets = inventory.get_targets(options[:nodes])
+        targets = options[:targets]
 
         results = nil
         outputter.print_head

data/lib/bolt/error.rb

@@ -29,6 +29,14 @@ module Bolt
     def to_puppet_error
       Puppet::DataTypes::Error.from_asserted_hash(to_h)
     end
+
+    def self.unknown_task(task)
+      "Could not find a task named \"#{task}\". For a list of available tasks, run \"bolt task show\""
+    end
+
+    def self.unknown_plan(plan)
+      "Could not find a plan named \"#{plan}\". For a list of available plans, run \"bolt plan show\""
+    end
   end
 
   class RunFailure < Error

data/lib/bolt/executor.rb

@@ -1,21 +1,31 @@
+# Used for $ERROR_INFO. This *must* be capitalized!
+require 'English'
 require 'json'
 require 'concurrent'
 require 'logging'
 require 'bolt/result'
 require 'bolt/config'
 require 'bolt/notifier'
-require 'bolt/node'
 require 'bolt/result_set'
+require 'bolt/transport/ssh'
+require 'bolt/transport/winrm'
+require 'bolt/transport/orch'
 
 module Bolt
   class Executor
-    attr_reader :noop
+    attr_reader :noop, :transports
     attr_accessor :run_as
 
     def initialize(config = Bolt::Config.new, noop = nil, plan_logging = false)
       @config = config
       @logger = Logging.logger[self]
 
+      @transports = {
+        'ssh' => Concurrent::Delay.new { Bolt::Transport::SSH.new(config[:transports][:ssh] || {}) },
+        'winrm' => Concurrent::Delay.new { Bolt::Transport::WinRM.new(config[:transports][:winrm] || {}) },
+        'pcp' => Concurrent::Delay.new { Bolt::Transport::Orch.new(config[:transports][:pcp] || {}) }
+      }
+
       # If a specific elevated log level has been requested, honor that.
       # Otherwise, escalate the log level to "info" if running in plan mode, so
       # that certain progress messages will be visible.
@@ -23,55 +33,14 @@ module Bolt
       @logger.level = @config[:log_level] || default_log_level
       @noop = noop
       @run_as = nil
+      @pool = Concurrent::CachedThreadPool.new(max_threads: @config[:concurrency])
+      @logger.debug { "Started with #{@config[:concurrency]} max thread(s)" }
       @notifier = Bolt::Notifier.new
     end
 
-    def from_targets(targets)
-      targets.map do |target|
-        Bolt::Node.from_target(target)
-      end
-    end
-    private :from_targets
-
-    def on(nodes, callback = nil)
-      results = Concurrent::Array.new
-
-      poolsize = [nodes.length, @config[:concurrency]].min
-      pool = Concurrent::FixedThreadPool.new(poolsize)
-      @logger.debug { "Started with #{poolsize} thread(s)" }
-
-      nodes.map(&:class).uniq.each do |klass|
-        klass.initialize_transport(@logger)
-      end
-
-      nodes.each { |node|
-        pool.post do
-          result =
-            begin
-              @notifier.notify(callback, type: :node_start, target: node.target) if callback
-              node.connect
-              yield node
-            rescue StandardError => ex
-              Bolt::Result.from_exception(node.target, ex)
-            ensure
-              begin
-                node.disconnect
-              rescue StandardError => ex
-                @logger.info("Failed to close connection to #{node.uri} : #{ex.message}")
-              end
-            end
-          results.concat([result])
-          @notifier.notify(callback, type: :node_result, result: result) if callback
-        end
-      }
-      pool.shutdown
-      pool.wait_for_termination
-
-      @notifier.shutdown
-
-      Bolt::ResultSet.new(results)
+    def transport(transport)
+      @transports[transport || 'ssh'].value
     end
-    private :on
 
     def summary(action, object, result)
       fc = result.error_set.length
@@ -81,90 +50,107 @@ module Bolt
     end
     private :summary
 
-    def get_run_as(node, options)
-      if node.run_as.nil? && run_as
-        { '_run_as' => run_as }.merge(options)
-      else
-        options
+    # Execute the given block on a list of nodes in parallel, one thread per "batch".
+    #
+    # This is the main driver of execution on a list of targets. It first
+    # groups targets by transport, then divides each group into batches as
+    # defined by the transport. Each batch, along with the corresponding
+    # transport, is yielded to the block in turn and the results all collected
+    # into a single ResultSet.
+    def batch_execute(targets)
+      promises = targets.group_by(&:protocol).flat_map do |protocol, _protocol_targets|
+        transport = transport(protocol)
+        transport.batches(targets).flat_map do |batch|
+          batch_promises = Hash[Array(batch).map { |target| [target, Concurrent::Promise.new(executor: :immediate)] }]
+          # Pass this argument through to avoid retaining a reference to a
+          # local variable that will change on the next iteration of the loop.
+          @pool.post(batch_promises) do |result_promises|
+            begin
+              results = yield transport, batch
+              Array(results).each do |result|
+                result_promises[result.target].set(result)
+              end
+            # NotImplementedError can be thrown if the transport is implemented improperly
+            rescue StandardError, NotImplementedError => e
+              result_promises.each do |target, promise|
+                promise.set(Bolt::Result.from_exception(target, e))
+              end
+            ensure
+              # Make absolutely sure every promise gets a result to avoid a
+              # deadlock. Use whatever exception is causing this block to
+              # execute, or generate one if we somehow got here without an
+              # exception and some promise is still missing a result.
+              result_promises.each do |target, promise|
+                next if promise.fulfilled?
+                error = $ERROR_INFO || Bolt::Error.new("No result was returned for #{target.uri}",
+                                                       "puppetlabs.bolt/missing-result-error")
+                promise.set(Bolt::Result.from_exception(error))
+              end
+            end
+          end
+          batch_promises.values
+        end
       end
+      ResultSet.new(promises.map(&:value))
     end
-    private :get_run_as
 
-    def with_exception_handling(node)
-      yield
-    rescue StandardError => e
-      Bolt::Result.from_exception(node.target, e)
-    end
-    private :with_exception_handling
-
-    def run_command(targets, command, options = {})
-      nodes = from_targets(targets)
-      @logger.info("Starting command run '#{command}' on #{nodes.map(&:uri)}")
-      callback = block_given? ? Proc.new : nil
+    def run_command(targets, command, options = {}, &callback)
+      @logger.info("Starting command run '#{command}' on #{targets.map(&:uri)}")
+      notify = proc { |event| @notifier.notify(callback, event) if callback }
+      options = { '_run_as' => run_as }.merge(options) if run_as
 
-      r = on(nodes, callback) do |node|
-        @logger.debug("Running command '#{command}' on #{node.uri}")
-        node_result = with_exception_handling(node) do
-          node.run_command(command, get_run_as(node, options))
-        end
-        @logger.debug("Result on #{node.uri}: #{JSON.dump(node_result.value)}")
-        node_result
+      results = batch_execute(targets) do |transport, batch|
+        transport.batch_command(batch, command, options, &notify)
       end
-      @logger.info(summary('command', command, r))
-      r
+
+      @logger.info(summary('command', command, results))
+      @notifier.shutdown
+      results
     end
 
-    def run_script(targets, script, arguments, options = {})
-      nodes = from_targets(targets)
-      @logger.info("Starting script run #{script} on #{nodes.map(&:uri)}")
+    def run_script(targets, script, arguments, options = {}, &callback)
+      @logger.info("Starting script run #{script} on #{targets.map(&:uri)}")
       @logger.debug("Arguments: #{arguments}")
-      callback = block_given? ? Proc.new : nil
+      notify = proc { |event| @notifier.notify(callback, event) if callback }
+      options = { '_run_as' => run_as }.merge(options) if run_as
 
-      r = on(nodes, callback) do |node|
-        @logger.debug { "Running script '#{script}' on #{node.uri}" }
-        node_result = with_exception_handling(node) do
-          node.run_script(script, arguments, get_run_as(node, options))
-        end
-        @logger.debug("Result on #{node.uri}: #{JSON.dump(node_result.value)}")
-        node_result
+      results = batch_execute(targets) do |transport, batch|
+        transport.batch_script(batch, script, arguments, options, &notify)
       end
-      @logger.info(summary('script', script, r))
-      r
+
+      @logger.info(summary('script', script, results))
+      @notifier.shutdown
+      results
     end
 
-    def run_task(targets, task, input_method, arguments, options = {})
-      nodes = from_targets(targets)
-      @logger.info("Starting task #{task} on #{nodes.map(&:uri)}")
-      @logger.debug("Arguments: #{arguments} Input method: #{input_method}")
-      callback = block_given? ? Proc.new : nil
+    def run_task(targets, task, arguments, options = {}, &callback)
+      task_name = task.name
+      @logger.info("Starting task #{task_name} on #{targets.map(&:uri)}")
+      @logger.debug("Arguments: #{arguments} Input method: #{task.input_method}")
+      notify = proc { |event| @notifier.notify(callback, event) if callback }
+      options = { '_run_as' => run_as }.merge(options) if run_as
 
-      r = on(nodes, callback) do |node|
-        @logger.debug { "Running task run '#{task}' on #{node.uri}" }
-        node_result = with_exception_handling(node) do
-          node.run_task(task, input_method, arguments, get_run_as(node, options))
-        end
-        @logger.debug("Result on #{node.uri}: #{JSON.dump(node_result.value)}")
-        node_result
+      results = batch_execute(targets) do |transport, batch|
+        transport.batch_task(batch, task, arguments, options, &notify)
       end
-      @logger.info(summary('task', task, r))
-      r
+
+      @logger.info(summary('task', task_name, results))
+      @notifier.shutdown
+      results
     end
 
-    def file_upload(targets, source, destination, options = {})
-      nodes = from_targets(targets)
-      @logger.info("Starting file upload from #{source} to #{destination} on #{nodes.map(&:uri)}")
-      callback = block_given? ? Proc.new : nil
+    def file_upload(targets, source, destination, options = {}, &callback)
+      @logger.info("Starting file upload from #{source} to #{destination} on #{targets.map(&:uri)}")
+      notify = proc { |event| @notifier.notify(callback, event) if callback }
+      options = { '_run_as' => run_as }.merge(options) if run_as
 
-      r = on(nodes, callback) do |node|
-        @logger.debug { "Uploading: '#{source}' to #{destination} on #{node.uri}" }
-        node_result = with_exception_handling(node) do
-          node.upload(source, destination, options)
-        end
-        @logger.debug("Result on #{node.uri}: #{JSON.dump(node_result.value)}")
-        node_result
+      results = batch_execute(targets) do |transport, batch|
+        transport.batch_upload(batch, source, destination, options, &notify)
       end
-      @logger.info(summary('upload', source, r))
-      r
+
+      @logger.info(summary('upload', source, results))
+      @notifier.shutdown
+      results
     end
   end
 end

data/lib/bolt/inventory.rb

@@ -26,6 +26,12 @@ module Bolt
       end
     end
 
+    class WildcardError < Bolt::Error
+      def initialize(target)
+        super("Found 0 nodes matching wildcard pattern #{target}", 'bolt.inventory/wildcard-error')
+      end
+    end
+
     def self.default_paths
       [File.expand_path(File.join('~', '.puppetlabs', 'bolt', 'inventory.yaml'))]
     end
@@ -35,21 +41,28 @@ module Bolt
 
       inventory = new(data, config)
       inventory.validate
+      inventory.collect_groups
       inventory
     end
 
     def initialize(data, config = nil)
       @logger = Logging.logger[self]
       # Config is saved to add config options to targets
-      @config = config || {}
+      @config = config || Bolt::Config.new
       @data = data ||= {}
       @groups = Group.new(data.merge('name' => 'all'))
+      @group_lookup = {}
     end
 
     def validate
       @groups.validate
     end
 
+    def collect_groups
+      # Provide a lookup map for finding a group by name
+      @group_lookup = @groups.collect_groups
+    end
+
     def get_targets(targets)
       targets = expand_targets(targets)
       targets = if targets.is_a? Array
@@ -87,6 +100,33 @@ module Bolt
       conf = Bolt::Util.deep_merge(@config.transport_conf, inv_conf)
       target.update_conf(conf)
     end
+    private :update_target
+
+    # If target is a group name, expand it to the members of that group.
+    # If a wildcard string, match against nodes in inventory (or error if none found).
+    # Else return [target].
+    def resolve_name(target)
+      if (group = @group_lookup[target])
+        group.node_names
+      elsif target.include?('*')
+        # Try to wildcard match nodes in inventory
+        # Ignore case because hostnames are generally case-insensitive
+        regexp = Regexp.new("^#{Regexp.escape(target).gsub('\*', '.*?')}$", Regexp::IGNORECASE)
+
+        nodes = []
+        @groups.node_names.each do |node|
+          if node =~ regexp
+            nodes << node
+          end
+        end
+
+        raise(WildcardError, target) if nodes.empty?
+        nodes
+      else
+        [target]
+      end
+    end
+    private :resolve_name
 
     def expand_targets(targets)
       if targets.is_a? Bolt::Target
@@ -94,8 +134,13 @@ module Bolt
       elsif targets.is_a? Array
         targets.map { |tish| expand_targets(tish) }
       elsif targets.is_a? String
-        Bolt::Target.new(targets)
+        # Expand a comma-separated list
+        targets.split(/[[:space:],]+/).reject(&:empty?).map do |name|
+          ts = resolve_name(name)
+          ts.map { |t| Bolt::Target.new(t) }
+        end
       end
     end
+    private :expand_targets
   end
 end