bolt 1.8.1 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65bfdfb39627ab98263d43ab9fd6389eaeca02f809091a85d169da3e56b7cdf4
-  data.tar.gz: 2184a841229576fe544824d8bf1bd687e1c319411651f29beffab00a5bd0e971
+  metadata.gz: 5818403d3dac52071b37dc23c99dc29a74c790493ea192b1f1855ca93cda721a
+  data.tar.gz: '019a929cfd4a7bfd2ce2bd51cbf4fa4e48cea6bb577213f68ae33dabaa704aa4'
 SHA512:
-  metadata.gz: af1ccedbc0ba492c0b3acc69f0b250c0dc48e32ee63f8ce6e9bfa8ef259e4dd2bdbe659a0d0e3a3f80ea4a731c394cc7c7582833a93ad7f0af07e05e0d1a0535
-  data.tar.gz: 65c4688066c5f2af97e22941820cb5c3cc7805a40f103a36d97c43c833f101f93957d3576aa1126046c64be67d06ebf4e2b59f7960f9678ce47985e6c00693dc
+  metadata.gz: 352f34c5cdef73ccde6a84df1fbf722a562b251801a9d9d49c7b221237de7a03758ba57139559fb07f8124ce3f19d4723b7313725bcdde0fa88f9cb1eeaf64e6
+  data.tar.gz: 35a9b64a4c97465b4f72e0ad7e7b98e2fdf657b08b89653a024e46d5ef78fa0fd44f5e9db66f97f1f9b1fed399d29a277461e9b8e63c9601307b9811f138a823

data/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -63,18 +63,19 @@ Puppet::Functions.create_function(:apply_prep) do
           need_install, installed = versions.partition { |r| r['version'].nil? }
           installed.each do |r|
             Puppet.debug "Puppet Agent #{r['version']} installed on #{r.target.name}"
+            inventory.set_feature(r.target, 'puppet-agent')
           end
 
           unless need_install.empty?
             need_install_targets = need_install.map(&:target)
             run_task(executor, need_install_targets, 'puppet_agent::install')
-
+            # Service task works best when targets have puppet-agent feature
+            need_install_targets.each { |target| inventory.set_feature(target, 'puppet-agent') }
             # Ensure the Puppet service is stopped after new install
             run_task(executor, need_install_targets, 'service', 'action' => 'stop', 'name' => 'puppet')
             run_task(executor, need_install_targets, 'service', 'action' => 'disable', 'name' => 'puppet')
           end
         end
-        targets.each { |target| inventory.set_feature(target, 'puppet-agent') }
 
         # Gather facts, including custom facts
         plugins = applicator.build_plugin_tarball do |mod|

data/bolt-modules/ctrl/lib/puppet/functions/ctrl/do_until.rb

@@ -3,7 +3,7 @@
 # Repeat the block until it returns a truthy value. Returns the value.
 Puppet::Functions.create_function(:'ctrl::do_until') do
   # @example Run a task until it succeeds
-  #   ctrl::do_until || {
+  #   ctrl::do_until() || {
   #     run_task('test', $target, _catch_errors => true).ok?
   #   }
   dispatch :do_until do

data/lib/bolt/applicator.rb

@@ -7,6 +7,7 @@ require 'json'
 require 'logging'
 require 'minitar'
 require 'open3'
+require 'bolt/error'
 require 'bolt/task'
 require 'bolt/apply_result'
 require 'bolt/util/puppet_log_level'

data/lib/bolt/inventory/group.rb

@@ -125,7 +125,7 @@ module Bolt
           # Require nodes to be parseable as a Target.
           begin
             Target.new(n)
-          rescue Addressable::URI::InvalidURIError => e
+          rescue Bolt::ParseError => e
             @logger.debug(e)
             raise ValidationError.new("Invalid node name #{n}", @name)
           end

data/lib/bolt/puppetdb.rb

@@ -10,4 +10,6 @@ module Bolt
       super(msg, "bolt/puppetdb-error")
     end
   end
+
+  class PuppetDBFailoverError < PuppetDBError; end
 end

data/lib/bolt/puppetdb/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'logging'
 require 'uri'
 require 'httpclient'
 
@@ -11,6 +12,9 @@ module Bolt
 
       def initialize(config)
         @config = config
+        @bad_urls = []
+        @current_url = nil
+        @logger = Logging.logger[self]
       end
 
       def query_certnames(query)
@@ -41,22 +45,35 @@ module Bolt
 
       def make_query(query, path = nil)
         body = JSON.generate(query: query)
-        url = "#{@config.uri}/pdb/query/v4"
+        url = "#{uri}/pdb/query/v4"
         url += "/#{path}" if path
 
         begin
           response = http_client.post(url, body: body, header: headers)
+        rescue SocketError, OpenSSL::SSL::SSLError, SystemCallError, Net::ProtocolError, IOError => err
+          raise Bolt::PuppetDBFailoverError, "Failed to query PuppetDB: #{err}"
         rescue StandardError => err
           raise Bolt::PuppetDBError, "Failed to query PuppetDB: #{err}"
         end
+
         if response.code != 200
-          raise Bolt::PuppetDBError, "Failed to query PuppetDB: #{response.body}"
+          msg = "Failed to query PuppetDB: #{response.body}"
+          if response.code == 400
+            raise Bolt::PuppetDBError, msg
+          else
+            raise Bolt::PuppetDBFailoverError, msg
+          end
         end
+
         begin
           JSON.parse(response.body)
         rescue JSON::ParserError
           raise Bolt::PuppetDBError, "Unable to parse response as JSON: #{response.body}"
         end
+      rescue Bolt::PuppetDBFailoverError => err
+        @logger.error("Request to puppetdb at #{@current_url} failed with #{err}.")
+        reject_url
+        make_query(query, path)
       end
 
       def http_client
@@ -68,6 +85,23 @@ module Bolt
         @http
       end
 
+      def reject_url
+        @bad_urls << @current_url if @current_url
+        @current_url = nil
+      end
+
+      def uri
+        @current_url ||= (@config.server_urls - @bad_urls).first
+        unless @current_url
+          msg = "Failed to connect to all PuppetDB server_urls: #{@config.server_urls.to_a.join(', ')}."
+          raise Bolt::PuppetDBError, msg
+        end
+
+        uri = URI.parse(@current_url)
+        uri.port ||= 8081
+        uri
+      end
+
       def headers
         headers = { 'Content-Type' => 'application/json' }
         headers['X-Authentication'] = @config.token if @config.token

data/lib/bolt/puppetdb/config.rb

@@ -20,6 +20,7 @@ module Bolt
       end
 
       def self.load_config(filename, options)
+        config = {}
         global_path = Bolt::Util.windows? ? DEFAULT_CONFIG[:win_global] : DEFAULT_CONFIG[:global]
         if filename
           if File.exist?(filename)
@@ -27,13 +28,20 @@ module Bolt
           else
             raise Bolt::PuppetDBError, "config file #{filename} does not exist"
           end
-        elsif File.exist?(DEFAULT_CONFIG[:user])
-          config = JSON.parse(File.read(DEFAULT_CONFIG[:user]))
-        elsif File.exist?(global_path)
-          config = JSON.parse(File.read(global_path))
         else
-          config = {}
+          if File.exist?(DEFAULT_CONFIG[:user])
+            filepath = DEFAULT_CONFIG[:user]
+          elsif File.exist?(global_path)
+            filepath = global_path
+          end
+
+          begin
+            config = JSON.parse(File.read(filepath)) if filepath
+          rescue StandardError => e
+            Logging.logger[self].error("Could not load puppetdb.conf from #{filepath}: #{e.message}")
+          end
         end
+
         config = config.fetch('puppetdb', {})
         new(config.merge(options))
       end
@@ -45,8 +53,11 @@ module Bolt
 
       def token
         return @token if @token
-        if @settings['token']
-          @token = File.read(@settings['token'])
+        # Allow nil in config to skip loading a token
+        if @settings.include?('token')
+          if @settings['token']
+            @token = File.read(@settings['token'])
+          end
         elsif File.exist?(DEFAULT_TOKEN)
           @token = File.read(DEFAULT_TOKEN)
         end
@@ -66,6 +77,19 @@ module Bolt
         true
       end
 
+      def server_urls
+        case @settings['server_urls']
+        when String
+          [@settings['server_urls']]
+        when Array
+          @settings['server_urls']
+        when nil
+          raise Bolt::PuppetDBError, "server_urls must be specified"
+        else
+          raise Bolt::PuppetDBError, "server_urls must be a string or array"
+        end
+      end
+
       def uri
         return @uri if @uri
         uri = case @settings['server_urls']

data/lib/bolt/target.rb

@@ -62,6 +62,8 @@ module Bolt
         # Initialize with an empty scheme to ensure we parse the hostname correctly
         Addressable::URI.parse("//#{string}")
       end
+    rescue Addressable::URI::InvalidURIError => e
+      raise Bolt::ParseError, "Could not parse target URI: #{e.message}"
     end
     private :parse
 

data/lib/bolt/transport/api.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'bolt/transport/api/connection'
+
+# A copy of the orchestrator transport which uses the api connection class
+# in order to bypass calling 'start_plan'
+module Bolt
+  module Transport
+    class Api < Orch
+      def initialize(*args)
+        super
+      end
+
+      def get_connection(conn_opts)
+        key = Bolt::Transport::Api::Connection.get_key(conn_opts)
+        unless (conn = @connections[key])
+          @connections[key] = Bolt::Transport::Api::Connection.new(conn_opts, logger)
+          conn = @connections[key]
+        end
+        conn
+      end
+
+      def batches(targets)
+        targets.group_by { |target| Bolt::Transport::Api::Connection.get_key(target.options) }.values
+      end
+    end
+  end
+end

data/lib/bolt/transport/api/connection.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This is a copy of the orchestrator transport connection, but without the
+# 'start_plan' call in the init function which is handled by the orchestrator
+# client
+module Bolt
+  module Transport
+    class Api < Orch
+      class Connection
+        attr_reader :logger, :key
+
+        CONTEXT_KEYS = Set.new(%i[plan_name description params]).freeze
+
+        def self.get_key(opts)
+          [
+            opts['service-url'],
+            opts['task-environment'],
+            opts['token-file']
+          ].join('-')
+        end
+
+        def initialize(opts, logger)
+          @logger = logger
+          @key = self.class.get_key(opts)
+          client_keys = %w[service-url token-file cacert]
+          client_opts = client_keys.each_with_object({}) do |k, acc|
+            acc[k] = opts[k] if opts.include?(k)
+          end
+          client_opts['User-Agent'] = "Bolt/#{VERSION}"
+          logger.debug("Creating orchestrator client for #{client_opts}")
+
+          @client = OrchestratorClient.new(client_opts, true)
+          @environment = opts["task-environment"]
+        end
+
+        def finish_plan(plan_result)
+          if @plan_job
+            @client.command.plan_finish(
+              plan_job: @plan_job,
+              result: plan_result.value || '',
+              status: plan_result.status
+            )
+          end
+        end
+
+        def build_request(targets, task, arguments, description = nil)
+          body = { task: task.name,
+                   environment: @environment,
+                   noop: arguments['_noop'],
+                   params: arguments.reject { |k, _| k.start_with?('_') },
+                   scope: {
+                     nodes: targets.map(&:host)
+                   } }
+          body[:description] = description if description
+          body[:plan_job] = @plan_job if @plan_job
+          body
+        end
+
+        def run_task(targets, task, arguments, options)
+          body = build_request(targets, task, arguments, options['_description'])
+          @client.run_task(body)
+        end
+
+        def query_inventory(targets)
+          @client.post('inventory', nodes: targets.map(&:host))
+        end
+      end
+    end
+  end
+end

data/lib/bolt/transport/local.rb

@@ -36,11 +36,15 @@ module Bolt
 
       def in_tmpdir(base)
         args = base ? [nil, base] : []
-        Dir.mktmpdir(*args) do |dir|
-          yield dir
-        end
-      rescue StandardError => e
-        raise Bolt::Node::FileError.new("Could not make tempdir: #{e.message}", 'TEMPDIR_ERROR')
+        dir = begin
+                Dir.mktmpdir(*args)
+              rescue StandardError => e
+                raise Bolt::Node::FileError.new("Could not make tempdir: #{e.message}", 'TEMPDIR_ERROR')
+              end
+
+        yield dir
+      ensure
+        FileUtils.remove_entry dir if dir
       end
       private :in_tmpdir
 

data/lib/bolt/transport/local/shell.rb

@@ -19,7 +19,7 @@ module Bolt
           result_output = Bolt::Node::Output.new
           result_output.stdout << stdout unless stdout.nil?
           result_output.stderr << stderr unless stderr.nil?
-          result_output.exit_code = rc.to_i
+          result_output.exit_code = rc.exitstatus
           result_output
         end
       end

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '1.8.1'
+  VERSION = '1.9.0'
 end

data/lib/plan_executor/app.rb

@@ -8,6 +8,7 @@ require 'bolt/inventory'
 require 'bolt/pal'
 require 'bolt/puppetdb'
 require 'plan_executor/applicator'
+require 'plan_executor/executor'
 require 'concurrent'
 require 'json'
 require 'json-schema'
@@ -31,7 +32,7 @@ module PlanExecutor
       @worker = Concurrent::SingleThreadExecutor.new
 
       # Create a basic executor, leave concurrency up to Orchestrator.
-      @executor = executor || Bolt::Executor.new(0, load_config: false)
+      @executor = executor || PlanExecutor::Executor.new(0)
       # Use an empty inventory until we figure out where this data comes from.
       @inventory = Bolt::Inventory.new(nil)
       # TODO: what should max compiles be set to for apply?

data/lib/plan_executor/executor.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+# Used for $ERROR_INFO. This *must* be capitalized!
+require 'English'
+require 'json'
+require 'logging'
+require 'set'
+require 'bolt/result'
+require 'bolt/config'
+require 'bolt/transport/api'
+require 'bolt/notifier'
+require 'bolt/result_set'
+require 'bolt/puppetdb'
+
+module PlanExecutor
+  class Executor
+    attr_reader :noop, :transport
+
+    def initialize(noop = nil)
+      @logger = Logging.logger[self]
+      @plan_logging = false
+      @noop = noop
+      @logger.debug { "Started" }
+      @notifier = Bolt::Notifier.new
+      @transport = Bolt::Transport::Api.new
+    end
+
+    # This handles running the job, catching errors, and turning the result
+    # into a result set
+    def execute(targets)
+      result_array = begin
+                       yield
+                     rescue StandardError => e
+                       @logger.warn(e)
+                       # CODEREVIEW how should we fail if there's an error?
+                       Array(Bolt::Result.from_exception(targets[0], e))
+                     end
+      Bolt::ResultSet.new(result_array)
+    end
+
+    # TODO: Remove in favor of service logging
+    def log_action(description, targets)
+      # When running a plan, info messages like starting a task are promoted to notice.
+      log_method = @plan_logging ? :notice : :info
+      target_str = if targets.length > 5
+                     "#{targets.count} targets"
+                   else
+                     targets.map(&:uri).join(', ')
+                   end
+
+      @logger.send(log_method, "Starting: #{description} on #{target_str}")
+
+      start_time = Time.now
+      results = yield
+      duration = Time.now - start_time
+
+      failures = results.error_set.length
+      plural = failures == 1 ? '' : 's'
+
+      @logger.send(log_method, "Finished: #{description} with #{failures} failure#{plural} in #{duration.round(2)} sec")
+
+      results
+    end
+
+    def log_plan(plan_name)
+      log_method = @plan_logging ? :notice : :info
+      @logger.send(log_method, "Starting: plan #{plan_name}")
+      start_time = Time.now
+
+      results = nil
+      begin
+        results = yield
+      ensure
+        duration = Time.now - start_time
+        @logger.send(log_method, "Finished: plan #{plan_name} in #{duration.round(2)} sec")
+      end
+
+      results
+    end
+
+    def run_command(targets, command, options = {}, &callback)
+      description = options.fetch('_description', "command '#{command}'")
+      log_action(description, targets) do
+        notify = proc { |event| @notifier.notify(callback, event) if callback }
+
+        results = execute(targets) do
+          @transport.batch_command(targets, command, options, &notify)
+        end
+
+        @notifier.shutdown
+        results
+      end
+    end
+
+    def run_script(targets, script, arguments, options = {}, &callback)
+      description = options.fetch('_description', "script #{script}")
+      log_action(description, targets) do
+        notify = proc { |event| @notifier.notify(callback, event) if callback }
+
+        results = execute(targets) do
+          @transport.batch_script(targets, script, arguments, options, &notify)
+        end
+
+        @notifier.shutdown
+        results
+      end
+    end
+
+    def run_task(targets, task, arguments, options = {}, &callback)
+      description = options.fetch('_description', "task #{task.name}")
+      log_action(description, targets) do
+        notify = proc { |event| @notifier.notify(callback, event) if callback }
+
+        arguments['_task'] = task.name
+
+        results = execute(targets) do
+          @transport.batch_task(targets, task, arguments, options, &notify)
+        end
+
+        @notifier.shutdown
+        results
+      end
+    end
+
+    def upload_file(targets, source, destination, options = {}, &callback)
+      description = options.fetch('_description', "file upload from #{source} to #{destination}")
+      log_action(description, targets) do
+        notify = proc { |event| @notifier.notify(callback, event) if callback }
+
+        results = execute(targets) do
+          @transport.batch_upload(targets, source, destination, options, &notify)
+        end
+
+        @notifier.shutdown
+        results
+      end
+    end
+
+    class TimeoutError < RuntimeError; end
+
+    def wait_until_available(targets,
+                             description: 'wait until available',
+                             wait_time: 120,
+                             retry_interval: 1)
+      log_action(description, targets) do
+        begin
+          wait_until(wait_time, retry_interval) { @transport.batch_connected?(targets) }
+          targets.map { |target| Bolt::Result.new(target) }
+        rescue TimeoutError => e
+          targets.map { |target| Bolt::Result.from_exception(target, e) }
+        end
+      end
+    end
+
+    def wait_until(timeout, retry_interval)
+      start = wait_now
+      until yield
+        raise(TimeoutError, 'Timed out waiting for target') if (wait_now - start).to_i >= timeout
+        sleep(retry_interval)
+      end
+    end
+
+    # Plan context doesn't make sense for most transports but it is tightly
+    # coupled with the orchestrator transport since the transport behaves
+    # differently when a plan is running. In order to limit how much this
+    # pollutes the transport API we only handle the orchestrator transport here.
+    # Since we callt this function without resolving targets this will result
+    # in the orchestrator transport always being initialized during plan runs.
+    # For now that's ok.
+    #
+    # In the future if other transports need this or if we want a plan stack
+    # we'll need to refactor.
+    def start_plan(plan_context)
+      @transport.plan_context = plan_context
+      @plan_logging = true
+    end
+
+    def finish_plan(plan_result)
+      @transport.finish_plan(plan_result)
+    end
+
+    def without_default_logging
+      old_log = @plan_logging
+      @plan_logging = false
+      yield
+    ensure
+      @plan_logging = old_log
+    end
+
+    def report_bundled_content(mode, name); end
+
+    def report_function_call(function); end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.9.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-04 00:00:00.000000000 Z
+date: 2019-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -357,6 +357,8 @@ files:
 - lib/bolt/task.rb
 - lib/bolt/task/puppet_server.rb
 - lib/bolt/task/remote.rb
+- lib/bolt/transport/api.rb
+- lib/bolt/transport/api/connection.rb
 - lib/bolt/transport/base.rb
 - lib/bolt/transport/docker.rb
 - lib/bolt/transport/docker/connection.rb
@@ -393,6 +395,7 @@ files:
 - lib/plan_executor/app.rb
 - lib/plan_executor/applicator.rb
 - lib/plan_executor/config.rb
+- lib/plan_executor/executor.rb
 - lib/plan_executor/schemas/run_plan.json
 - libexec/apply_catalog.rb
 - libexec/bolt_catalog