openbolt 5.0.0.rc1 → 5.1.0

data/lib/bolt/transport/orch.rb

@@ -1,271 +0,0 @@
-# frozen_string_literal: true
-
-require 'base64'
-require 'find'
-require 'json'
-require 'pathname'
-require_relative '../../bolt/transport/base'
-require_relative 'orch/connection'
-
-module Bolt
-  module Transport
-    class Orch < Base
-      BOLT_COMMAND_TASK = Struct.new(:name).new('bolt_shim::command').freeze
-      BOLT_SCRIPT_TASK = Struct.new(:name).new('bolt_shim::script').freeze
-      BOLT_UPLOAD_TASK = Struct.new(:name).new('bolt_shim::upload').freeze
-
-      attr_writer :plan_context
-
-      def provided_features
-        ['puppet-agent']
-      end
-
-      def initialize(*args)
-        # lazy-load expensive gem code
-        require 'orchestrator_client'
-
-        @connections = {}
-        super
-      end
-
-      def finish_plan(result)
-        if result.is_a? Bolt::PlanResult
-          @connections.each_value do |conn|
-            conn.finish_plan(result)
-          rescue StandardError => e
-            @logger.trace("Failed to finish plan on #{conn.key}: #{e.message}")
-          end
-        end
-      end
-
-      # It's safe to create connections here for now because the
-      # batches/threads are per connection.
-      def get_connection(conn_opts)
-        key = Connection.get_key(conn_opts)
-        unless (conn = @connections[key])
-          conn = @connections[key] = Connection.new(conn_opts, @plan_context, logger)
-        end
-        conn
-      end
-
-      def process_run_results(targets, results, task_name, position = [])
-        targets_by_name = Hash[targets.map { |t| t.host || t.name }.zip(targets)]
-        results.map do |node_result|
-          target = targets_by_name[node_result['name']]
-          state = node_result['state']
-          result = node_result['result']
-
-          # If it's finished or already has a proper error simply pass it to the
-          # the result otherwise make sure an error is generated
-          if state == 'finished' || (result && result['_error'])
-            if result['_error']
-              unless result['_error'].is_a?(Hash)
-                result['_error'] = { 'kind' => 'puppetlabs.tasks/task-error',
-                                     'issue_code' => 'TASK_ERROR',
-                                     'msg' => result['_error'],
-                                     'details' => {} }
-              end
-
-              result['_error']['details'] ||= {}
-              unless result['_error']['details'].is_a?(Hash)
-                deets = result['_error']['details']
-                result['_error']['details'] = { 'msg' => deets }
-              end
-              file_line = %w[file line].zip(position).to_h.compact
-              result['_error']['details'].merge!(file_line) unless result['_error']['details']['file']
-            end
-
-            Bolt::Result.new(target, value: result, action: 'task', object: task_name)
-          elsif state == 'skipped'
-            details = %w[file line].zip(position).to_h.compact
-            Bolt::Result.new(
-              target,
-              value: { '_error' => {
-                'kind' => 'puppetlabs.tasks/skipped-node',
-                'msg' => "Target #{target.safe_name} was skipped",
-                'details' => details
-              } },
-              action: 'task', object: task_name
-            )
-          else
-            # Make a generic error with a unkown exit_code
-            Bolt::Result.for_task(target, result.to_json, '', 'unknown', task_name, position)
-          end
-        end
-      end
-
-      def batch_command(targets, command, options = {}, position = [], &callback)
-        if options[:env_vars] && !options[:env_vars].empty?
-          raise NotImplementedError, "pcp transport does not support setting environment variables"
-        end
-
-        params = {
-          'command' => command
-        }
-        results = run_task_job(targets,
-                               BOLT_COMMAND_TASK,
-                               params,
-                               options,
-                               position,
-                               &callback)
-        callback ||= proc {}
-        results.map! { |result| unwrap_bolt_result(result.target, result, 'command', command) }
-        results.each do |result|
-          callback.call(type: :node_result, result: result)
-        end
-      end
-
-      def batch_script(targets, script, arguments, options = {}, position = [], &callback)
-        if options[:env_vars] && !options[:env_vars].empty?
-          raise NotImplementedError, "pcp transport does not support setting environment variables"
-        end
-
-        content = File.open(script, &:read)
-        content = Base64.encode64(content)
-        params = {
-          'content' => content,
-          'arguments' => arguments,
-          'name' => Pathname(script).basename.to_s
-        }
-        callback ||= proc {}
-        results = run_task_job(targets, BOLT_SCRIPT_TASK, params, options, position, &callback)
-        results.map! { |result| unwrap_bolt_result(result.target, result, 'script', script) }
-        results.each do |result|
-          callback.call(type: :node_result, result: result)
-        end
-      end
-
-      def pack(directory)
-        # lazy-load expensive gem code
-        require 'minitar'
-        require 'zlib'
-
-        start_time = Time.now
-        io = StringIO.new
-        output = Minitar::Output.new(Zlib::GzipWriter.new(io))
-        Find.find(directory) do |file|
-          next unless File.file?(file)
-
-          tar_path = Pathname.new(file).relative_path_from(Pathname.new(directory))
-          @logger.trace("Packing #{file} to #{tar_path}")
-          stat = File.stat(file)
-          content = File.binread(file)
-          output.tar.add_file_simple(
-            tar_path.to_s,
-            data: content,
-            size: content.size,
-            mode: stat.mode & 0o777,
-            mtime: stat.mtime
-          )
-        end
-
-        duration = Time.now - start_time
-        @logger.trace("Packed upload in #{duration * 1000} ms")
-
-        output.close
-        io.string
-      ensure
-        # Closes both tar and sgz.
-        output&.close
-      end
-
-      def batch_upload(targets, source, destination, options = {}, position = [], &callback)
-        stat = File.stat(source)
-        content = if stat.directory?
-                    pack(source)
-                  else
-                    File.open(source, &:read)
-                  end
-        content = Base64.encode64(content)
-        mode = File.stat(source).mode
-        params = {
-          'path' => destination,
-          'content' => content,
-          'mode' => mode,
-          'directory' => stat.directory?
-        }
-        callback ||= proc {}
-        results = run_task_job(targets, BOLT_UPLOAD_TASK, params, options, position, &callback)
-        results.map! do |result|
-          if result.error_hash
-            result
-          else
-            Bolt::Result.for_upload(result.target, source, destination)
-          end
-        end
-        results.each do |result|
-          callback&.call(type: :node_result, result: result)
-        end
-      end
-
-      def batch_download(targets, *_args)
-        error = {
-          'kind'    => 'bolt/not-supported-error',
-          'msg'     => 'pcp transport does not support downloading files',
-          'details' => {}
-        }
-
-        targets.map do |target|
-          Bolt::Result.new(target, error: error, action: 'download')
-        end
-      end
-
-      def batches(targets)
-        targets.group_by { |target| Connection.get_key(target.options) }.values
-      end
-
-      def run_task_job(targets, task, arguments, options, position)
-        targets.each do |target|
-          yield(type: :node_start, target: target) if block_given?
-        end
-
-        begin
-          # unpack any Sensitive data
-          arguments = unwrap_sensitive_args(arguments)
-          results = get_connection(targets.first.options).run_task(targets, task, arguments, options)
-
-          process_run_results(targets, results, task.name, position)
-        rescue OrchestratorClient::ApiError => e
-          targets.map do |target|
-            Bolt::Result.new(target, error: e.data)
-          end
-        rescue StandardError => e
-          targets.map do |target|
-            Bolt::Result.from_exception(target, e, action: 'task')
-          end
-        end
-      end
-
-      def batch_task(targets, task, arguments, options = {}, position = [], &callback)
-        callback ||= proc {}
-        results = run_task_job(targets, task, arguments, options, position, &callback)
-        results.each do |result|
-          callback.call(type: :node_result, result: result)
-        end
-      end
-
-      def batch_task_with(_targets, _task, _target_mapping, _options = {}, _position = [])
-        raise NotImplementedError, "pcp transport does not support run_task_with()"
-      end
-
-      def batch_connected?(targets)
-        resp = get_connection(targets.first.options).query_inventory(targets)
-        resp['items'].all? { |node| node['connected'] }
-      end
-
-      # run_task generates a result that makes sense for a generic task which
-      # needs to be unwrapped to extract stdout/stderr/exitcode.
-      #
-      def unwrap_bolt_result(target, result, action, obj)
-        if result.error_hash
-          # something went wrong return the failure
-          return result
-        end
-
-        # If we get here, there's no error so we don't need the file or line
-        # number
-        Bolt::Result.for_command(target, result.value, action, obj, [])
-      end
-    end
-  end
-end

data/lib/bolt_server/acl.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails/auth/rack'
-
-module BoltServer
-  class ACL < Rails::Auth::ErrorPage::Middleware
-    class X509Matcher
-      def initialize(options)
-        @options = options.freeze
-      end
-
-      def match(env)
-        certificate = Rails::Auth::X509::Certificate.new(env['puma.peercert'])
-        # This can be extended fairly easily to search OpenSSL::X509::Certificate#extensions for subjectAltNames.
-        @options.all? { |name, value| certificate[name] == value }
-      end
-    end
-
-    def initialize(app, allowlist)
-      acls = []
-      allowlist.each do |entry|
-        acls << {
-          'resources' => [
-            {
-              'method' => 'ALL',
-              'path' => '/.*'
-            }
-          ],
-          'allow_x509_subject' => {
-            'cn' => entry
-          }
-        }
-      end
-      acl = Rails::Auth::ACL.new(acls, matchers: { allow_x509_subject: X509Matcher })
-      mid = Rails::Auth::ACL::Middleware.new(app, acl: acl)
-      super(mid, page_body: 'Access denied')
-    end
-  end
-end

data/lib/bolt_server/base_config.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-require 'hocon'
-require 'bolt/error'
-
-module BoltServer
-  class BaseConfig
-    def config_keys
-      %w[host port ssl-cert ssl-key ssl-ca-cert
-         ssl-cipher-suites loglevel logfile allowlist
-         environments-codedir
-         environmentpath basemodulepath]
-    end
-
-    def env_keys
-      %w[ssl-cert ssl-key ssl-ca-cert loglevel]
-    end
-
-    def defaults
-      { 'host' => '127.0.0.1',
-        'loglevel' => 'warn',
-        'ssl-cipher-suites' => %w[ECDHE-ECDSA-AES256-GCM-SHA384
-                                  ECDHE-RSA-AES256-GCM-SHA384
-                                  ECDHE-ECDSA-CHACHA20-POLY1305
-                                  ECDHE-RSA-CHACHA20-POLY1305
-                                  ECDHE-ECDSA-AES128-GCM-SHA256
-                                  ECDHE-RSA-AES128-GCM-SHA256
-                                  ECDHE-ECDSA-AES256-SHA384
-                                  ECDHE-RSA-AES256-SHA384
-                                  ECDHE-ECDSA-AES128-SHA256
-                                  ECDHE-RSA-AES128-SHA256] }
-    end
-
-    def ssl_keys
-      %w[ssl-cert ssl-key ssl-ca-cert]
-    end
-
-    def required_keys
-      ssl_keys
-    end
-
-    def service_name
-      raise "Method service_name must be defined in the service class"
-    end
-
-    def initialize(config = nil)
-      @data = defaults
-      @data = @data.merge(config.select { |key, _| config_keys.include?(key) }) if config
-      @config_path = nil
-    end
-
-    def load_file_config(path)
-      @config_path = path
-      begin
-        # This lets us get the actual config values without needing to
-        # know the service name
-        parsed_hocon = Hocon.load(path)[service_name]
-      rescue Hocon::ConfigError => e
-        raise "Hocon data in '#{path}' failed to load.\n Error: '#{e.message}'"
-      rescue Errno::EACCES
-        raise "Your user doesn't have permission to read #{path}"
-      end
-
-      raise "Could not find service config at #{path}" if parsed_hocon.nil?
-
-      parsed_hocon = parsed_hocon.select { |key, _| config_keys.include?(key) }
-
-      @data = @data.merge(parsed_hocon)
-    end
-
-    def load_env_config
-      raise "load_env_config should be defined in the service class"
-    end
-
-    def natural?(num)
-      num.is_a?(Integer) && num.positive?
-    end
-
-    def validate
-      required_keys.each do |k|
-        # Handled nested config
-        if k.is_a?(Array)
-          next unless @data.dig(*k).nil?
-        else
-          next unless @data[k].nil?
-        end
-        raise Bolt::ValidationError, "You must configure #{k} in #{@config_path}"
-      end
-
-      unless natural?(@data['port'])
-        raise Bolt::ValidationError, "Configured 'port' must be a valid integer greater than 0"
-      end
-      ssl_keys.each do |sk|
-        unless File.file?(@data[sk]) && File.readable?(@data[sk])
-          raise Bolt::ValidationError, "Configured #{sk} must be a valid filepath"
-        end
-      end
-
-      unless @data['ssl-cipher-suites'].is_a?(Array)
-        raise Bolt::ValidationError, "Configured 'ssl-cipher-suites' must be an array of cipher suite names"
-      end
-
-      unless @data['allowlist'].nil? || @data['allowlist'].is_a?(Array)
-        raise Bolt::ValidationError, "Configured 'allowlist' must be an array of names"
-      end
-    end
-
-    def [](key)
-      @data[key]
-    end
-  end
-end

data/lib/bolt_server/config.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require 'hocon'
-require 'bolt_server/base_config'
-require 'bolt/error'
-
-module BoltServer
-  class Config < BoltServer::BaseConfig
-    def config_keys
-      super + %w[concurrency cache-dir file-server-conn-timeout
-                 file-server-uri environments-codedir
-                 environmentpath basemodulepath builtin-content-dir]
-    end
-
-    def env_keys
-      super + %w[concurrency file-server-conn-timeout file-server-uri]
-    end
-
-    def int_keys
-      %w[concurrency file-server-conn-timeout]
-    end
-
-    def defaults
-      super.merge(
-        'port' => 62658,
-        'concurrency' => 100,
-        'cache-dir' => "/opt/puppetlabs/server/data/bolt-server/cache",
-        'file-server-conn-timeout' => 120
-      )
-    end
-
-    def required_keys
-      super + %w[file-server-uri]
-    end
-
-    def service_name
-      'bolt-server'
-    end
-
-    def load_env_config
-      env_keys.each do |key|
-        transformed_key = "BOLT_#{key.tr('-', '_').upcase}"
-        next unless ENV.key?(transformed_key)
-        @data[key] = if int_keys.include?(key)
-                       ENV[transformed_key].to_i
-                     else
-                       ENV[transformed_key]
-                     end
-      end
-    end
-
-    def validate
-      super
-
-      unless natural?(@data['concurrency'])
-        raise Bolt::ValidationError, "Configured 'concurrency' must be a positive integer"
-      end
-
-      unless natural?(@data['file-server-conn-timeout'])
-        raise Bolt::ValidationError, "Configured 'file-server-conn-timeout' must be a positive integer"
-      end
-    end
-  end
-end

data/lib/bolt_server/file_cache.rb

@@ -1,200 +0,0 @@
-# frozen_string_literal: true
-
-require 'concurrent/atomic/read_write_lock'
-require 'concurrent/executor/single_thread_executor'
-require 'concurrent/promise'
-require 'concurrent/timer_task'
-require 'digest'
-require 'fileutils'
-require 'net/http'
-require 'logging'
-require 'timeout'
-
-require 'bolt/error'
-
-module BoltServer
-  class FileCache
-    class Error < Bolt::Error
-      def initialize(msg)
-        super(msg, 'bolt-server/file-cache-error')
-      end
-    end
-
-    PURGE_TIMEOUT = 60 * 60
-    PURGE_INTERVAL = 24 * PURGE_TIMEOUT
-    PURGE_TTL = 7 * PURGE_INTERVAL
-
-    def initialize(config,
-                   executor: Concurrent::SingleThreadExecutor.new,
-                   purge_interval: PURGE_INTERVAL,
-                   purge_timeout: PURGE_TIMEOUT,
-                   purge_ttl: PURGE_TTL,
-                   cache_dir_mutex: Concurrent::ReadWriteLock.new,
-                   do_purge: true)
-      @executor = executor
-      @cache_dir = config['cache-dir']
-      @config = config
-      @logger = Bolt::Logger.logger(self)
-      @cache_dir_mutex = cache_dir_mutex
-
-      if do_purge
-        @purge = Concurrent::TimerTask.new(execution_interval: purge_interval,
-                                           run_now: true) { expire(purge_ttl, purge_timeout) }
-        @purge.execute
-      end
-    end
-
-    def tmppath
-      File.join(@cache_dir, 'tmp')
-    end
-
-    def setup
-      FileUtils.mkdir_p(@cache_dir)
-      FileUtils.mkdir_p(tmppath)
-      self
-    end
-
-    def ssl_cert
-      @ssl_cert ||= File.read(@config['ssl-cert'])
-    end
-
-    def ssl_key
-      @ssl_key ||= File.read(@config['ssl-key'])
-    end
-
-    def client
-      # rubocop:disable Naming/VariableNumber
-      @client ||= begin
-        uri = URI(@config['file-server-uri'])
-        https = Net::HTTP.new(uri.host, uri.port)
-        https.use_ssl = true
-        https.ssl_version = :TLSv1_2
-        https.ca_file = @config['ssl-ca-cert']
-        https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
-        https.key = OpenSSL::PKey::RSA.new(ssl_key)
-        https.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        https.open_timeout = @config['file-server-conn-timeout']
-        https
-      end
-      # rubocop:enable Naming/VariableNumber
-    end
-
-    def request_file(path, params, file)
-      uri = "#{@config['file-server-uri'].chomp('/')}#{path}"
-      uri = URI(uri)
-      uri.query = URI.encode_www_form(params)
-
-      req = Net::HTTP::Get.new(uri)
-
-      begin
-        client.request(req) do |resp|
-          if resp.code != "200"
-            msg = "Failed to download file: #{resp.body}"
-            @logger.warn resp.body
-            raise Error, msg
-          end
-          resp.read_body do |chunk|
-            file.write(chunk)
-          end
-        end
-      rescue StandardError => e
-        if e.is_a?(Bolt::Error)
-          raise e
-        else
-          @logger.warn e
-          raise Error, "Failed to download file: #{e.message}"
-        end
-      end
-    ensure
-      file.close
-    end
-
-    def check_file(file_path, sha)
-      File.exist?(file_path) && Digest::SHA256.file(file_path) == sha
-    end
-
-    def serial_execute(&block)
-      promise = Concurrent::Promise.new(executor: @executor, &block).execute.wait
-      raise promise.reason if promise.rejected?
-      promise.value
-    end
-
-    # Create a cache dir if necessary and update it's last write time. Returns the dir.
-    # Acquires @cache_dir_mutex to ensure we don't try to purge the directory at the same time.
-    # Uses the directory mtime because it's simpler to ensure the directory exists and update
-    # mtime in a single place than with a file in a directory that may not exist.
-    def create_cache_dir(sha)
-      file_dir = File.join(@cache_dir, sha)
-      @cache_dir_mutex.with_read_lock do
-        # mkdir_p doesn't error if the file exists
-        FileUtils.mkdir_p(file_dir, mode: 0o750)
-        FileUtils.touch(file_dir)
-      end
-      file_dir
-    end
-
-    def download_file(file_path, sha, uri)
-      if check_file(file_path, sha)
-        @logger.debug("File was downloaded while queued: #{file_path}")
-        return file_path
-      end
-
-      @logger.debug("Downloading file: #{file_path}")
-
-      tmpfile = Tempfile.new(sha, tmppath)
-      request_file(uri['path'], uri['params'], tmpfile)
-
-      if Digest::SHA256.file(tmpfile.path) == sha
-        # mv doesn't error if the file exists
-        FileUtils.mv(tmpfile.path, file_path)
-        @logger.debug("Downloaded file: #{file_path}")
-        file_path
-      else
-        msg = "Downloaded file did not match checksum for: #{file_path}"
-        @logger.warn msg
-        raise Error, msg
-      end
-    end
-
-    # If the file doesn't exist or is invalid redownload it
-    # This downloads, validates and moves into place
-    def update_file(file_data)
-      sha = file_data['sha256']
-      file_dir = create_cache_dir(file_data['sha256'])
-      file_path = File.join(file_dir, File.basename(file_data['filename']))
-      if check_file(file_path, sha)
-        @logger.debug("Using prexisting file: #{file_path}")
-        return file_path
-      end
-
-      @logger.debug("Queueing download for: #{file_path}")
-      serial_execute { download_file(file_path, sha, file_data['uri']) }
-    end
-
-    def expire(purge_ttl, purge_timeout)
-      expired_time = Time.now - purge_ttl
-      Timeout.timeout(purge_timeout) do
-        @cache_dir_mutex.with_write_lock do
-          Dir.glob(File.join(@cache_dir, '*')).select { |f| File.directory?(f) }.each do |dir|
-            if (mtime = File.mtime(dir)) < expired_time && dir != tmppath
-              @logger.debug("Removing #{dir}, last used at #{mtime}")
-              FileUtils.remove_dir(dir)
-            end
-          end
-        end
-      end
-    end
-
-    def get_cached_project_file(versioned_project, file_name)
-      file_dir = create_cache_dir(versioned_project)
-      file_path = File.join(file_dir, file_name)
-      serial_execute { File.read(file_path) if File.exist?(file_path) }
-    end
-
-    def cache_project_file(versioned_project, file_name, data)
-      file_dir = create_cache_dir(versioned_project)
-      file_path = File.join(file_dir, file_name)
-      serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
-    end
-  end
-end

data/lib/bolt_server/request_error.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require 'bolt/error'
-
-module BoltServer
-  class RequestError < Bolt::Error
-    def initialize(msg, details = {})
-      super(msg, 'bolt-server/request-error', details)
-    end
-  end
-end