agentless-catalog-executor 0.9.0

data/lib/ace/schemas/ace-execute_catalog.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "ACE execute_catalog request",
+  "description": "POST /execute_catalog request schema for ACE",
+  "type": "object",
+  "properties": {
+    "target": {
+      "type": "object",
+      "description": "Contains the Transport schema to connect to the remote target",
+      "properties": {
+        "remote-transport": {
+          "type": "string",
+          "description": "The name of the transport being used"
+        }
+      },
+      "additionalProperties": true,
+      "required": ["remote-transport"]
+    },
+    "compiler": {
+      "type": "object",
+      "description": "Contains additional information to compile the catalog",
+      "properties": {
+        "certname": {
+          "type": "string",
+          "description": "The certname of the target"
+        },
+        "environment": {
+          "type": "string",
+          "description": "The name of the environment for which to compile the catalog."
+        },
+        "transaction_uuid": {
+          "type": "string",
+          "description": "The id for tracking the catalog compilation and report submission."
+        },
+        "job_id": {
+          "type": "string",
+          "description": "The id of the orchestrator job that triggered this run."
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "certname",
+        "environment"
+      ]
+    }
+  },
+  "required": ["target", "compiler"]
+}

data/lib/ace/schemas/ace-run_task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "ACE run_task request",
+  "description": "POST /run_task request schema for ACE",
+  "type": "object",
+  "properties": {
+    "target": {
+      "type": "object",
+      "description": "Contains the Transport schema to connect to the remote target",
+      "properties": {
+        "remote-transport": {
+          "type": "string",
+          "description": "The name of the transport being used"
+        },
+        "run-on": {
+          "type": "string",
+          "description": ""
+        }
+      },
+      "additionalProperties": true,
+      "required": ["remote-transport"]
+    },
+    "task": { "$ref": "file:task"},
+    "parameters": {
+      "type": "object",
+      "description": "JSON formatted parameters to be provided to task"
+    }
+  },
+  "required": ["target", "task"]
+}

data/lib/ace/schemas/task.json

@@ -0,0 +1,74 @@
+{
+  "id": "file:task",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Task",
+  "description": "Task schema for bolt-server",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string",
+      "description": "Task name"
+    },
+    "metadata": {
+      "type": "object",
+      "description": "The metadata object is optional, and contains metadata about the task being run",
+      "properties": {
+        "description": {
+          "type": "string",
+          "description": "The task description from it's metadata"
+        },
+        "parameters": {
+          "type": "object",
+          "description": "Object whose keys are parameter names, and values are objects",
+          "properties": {
+            "description": {
+              "type": "string",
+              "description": "Parameter description"
+            },
+            "type": {
+              "type": "string",
+              "description": "The type the parameter should accept"
+            },
+            "sensitive": {
+              "description": "Whether the task runner should treat the parameter value as sensitive",
+              "type": "boolean"
+            }
+          }
+        },
+        "input_method": {
+          "type": "string",
+          "enum": ["stdin", "environment", "powershell"],
+          "description": "What input method should be used to pass params to the task"
+        }
+      }
+    },
+    "files": {
+      "type": "array",
+      "description": "Description of task files",
+      "items": {
+        "type": "object",
+        "properties": {
+          "uri": {
+            "type": "object",
+            "description": "Where is the file"
+          },
+          "sha256": {
+            "type": "string",
+            "description": "checksum of file"
+          },
+          "filename": {
+            "type": "string",
+            "description": "Name of file"
+          },
+          "size": {
+            "type": "number",
+            "description": "Size of file"
+          }
+        }
+      },
+      "required": ["filename", "uri", "sha256"]
+    }
+  },
+  "required": ["name", "files"],
+  "additionalProperties": false
+}

data/lib/ace/transport_app.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'ace/error'
+require 'ace/fork_util'
+require 'ace/puppet_util'
+require 'ace/configurer'
+require 'ace/plugin_cache'
+require 'bolt_server/file_cache'
+require 'bolt/executor'
+require 'bolt/inventory'
+require 'bolt/target'
+require 'bolt/task/puppet_server'
+require 'json-schema'
+require 'json'
+require 'sinatra'
+require 'puppet/util/network_device/base'
+
+module ACE
+  class TransportApp < Sinatra::Base
+    def initialize(config = nil)
+      @config = config
+      @executor = Bolt::Executor.new(0, load_config: false)
+      tasks_cache_dir = File.join(@config['cache-dir'], 'tasks')
+      @file_cache = BoltServer::FileCache.new(@config.data.merge('cache-dir' => tasks_cache_dir)).setup
+      environments_cache_dir = File.join(@config['cache-dir'], 'environments')
+      @plugins = ACE::PluginCache.new(environments_cache_dir).setup
+
+      @schemas = {
+        "run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-run_task.json'))),
+        "execute_catalog" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-execute_catalog.json')))
+      }
+      shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
+                                       Addressable::URI.parse("file:task"))
+      JSON::Validator.add_schema(shared_schema)
+
+      ACE::PuppetUtil.init_global_settings(config['ssl-ca-cert'],
+                                           config['ssl-ca-crls'],
+                                           config['ssl-key'],
+                                           config['ssl-cert'],
+                                           config['cache-dir'],
+                                           URI.parse(config['puppet-server-uri']))
+
+      super(nil)
+    end
+
+    # Initialises the puppet target.
+    # @param certname   The certificate name of the target.
+    # @param transport  The transport provider of the target.
+    # @param target     Target connection hash or legacy connection URI
+    # @return [Puppet device instance] Returns Puppet device instance
+    # @raise  [puppetlabs/ace/invalid_param] If nil parameter or no connection detail found
+    # @example Connect to device.
+    #   init_puppet_target('test_device.domain.com', 'panos',  JSON.parse("target":{
+    #                                                                       "remote-transport":"panos",
+    #                                                                       "host":"fw.example.net",
+    #                                                                       "user":"foo",
+    #                                                                       "password":"wibble"
+    #                                                                     }) ) => panos.device
+    def self.init_puppet_target(certname, transport, target)
+      unless target
+        raise ACE::Error.new("There was an error parsing the Puppet target. 'target' not found",
+                             'puppetlabs/ace/invalid_param')
+      end
+      unless certname
+        raise ACE::Error.new("There was an error parsing the Puppet compiler details. 'certname' not found",
+                             'puppetlabs/ace/invalid_param')
+      end
+      unless transport
+        raise ACE::Error.new("There was an error parsing the Puppet target. 'transport' not found",
+                             'puppetlabs/ace/invalid_param')
+      end
+
+      if target['uri']
+        if target['uri'] =~ URI::DEFAULT_PARSER.make_regexp
+          # Correct URL
+          url = target['uri']
+        else
+          raise ACE::Error.new("There was an error parsing the URI of the Puppet target",
+                               'puppetlabs/ace/invalid_param')
+        end
+      else
+        url = Hash[target.map { |(k, v)| [k.to_sym, v] }]
+        url.delete(:"remote-transport")
+      end
+
+      device_struct = Struct.new(:provider, :url, :name, :options)
+      # Return device
+      Puppet::Util::NetworkDevice.init(device_struct.new(transport,
+                                                         url,
+                                                         certname,
+                                                         {}))
+    end
+
+    def scrub_stack_trace(result)
+      if result.dig(:result, '_error', 'details', 'stack_trace')
+        result[:result]['_error']['details'].reject! { |k| k == 'stack_trace' }
+      end
+      if result.dig(:result, '_error', 'details', 'backtrace')
+        result[:result]['_error']['details'].reject! { |k| k == 'backtrace' }
+      end
+      result
+    end
+
+    def validate_schema(schema, body)
+      schema_error = JSON::Validator.fully_validate(schema, body)
+      if schema_error.any?
+        ACE::Error.new("There was an error validating the request body.",
+                       'puppetlabs/ace/schema-error',
+                       schema_error)
+      end
+    end
+
+    # returns a hash of trusted facts that will be used
+    # to request a catalog for the target
+    def self.trusted_facts(certname)
+      # if the certname is a valid FQDN, it will split
+      # it in to the correct hostname.domain format
+      # otherwise hostname will be the certname and domain
+      # will be empty
+      hostname, domain = certname.split('.', 2)
+      trusted_facts = {
+        "authenticated": "remote",
+        "extensions": {},
+        "certname": certname,
+        "hostname": hostname
+      }
+      trusted_facts[:domain] = domain if domain
+      trusted_facts
+    end
+
+    get "/" do
+      200
+    end
+
+    post "/check" do
+      [200, 'OK']
+    end
+
+    # :nocov:
+    if ENV['RACK_ENV'] == 'dev'
+      get '/admin/gc' do
+        GC.start
+        200
+      end
+    end
+
+    get '/admin/gc_stat' do
+      [200, GC.stat.to_json]
+    end
+    # :nocov:
+
+    # run this with "curl -X POST http://0.0.0.0:44633/run_task -d '{}'"
+    post '/run_task' do
+      content_type :json
+
+      begin
+        body = JSON.parse(request.body.read)
+      rescue StandardError => e
+        request_error = {
+          _error: ACE::Error.new(e.message,
+                                 'puppetlabs/ace/request_exception',
+                                 class: e.class, backtrace: e.backtrace)
+        }
+        return [400, request_error.to_json]
+      end
+
+      error = validate_schema(@schemas["run_task"], body)
+      return [400, error.to_json] unless error.nil?
+
+      opts = body['target'].merge('protocol' => 'remote')
+
+      # This is a workaround for Bolt due to the way it expects to receive the target info
+      # see: https://github.com/puppetlabs/bolt/pull/915#discussion_r268280535
+      # Systems calling into ACE will need to determine the nodename/certname and pass this as `name`
+      target = [Bolt::Target.new(body['target']['host'] || body['target']['name'], opts)]
+
+      inventory = Bolt::Inventory.new(nil)
+
+      target.first.inventory = inventory
+
+      task = Bolt::Task::PuppetServer.new(body['task'], @file_cache)
+
+      parameters = body['parameters'] || {}
+
+      result = ForkUtil.isolate do
+        # Since this will only be on one node we can just return the first result
+        results = @executor.run_task(target, task, parameters)
+        scrub_stack_trace(results.first.status_hash)
+      end
+      [200, result.to_json]
+    end
+
+    post '/execute_catalog' do
+      content_type :json
+
+      begin
+        body = JSON.parse(request.body.read)
+      rescue StandardError => e
+        request_error = {
+          _error: ACE::Error.new(e.message,
+                                 'puppetlabs/ace/request_exception',
+                                 class: e.class, backtrace: e.backtrace)
+        }
+        return [400, request_error.to_json]
+      end
+
+      error = validate_schema(@schemas["execute_catalog"], body)
+      return [400, error.to_json] unless error.nil?
+
+      environment = body['compiler']['environment']
+      certname = body['compiler']['certname']
+
+      # TODO: (needs groomed) - proper error handling, errors within the block can be rescued
+      # and handled correctly, ACE::Errors we can handle similar to the task
+      # workflow, errors within the Configuer and not as pleasant - can have
+      # _some_ control over them, especially around status codes from
+      # the /v4/catalog endpoint
+      @plugins.with_synced_libdir(environment, certname) do
+        ACE::TransportApp.init_puppet_target(certname, body['target']['remote-transport'], body['target'])
+        configurer = ACE::Configurer.new(body['compiler']['transaction_uuid'], body['compiler']['job_id'])
+        configurer.run(transport_name: certname,
+                       environment: environment,
+                       network_device: true,
+                       pluginsync: false,
+                       trusted_facts: ACE::TransportApp.trusted_facts(certname))
+      end
+
+      # simulate expected error cases
+      if body['compiler']['certname'] == 'fail.example.net'
+        [200, { _error: {
+          msg: 'catalog compile failed',
+          kind: 'puppetlabs/ace/compile_failed',
+          details: 'upstream api errors go here'
+        } }.to_json]
+      elsif body['compiler']['certname'] == 'credentials.example.net'
+        [200, { _error: {
+          msg: 'target specification invalid',
+          kind: 'puppetlabs/ace/target_spec',
+          details: 'upstream api errors go here'
+        } }.to_json]
+      elsif body['compiler']['certname'] == 'reports.example.net'
+        [200, { _error: {
+          msg: 'report submission failed',
+          kind: 'puppetlabs/ace/reporting_failed',
+          details: 'upstream api errors go here'
+        } }.to_json]
+      else
+        [200, '{}']
+      end
+    end
+  end
+end

data/lib/ace/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ACE
+  VERSION = "0.9.0"
+end

data/lib/puppet/indirector/catalog/certless.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'puppet/resource/catalog'
+require 'puppet/indirector/rest'
+
+module Puppet
+  class Resource
+    class Catalog
+      class Certless < Puppet::Indirector::REST
+        desc "Find certless catalogs over HTTP via REST."
+
+        # Override the REST indirector headers for the certless
+        # catalog indirector as the `puppet/pson` header throws of the
+        # request body and wraps it in a "value":{<request>} which
+        # causes a validation error on the v4/catalog endpoint
+        def headers
+          common_headers = {
+            "Content-Type" => 'text/json',
+            "Accept" => 'application/json',
+            Puppet::Network::HTTP::HEADER_PUPPET_VERSION => Puppet.version
+          }
+
+          add_accept_encoding(common_headers)
+        end
+
+        def find(request)
+          uri = "/puppet/v4/catalog"
+
+          body = {
+            "certname": request.key,
+            "persistence": {
+              "facts": true, "catalog": true
+            },
+            "environment": request.environment.name.to_s,
+            "facts": {
+              "values": request.options[:transport_facts]
+            },
+            "trusted_facts": {
+              "values": request.options[:trusted_facts]
+            },
+            "transaction_uuid": request.options[:transaction_uuid],
+            "job_id": request.options[:job_id],
+            "options": {
+              "prefer_requested_environment": true,
+              "capture_logs": false
+            }
+          }
+
+          response = do_request(request) do |req|
+            http_post(req, uri, body.to_json, headers)
+          end
+
+          if is_http_200?(response)
+            content_type, body = parse_response(response)
+            # the response from the `v4/catalog` endpoint is in the format of
+            # {"catalog": {}} whereas the configurer expects it to be a
+            # flatter structurer, so passing it the catalog contents from the body
+            # is suited as the API is unlikely to change for
+            # this release
+            result = deserialize_find(content_type, JSON.parse(body)['catalog'].to_json)
+            result.name = request.key if result.respond_to?(:name=)
+            result
+
+          elsif is_http_404?(response)
+            return nil unless request.options[:fail_on_404]
+
+            # 404 can get special treatment as the indirector API can not produce a meaningful
+            # reason to why something is not found - it may not be the thing the user is
+            # expecting to find that is missing, but something else (like the environment).
+            # While this way of handling the issue is not perfect, there is at least an error
+            # that makes a user aware of the reason for the failure.
+            #
+            _, body = parse_response(response)
+            msg = format(_("Find %<uri>s resulted in 404 with the message: %<body>s"),
+                         uri: elide(uri, 100),
+                         body: body)
+            raise Puppet::Error, msg
+          end
+        end
+      end
+    end
+  end
+end