agentless-catalog-executor 0.9.0

data/README.md

@@ -0,0 +1,23 @@
+# Agentless::Catalog::Executor
+
+## Installation
+
+ACE is installed as pe-ace-server as part of Puppet Enterprise.
+
+## Usage
+
+For development or experimenting, run the puma server to get an instance started:
+
+```
+ACE_CONF=config/local.conf bundle exec puma -C config/transport_tasks_config.rb
+```
+
+## Development
+
+As ACE is dependent on a PuppetServer, there is a docker-compose file within the `spec/` directory which is advisable to run first to ensure that the certs and keys are valid before running the ACE service, more information can be found in the [docker documentation](developer-docs/docker).
+
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). Released gems will eventually be consumed by [ace-vanagon](https://github.com/puppetlabs/ace-vanagon) and promoted into PE.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/puppetlabs/ace. See the `.travis.yml` file for checks to run on your code before submitting. Please always include tests with your changes.

data/Rakefile

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['--display-cop-names']
+end
+
+task default: %i[rubocop spec license_finder]
+
+#### RSPEC ####
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.exclude_pattern = 'spec/acceptance/**/*_spec.rb'
+end
+
+namespace :spec do
+  desc 'Run RSpec code examples with coverage collection'
+  task :coverage do
+    ENV['COVERAGE'] = 'yes'
+    Rake::Task['spec'].execute
+  end
+end
+
+#### LICENSE_FINDER ####
+desc 'Check for unapproved licenses in dependencies'
+task(:license_finder) do
+  unless system('license_finder --decisions-file=.dependency_decisions.yml')
+    raise(StandardError, 'Unapproved license(s) found on dependencies')
+  end
+end
+
+#### CHANGELOG ####
+begin
+  require 'github_changelog_generator/task'
+  GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+    require 'ace/version'
+    config.future_release = "v#{ACE::VERSION}"
+    config.header = "# Changelog\n\n" \
+      "All significant changes to this repo will be summarized in this file.\n"
+    # config.include_labels = %w[enhancement bug]
+    config.user = 'puppetlabs'
+    config.project = 'ace'
+  end
+rescue LoadError
+  desc 'Install github_changelog_generator to get access to automatic changelog generation'
+  task :changelog do
+    raise 'Install github_changelog_generator to get access to automatic changelog generation'
+  end
+end

data/acceptance/.gitignore

@@ -0,0 +1,10 @@
+.vagrant
+.bundle
+Gemfile.lock
+local_options.rb
+log
+junit
+id_rsa-acceptance
+id_rsa-acceptance.pub
+merged_options.rb
+tmp

data/acceptance/Gemfile

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+source ENV['GEM_SOURCE'] || "https://rubygems.org"
+
+def location_for(place, fake_version = nil)
+  git_match = place.match(/^(git:[^#]*)#(.*)/)
+  file_match = place.match(%r{^file://(.*)})
+  if git_match
+    git, branch = git_match.captures
+    [fake_version, { git: git, branch: branch, require: false }].compact
+  elsif file_match
+    file_path = file_match[1]
+    ['>= 0', { path: File.expand_path(file_path), require: false }]
+  else
+    [place, { require: false }]
+  end
+end
+
+gem "beaker", *location_for(ENV['BEAKER_VERSION'] || "~> 3.10")
+gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || "~> 0.2")
+gem "beaker-hostgenerator", *location_for(ENV['BEAKER_HOSTGENERATOR_VERSION'] || "~> 1.0")
+gem 'beaker-puppet', *location_for(ENV['BEAKER_PUPPET_VERSION'] || '~> 0.15')
+gem 'rake', "~> 12.1"
+gem 'rototiller'
+
+if File.exist? "Gemfile.local"
+  eval_gemfile "Gemfile.local"
+end

data/acceptance/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'rototiller'
+require_relative 'lib/acceptance/ace_setup_helper'
+
+# rubocop:disable Style/MixinUsage
+extend Acceptance::AceSetupHelper
+# rubocop:enable Style/MixinUsage
+
+namespace :test do
+  desc "Run ace acceptance tests against a published gem"
+
+  desc "Run ace acceptance tests against a git repo"
+
+  desc "Run ace acceptance tests against a package"
+end

data/acceptance/config/gem/options.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+{
+  pre_suite: [],
+  load_path: './lib/acceptance'
+}

data/acceptance/config/git/options.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+{
+  pre_suite: [],
+  load_path: './lib/acceptance',
+  ssh: { forward_agent: false }
+}

data/acceptance/config/package/options.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+{
+  pre_suite: [],
+  load_path: './lib/acceptance'
+}

data/acceptance/lib/acceptance/ace_command_helper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Acceptance
+  module AceCommandHelper
+    # A helper to build a bolt command used in acceptance testing
+    # @param [Beaker::Host] host the host to execute the command on
+    # @param [String] command the command to execute on the bolt SUT
+    # @param [Hash] flags the command flags to append to the command
+    # @option flags [String] '--nodes' the nodes to run on
+    # @option flags [String] '--user' the user to run the command as
+    # @option flags [String] '--password' the password for the user
+    # @option flags [nil] '--no-host-key-check' specify nil to use
+    # @option flags [nil] '--no-ssl' specify nil to use
+    # @param [Hash] opts the options hash for this method
+    def bolt_command_on(host, command, flags = {}, opts = {})
+      bolt_command = command.dup
+      flags.each { |k, v| bolt_command << " #{k} #{v}" }
+
+      case host['platform']
+      when /windows/
+        execute_powershell_script_on(host, bolt_command, opts)
+      when /osx/
+        # Ensure Bolt runs with UTF-8 under macOS. Otherwise we get issues with
+        # UTF-8 content in task results.
+        env = 'source /etc/profile  ~/.bash_profile ~/.bash_login ~/.profile && env LANG=en_US.UTF-8'
+        on(host, env + ' ' + bolt_command)
+      else
+        on(host, bolt_command, opts)
+      end
+    end
+
+    def default_boltdir
+      @default_boltdir ||= begin
+        query = bolt['platform'] =~ /windows/ ? 'cygpath -m $(printenv HOME)' : 'printenv HOME'
+        home = on(bolt, query).stdout.chomp
+        File.join(home, '.puppetlabs/bolt')
+      end
+    end
+
+    def modulepath(extra)
+      case bolt['platform']
+      when /windows/
+        "\"#{default_boltdir}/modules;#{extra}\""
+      else
+        "#{default_boltdir}/modules:#{extra}"
+      end
+    end
+  end
+end

data/acceptance/lib/acceptance/ace_setup_helper.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Acceptance
+  module AceSetupHelper
+    def ssh_user
+      ENV['SSH_USER'] || 'root'
+    end
+
+    def ssh_password
+      ENV['SSH_PASSWORD'] || 'bolt_secret_password'
+    end
+
+    def winrm_user
+      ENV['WINRM_USER'] || 'Administrator'
+    end
+
+    def winrm_password
+      ENV['WINRM_PASSWORD'] || 'bolt_secret_password'
+    end
+
+    def gem_version
+      ENV['GEM_VERSION'] || '> 0.1.0'
+    end
+
+    def gem_source
+      ENV['GEM_SOURCE'] || 'https://rubygems.org'
+    end
+
+    def git_server
+      ENV['GIT_SERVER'] || 'https://github.com'
+    end
+
+    def git_fork
+      ENV['GIT_FORK'] || 'puppetlabs/bolt'
+    end
+
+    def git_branch
+      ENV['GIT_BRANCH'] || 'master'
+    end
+
+    def git_sha
+      ENV['GIT_SHA'] || ''
+    end
+  end
+end

data/agentless-catalog-executor.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "ace/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "agentless-catalog-executor"
+  spec.version       = ACE::VERSION
+  spec.authors       = ["David Schmitt"]
+  spec.email         = ["david.schmitt@puppet.com"]
+
+  spec.summary       = 'ACE lets you run remote tasks and catalogs using puppet and bolt.'
+  spec.homepage      = "https://github.com/puppetlabs/agentless-catalog-executor"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "bolt", "~> 1.15"
+
+  # server-side dependencies cargo culted from https://github.com/puppetlabs/bolt/blob/4418da408643aa7eb5ed64f4c9704b941ea878dc/Gemfile#L10-L16
+  spec.add_dependency "hocon", ">= 1.2.5"
+  spec.add_dependency "json-schema", ">= 2.8.0"
+  spec.add_dependency "puma", ">= 3.12.0"
+  spec.add_dependency "rack", ">= 2.0.5"
+  spec.add_dependency "rails-auth", ">= 2.1.4"
+  spec.add_dependency "sinatra", ">= 2.0.4"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rack-test", "~> 1.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 0.50"
+end

data/config/docker.conf

@@ -0,0 +1,9 @@
+ace-server: {
+    ssl-cert: "spec/volumes/puppet/ssl/certs/aceserver.pem"
+    ssl-key: "spec/volumes/puppet/ssl/private_keys/aceserver.pem"
+    ssl-ca-cert: "spec/volumes/puppet/ssl/certs/ca.pem"
+    ssl-ca-crls: "spec/volumes/puppet/ssl/certs/crl.pem"
+    puppet-server-uri: "https://spec_puppet_1:8140"
+    loglevel: debug
+    host: "0.0.0.0"
+}

data/config/local.conf

@@ -0,0 +1,9 @@
+ace-server: {
+    ssl-cert: "spec/volumes/puppet/ssl/certs/aceserver.pem"
+    ssl-key: "spec/volumes/puppet/ssl/private_keys/aceserver.pem"
+    ssl-ca-cert: "spec/volumes/puppet/ssl/certs/ca.pem"
+    ssl-ca-crls: "spec/volumes/puppet/ssl/ca/ca_crl.pem"
+    puppet-server-uri: "https://0.0.0.0:8140"
+    cache-dir: "tmp/"
+    loglevel: "debug"
+}

data/config/transport_tasks_config.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+####################################################
+## DO NOT EDIT THIS FILE                          ##
+## Use /etc/puppetlabs/ace/conf.d/ace-server.conf ##
+## to configure the sinatra server                ##
+####################################################
+
+require 'ace/transport_app'
+require 'ace/config'
+require 'bolt_server/acl'
+require 'bolt/logger'
+
+Bolt::Logger.initialize_logging
+
+config_path = ENV['ACE_CONF'] || '/etc/puppetlabs/ace-server/conf.d/ace-server.conf'
+
+config = ACE::Config.new
+config.load_file_config(config_path)
+config.load_env_config
+config.make_compatible
+config.validate
+
+Logging.logger[:root].add_appenders Logging.appenders.stderr(
+  'console',
+  layout: Bolt::Logger.default_layout,
+  level: config['loglevel']
+)
+
+if config['logfile']
+  stdout_redirect config['logfile'], config['logfile'], true
+end
+
+bind_addr = +"ssl://#{config['host']}:#{config['port']}?"
+bind_addr << "cert=#{config['ssl-cert']}"
+bind_addr << "&key=#{config['ssl-key']}"
+bind_addr << "&ca=#{config['ssl-ca-cert']}"
+bind_addr << "&verify_mode=force_peer"
+bind_addr << "&ssl_cipher_filter=#{config['ssl-cipher-suites'].join(':')}"
+bind bind_addr
+
+threads 0, config['concurrency']
+
+impl = ACE::TransportApp.new(config)
+unless config['whitelist'].nil?
+  impl = BoltServer::ACL.new(impl, config['whitelist'])
+end
+
+app impl

data/developer-docs/api.md

@@ -0,0 +1,139 @@
+# ACE API
+
+## Overview
+ACE provides 2 APIs to enable the execution of Tasks and Catalog compilation for a remote target.
+
+## API Endpoints
+Each API endpoint accepts a request as described below. The request body must be a JSON object.
+
+### POST /run_task
+- `target`: [RSAPI Transport Object](#rsapi-transport-object), *required* - Target information to run task on.
+- `task`: [Task Object](#task-object), *required* - Task to run on target.
+- `parameters`: Object, *optional* - JSON formatted parameters to be provided to task.
+
+For example, the following runs the 'commit' task on `fw.example.net`:
+```
+{
+  "target":{
+    "remote-transport":"panos",
+    "host":"fw.example.net",
+    "user":"foo",
+    "password":"wibble"
+  },
+  "task":{
+    "metadata":{},
+    "name":"panos::commit",
+    "files":[
+      {
+        "filename":"commit.rb",
+        "sha256":"c5abefbdecee006bd65ef6f625e73f0ebdd1ef3f1b8802f22a1b9644a516ce40",
+        "size_bytes":640,
+        "uri":{
+          "path":"/puppet/v3/file_content/tasks/panos/commit.rb",
+          "params":{
+            "environment":"production"
+          }
+        }
+      }
+    ]
+  },
+  "parameters":{
+    "message":"Hello world"
+  }
+}
+```
+
+#### Response
+If the task runs the response will have status 200.
+The response will be a standard bolt Result JSON object.
+
+
+### POST /execute_catalog
+- `target`: [RSAPI Transport Object](#rsapi-transport-object), *required* - Target information to execute the catalog on.
+- `compiler`: [Compiler Request Object](#compiler-request-object), *required* - Details on the requested compile.
+
+For example, the following will compile and execute a catalog on fw.example.net:
+```
+{
+  "target":{
+    "remote-transport":"panos",
+    "host":"fw.example.net",
+    "user":"foo",
+    "password":"wibble"
+  },
+  "compiler":{
+    "certname":"fw.example.net",
+    "environment":"development",
+    "transaction_uuid":"<uuid string>",
+    "job_id":"<id string>"
+  }
+}
+```
+
+For pre-Transport devices (like F5), a uri can be sent:
+
+```
+{
+  "target":{
+    "remote-transport":"f5",
+    "uri":"https://foo:wibble@f5.example.net/"
+  },
+  "compiler":{
+    "certname":"f5.example.net",
+    "environment":"development",
+    "transaction_uuid":"<uuid string>",
+    "job_id":"<id string>"
+  }
+}
+```
+
+#### Response
+TBD based on orchestrator's needs for feedback.
+
+## Data Object Definitions
+
+### RSAPI Transport Object
+The `target` is a JSON object which reflects the schema of the `remote-transport`.
+e.g. If `remote-transport` is `panos`, the object should validate against the panos transport schema.
+
+Read more about [Transports](https://github.com/puppetlabs/puppet-resource_api#remote-resources) in the Resource API README. The `target` will contain both connection info and bolt's keywords for connection management.
+
+### Compiler Request Object
+The `compiler` is a JSON object which contains parameters regarding the compilation of the catalog for this request. It contains four attributes that have the same definition as the attributes of the same name in the [puppet server catalog API](https://github.com/puppetlabs/puppetserver/blob/master/documentation/puppet-api/v4/catalog.markdown):
+
+* `certname`
+* `environment`
+* `transaction_uuid`
+* `job_id`
+
+### Task Object
+This is a copy of [bolt's task object](https://github.com/puppetlabs/bolt/blob/master/developer-docs/bolt-api-servers.md#task-object)
+
+
+## Running ACE in a container
+*Recommended*
+
+From your checkout of ACE start the docker-compose to run ACE
+
+```
+docker-compose up -d --build
+```
+
+You can now make a curl request to ACE, which should respond with 'OK':
+
+```
+curl -X POST http://0.0.0.0:44633/check
+```
+
+## Running from source
+
+From your checkout of ACE run
+
+```
+bundle exec puma -p 44633 -C puma_config.rb
+```
+
+You can now make a curl request to ACE, which should respond with 'OK':
+```
+curl -X POST http://0.0.0.0:44633/check
+```