bolt 1.1.0 → 1.2.0

data/lib/bolt_server/config.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'hocon'
+require 'bolt/error'
+
+module BoltServer
+  class Config
+    CONFIG_KEYS = ['host', 'port', 'ssl-cert', 'ssl-key', 'ssl-ca-cert',
+                   'ssl-cipher-suites', 'loglevel', 'logfile', 'whitelist', 'concurrency',
+                   'cache-dir', 'file-server-conn-timeout', 'file-server-uri'].freeze
+
+    DEFAULTS = {
+      'host' => '127.0.0.1',
+      'port' => 62658,
+      'ssl-cipher-suites' => ['ECDHE-ECDSA-AES256-GCM-SHA384',
+                              'ECDHE-RSA-AES256-GCM-SHA384',
+                              'ECDHE-ECDSA-CHACHA20-POLY1305',
+                              'ECDHE-RSA-CHACHA20-POLY1305',
+                              'ECDHE-ECDSA-AES128-GCM-SHA256',
+                              'ECDHE-RSA-AES128-GCM-SHA256',
+                              'ECDHE-ECDSA-AES256-SHA384',
+                              'ECDHE-RSA-AES256-SHA384',
+                              'ECDHE-ECDSA-AES128-SHA256',
+                              'ECDHE-RSA-AES128-SHA256'],
+      'loglevel' => 'notice',
+      'concurrency' => 100,
+      'cache-dir' => "/opt/puppetlabs/server/data/bolt-server/cache",
+      'file-server-conn-timeout' => 120
+    }.freeze
+
+    CONFIG_KEYS.each do |key|
+      define_method(key.tr('-', '_').to_sym) do
+        @data[key]
+      end
+    end
+
+    def initialize(config = nil)
+      @data = DEFAULTS.clone
+      @data = @data.merge(config.select { |key, _| CONFIG_KEYS.include?(key) }) if config
+      @config_path = nil
+    end
+
+    def load_config(path)
+      @config_path = path
+      begin
+        parsed_hocon = Hocon.load(path)['bolt-server']
+      rescue Hocon::ConfigError => e
+        raise "Hocon data in '#{path}' failed to load.\n Error: '#{e.message}'"
+      rescue Errno::EACCES
+        raise "Your user doesn't have permission to read #{path}"
+      end
+
+      raise "Could not find bolt-server config at #{path}" if parsed_hocon.nil?
+
+      parsed_hocon = parsed_hocon.select { |key, _| CONFIG_KEYS.include?(key) }
+      @data = @data.merge(parsed_hocon)
+
+      validate
+      self
+    end
+
+    def natural?(num)
+      num.is_a?(Integer) && num.positive?
+    end
+
+    def validate
+      ssl_keys = ['ssl-cert', 'ssl-key', 'ssl-ca-cert']
+      required_keys = ssl_keys + ['file-server-uri']
+
+      required_keys.each do |k|
+        next unless @data[k].nil?
+        raise Bolt::ValidationError, "You must configure #{k} in #{@config_path}"
+      end
+
+      unless natural?(port)
+        raise Bolt::ValidationError, "Configured 'port' must be a valid integer greater than 0"
+      end
+      ssl_keys.each do |sk|
+        unless File.file?(@data[sk]) && File.readable?(@data[sk])
+          raise Bolt::ValidationError, "Configured #{sk} must be a valid filepath"
+        end
+      end
+
+      unless ssl_cipher_suites.is_a?(Array)
+        raise Bolt::ValidationError, "Configured 'ssl-cipher-suites' must be an array of cipher suite names"
+      end
+
+      unless whitelist.nil? || whitelist.is_a?(Array)
+        raise Bolt::ValidationError, "Configured 'whitelist' must be an array of names"
+      end
+
+      unless natural?(concurrency)
+        raise Bolt::ValidationError, "Configured 'concurrency' must be a positive integer"
+      end
+
+      unless natural?(file_server_conn_timeout)
+        raise Bolt::ValidationError, "Configured 'file-server-conn-timeout' must be a positive integer"
+      end
+    end
+
+    def [](key)
+      @data[key]
+    end
+  end
+end

data/lib/bolt_server/file_cache.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+require 'digest'
+require 'fileutils'
+require 'net/http'
+require 'logging'
+
+require 'bolt/error'
+
+module BoltServer
+  class FileCache
+    class Error < Bolt::Error
+      def initialize(msg)
+        super(msg, 'bolt-server/file-cache-error')
+      end
+    end
+
+    PURGE_TIMEOUT = 60 * 60
+    PURGE_INTERVAL = 24 * PURGE_TIMEOUT
+    PURGE_TTL = 7 * PURGE_INTERVAL
+
+    def initialize(config,
+                   executor: Concurrent::SingleThreadExecutor.new,
+                   purge_interval: PURGE_INTERVAL,
+                   purge_timeout: PURGE_TIMEOUT,
+                   purge_ttl: PURGE_TTL)
+      @executor = executor
+      @cache_dir = config.cache_dir
+      @config = config
+      @logger = Logging.logger[self]
+      @cache_dir_mutex = Concurrent::ReadWriteLock.new
+
+      @purge = Concurrent::TimerTask.new(execution_interval: purge_interval,
+                                         timeout_interval: purge_timeout,
+                                         run_now: true) { expire(purge_ttl) }
+      @purge.execute
+    end
+
+    def tmppath
+      File.join(@cache_dir, 'tmp')
+    end
+
+    def setup
+      FileUtils.mkdir_p(@cache_dir)
+      FileUtils.mkdir_p(tmppath)
+      self
+    end
+
+    def ssl_cert
+      @ssl_cert ||= File.read(@config.ssl_cert)
+    end
+
+    def ssl_key
+      @ssl_key ||= File.read(@config.ssl_key)
+    end
+
+    def client
+      @client ||= begin
+                    uri = URI(@config.file_server_uri)
+                    https = Net::HTTP.new(uri.host, uri.port)
+                    https.use_ssl = true
+                    https.ssl_version = :TLSv1_2
+                    https.ca_file = @config.ssl_ca_cert
+                    https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
+                    https.key = OpenSSL::PKey::RSA.new(ssl_key)
+                    https.verify_mode = OpenSSL::SSL::VERIFY_PEER
+                    https.open_timeout = @config.file_server_conn_timeout
+                    https
+                  end
+    end
+
+    def request_file(path, params, file)
+      uri = "#{@config.file_server_uri.chomp('/')}#{path}"
+      uri = URI(uri)
+      uri.query = URI.encode_www_form(params)
+
+      req = Net::HTTP::Get.new(uri)
+
+      begin
+        client.request(req) do |resp|
+          if resp.code != "200"
+            msg = "Failed to download task: #{resp.body}"
+            @logger.warn resp.body
+            raise Error, msg
+          end
+          resp.read_body do |chunk|
+            file.write(chunk)
+          end
+        end
+      rescue StandardError => e
+        if e.is_a?(Bolt::Error)
+          raise e
+        else
+          @logger.warn e
+          raise Error, "Failed to download task: #{e.message}"
+        end
+      end
+    ensure
+      file.close
+    end
+
+    def check_file(file_path, sha)
+      File.exist?(file_path) && Digest::SHA256.file(file_path) == sha
+    end
+
+    def serial_execute(&block)
+      promise = Concurrent::Promise.new(executor: @executor, &block).execute.wait
+      raise promise.reason if promise.state == :rejected
+      promise.value
+    end
+
+    # Create a cache dir if necessary and update it's last write time. Returns the dir.
+    # Acquires @cache_dir_mutex to ensure we don't try to purge the directory at the same time.
+    # Uses the directory mtime because it's simpler to ensure the directory exists and update
+    # mtime in a single place than with a file in a directory that may not exist.
+    def create_cache_dir(sha)
+      file_dir = File.join(@cache_dir, sha)
+      @cache_dir_mutex.with_read_lock do
+        # mkdir_p doesn't error if the file exists
+        FileUtils.mkdir_p(file_dir, mode: 0o750)
+        FileUtils.touch(file_dir)
+      end
+      file_dir
+    end
+
+    def download_file(file_path, sha, uri)
+      if check_file(file_path, sha)
+        @logger.debug("File was downloaded while queued: #{file_path}")
+        return file_path
+      end
+
+      @logger.debug("Downloading file: #{file_path}")
+
+      tmpfile = Tempfile.new(sha, tmppath)
+      request_file(uri['path'], uri['params'], tmpfile)
+
+      if Digest::SHA256.file(tmpfile.path) == sha
+        # mv doesn't error if the file exists
+        FileUtils.mv(tmpfile.path, file_path)
+        @logger.debug("Downloaded file: #{file_path}")
+        file_path
+      else
+        msg = "Downloaded file did not match checksum for: #{file_path}"
+        @logger.warn msg
+        raise Error, msg
+      end
+    end
+
+    # If the file doesn't exist or is invalid redownload it
+    # This downloads, validates and moves into place
+    def update_file(file_data)
+      sha = file_data['sha256']
+      file_dir = create_cache_dir(file_data['sha256'])
+      file_path = File.join(file_dir, File.basename(file_data['filename']))
+      if check_file(file_path, sha)
+        @logger.debug("Using prexisting task file: #{file_path}")
+        return file_path
+      end
+
+      @logger.debug("Queueing download for: #{file_path}")
+      serial_execute { download_file(file_path, sha, file_data['uri']) }
+    end
+
+    def expire(purge_ttl)
+      expired_time = Time.now - purge_ttl
+      @cache_dir_mutex.with_write_lock do
+        Dir.glob(File.join(@cache_dir, '*')).select { |f| File.directory?(f) }.each do |dir|
+          if (mtime = File.mtime(dir)) < expired_time
+            @logger.debug("Removing #{dir}, last used at #{mtime}")
+            FileUtils.remove_dir(dir)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/{bolt_ext → bolt_server}/schemas/task.json

@@ -42,23 +42,33 @@
         }
       }
     },
-    "file": {
-      "type": "object",
-      "description": "File name and content",
-      "properties": {
-        "filename": {
-          "type": "string",
-          "description": "Name of the task file"
-        },
-        "file_content": {
-          "type": "string",
-          "description": "Task's base64 encoded file content"
+    "files": {
+      "type": "array",
+      "description": "Description of task files",
+      "items": {
+        "type": "object",
+        "properties": {
+          "uri": {
+            "type": "object",
+            "description": "Where is the file"
+          },
+          "sha256": {
+            "type": "string",
+            "description": "checksum of file"
+          },
+          "filename": {
+            "type": "string",
+            "description": "Name of file"
+          },
+          "size": {
+            "type": "number",
+            "description": "Size of file"
+          }
         }
       },
-      "required": ["filename", "file_content"],
-      "additionalProperties": false
+      "required": ["filename", "uri", "sha256"]
     }
   },
-  "required": ["name", "file"],
+  "required": ["name", "files"],
   "additionalProperties": false
 }

data/lib/bolt_server/transport_app.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'sinatra'
+require 'bolt'
+require 'bolt/target'
+require 'bolt/task/puppet_server'
+require 'bolt_server/file_cache'
+require 'json'
+require 'json-schema'
+
+module BoltServer
+  class TransportApp < Sinatra::Base
+    # This disables Sinatra's error page generation
+    set :show_exceptions, false
+
+    def initialize(config)
+      @config = config
+      @schemas = {
+        "ssh-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ssh-run_task.json'))),
+        "winrm-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'winrm-run_task.json')))
+      }
+      shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
+                                       Addressable::URI.parse("file:task"))
+      JSON::Validator.add_schema(shared_schema)
+
+      @executor = Bolt::Executor.new(0, load_config: false)
+
+      @file_cache = BoltServer::FileCache.new(@config).setup
+
+      super(nil)
+    end
+
+    get '/' do
+      200
+    end
+
+    if ENV['RACK_ENV'] == 'dev'
+      get '/admin/gc' do
+        GC.start
+        200
+      end
+    end
+
+    get '/admin/gc_stat' do
+      [200, GC.stat.to_json]
+    end
+
+    get '/500_error' do
+      raise 'Unexpected error'
+    end
+
+    post '/ssh/run_task' do
+      content_type :json
+
+      body = JSON.parse(request.body.read)
+      schema_error = JSON::Validator.fully_validate(@schemas["ssh-run_task"], body)
+      return [400, schema_error.join] if schema_error.any?
+
+      opts = body['target']
+      if opts['private-key-content']
+        opts['private-key'] = { 'key-data' => opts['private-key-content'] }
+        opts.delete('private-key-content')
+      end
+
+      target = [Bolt::Target.new(body['target']['hostname'], opts)]
+
+      task = Bolt::Task::PuppetServer.new(body['task'], @file_cache)
+
+      parameters = body['parameters'] || {}
+
+      # Since this will only be on one node we can just return the first result
+      results = @executor.run_task(target, task, parameters)
+      [200, results.first.to_json]
+    end
+
+    post '/winrm/run_task' do
+      content_type :json
+
+      body = JSON.parse(request.body.read)
+      schema_error = JSON::Validator.fully_validate(@schemas["winrm-run_task"], body)
+      return [400, schema_error.join] if schema_error.any?
+
+      opts = body['target'].merge('protocol' => 'winrm')
+
+      target = [Bolt::Target.new(body['target']['hostname'], opts)]
+
+      task = Bolt::Task::PuppetServer.new(body['task'], @file_cache)
+
+      parameters = body['parameters'] || {}
+
+      # Since this will only be on one node we can just return the first result
+      results = @executor.run_task(target, task, parameters)
+      [200, results.first.to_json]
+    end
+
+    error 404 do
+      [404, "Could not find route #{request.path}"]
+    end
+
+    error 500 do
+      e = env['sinatra.error']
+      [500, "500: Unknown error: #{e.message}"]
+    end
+  end
+end

data/lib/bolt_spec/run.rb

@@ -9,7 +9,7 @@ require 'bolt/puppetdb'
 require 'bolt/util'
 
 # This is intended to provide a relatively stable method of executing bolt in process from tests.
-# Currently it provides run_task, run_plan and run_command helpers.
+# Currently it provides run_task, run_plan, run_script and run_command helpers.
 module BoltSpec
   module Run
     def run_task(task_name, targets, params = nil, config: nil, inventory: nil)
@@ -41,6 +41,14 @@ module BoltSpec
       Bolt::Util.walk_keys(result, &:to_s)
     end
 
+    def run_script(script, targets, arguments = nil, options = {}, config: nil, inventory: nil)
+      result = BoltRunner.with_runner(config, inventory) do |runner|
+        runner.run_script(script, targets, arguments, options)
+      end
+      result = result.to_a
+      Bolt::Util.walk_keys(result, &:to_s)
+    end
+
     class BoltRunner
       # Creates a temporary boltdir so no settings are picked up
       # WARNING: puppetdb config and orch config which do not use the boltdir may
@@ -93,6 +101,12 @@ module BoltSpec
         targets = inventory.get_targets(targets)
         executor.run_command(targets, command, params || {})
       end
+
+      def run_script(script, targets, arguments = nil, options = {})
+        executor = Bolt::Executor.new(config.concurrency, @analytics)
+        targets = inventory.get_targets(targets)
+        executor.run_script(targets, script, arguments, options)
+      end
     end
   end
 end

data/libexec/bolt_catalog

@@ -34,7 +34,7 @@ require 'json'
 command = ARGV[0]
 if command == "parse"
   code = File.open(ARGV[1], &:read)
-  puts JSON.pretty_generate(Bolt::Catalog.new.generate_ast(code))
+  puts JSON.pretty_generate(Bolt::Catalog.new.generate_ast(code, ARGV[1]))
 elsif command == "compile"
   request = if ARGV[1]
               File.open(ARGV[1]) { |fh| JSON.parse(fh.read) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-16 00:00:00.000000000 Z
+date: 2018-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: CFPropertyList
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -307,6 +321,7 @@ files:
 - lib/bolt/result_set.rb
 - lib/bolt/target.rb
 - lib/bolt/task.rb
+- lib/bolt/task/puppet_server.rb
 - lib/bolt/transport/base.rb
 - lib/bolt/transport/local.rb
 - lib/bolt/transport/local/shell.rb
@@ -320,12 +335,13 @@ files:
 - lib/bolt/util/puppet_log_level.rb
 - lib/bolt/version.rb
 - lib/bolt_ext/puppetdb_inventory.rb
-- lib/bolt_ext/schemas/ssh-run_task.json
-- lib/bolt_ext/schemas/task.json
-- lib/bolt_ext/schemas/winrm-run_task.json
-- lib/bolt_ext/server.rb
-- lib/bolt_ext/server_acl.rb
-- lib/bolt_ext/server_config.rb
+- lib/bolt_server/acl.rb
+- lib/bolt_server/config.rb
+- lib/bolt_server/file_cache.rb
+- lib/bolt_server/schemas/ssh-run_task.json
+- lib/bolt_server/schemas/task.json
+- lib/bolt_server/schemas/winrm-run_task.json
+- lib/bolt_server/transport_app.rb
 - lib/bolt_spec/plans.rb
 - lib/bolt_spec/plans/mock_executor.rb
 - lib/bolt_spec/run.rb

data/lib/bolt_ext/server.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-require 'sinatra'
-require 'bolt'
-require 'bolt/target'
-require 'bolt/task'
-require 'json'
-require 'json-schema'
-
-class TransportAPI < Sinatra::Base
-  # This disables Sinatra's error page generation
-  set :show_exceptions, false
-
-  def initialize(app = nil)
-    @schemas = {
-      "ssh-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ssh-run_task.json'))),
-      "winrm-run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'winrm-run_task.json')))
-    }
-    shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
-                                     Addressable::URI.parse("file:task"))
-    JSON::Validator.add_schema(shared_schema)
-
-    @executor = Bolt::Executor.new(0, load_config: false)
-
-    super(app)
-  end
-
-  get '/' do
-    200
-  end
-
-  if ENV['RACK_ENV'] == 'dev'
-    get '/admin/gc' do
-      GC.start
-      200
-    end
-  end
-
-  get '/admin/gc_stat' do
-    [200, GC.stat.to_json]
-  end
-
-  get '/500_error' do
-    raise 'Unexpected error'
-  end
-
-  post '/ssh/run_task' do
-    content_type :json
-
-    body = JSON.parse(request.body.read)
-    schema_error = JSON::Validator.fully_validate(@schemas["ssh-run_task"], body)
-    return [400, schema_error.join] if schema_error.any?
-
-    # CODEREVIEW: the schema is additionalProperties false do we need this?
-    keys = %w[user password port connect-timeout run-as-command run-as
-              tmpdir host-key-check known-hosts-content private-key-content sudo-password
-              tty]
-    opts = body['target'].select { |k, _| keys.include? k }
-
-    if opts['private-key-content']
-      opts['private-key'] = { 'key-data' => opts['private-key-content'] }
-      opts.delete('private-key-content')
-    end
-
-    target = [Bolt::Target.new(body['target']['hostname'], opts)]
-    task = Bolt::Task.new(body['task'])
-    parameters = body['parameters'] || {}
-
-    # Since this will only be on one node we can just return the first result
-    results = @executor.run_task(target, task, parameters)
-    [200, results.first.to_json]
-  end
-
-  post '/winrm/run_task' do
-    content_type :json
-
-    body = JSON.parse(request.body.read)
-    schema_error = JSON::Validator.fully_validate(@schemas["winrm-run_task"], body)
-    return [400, schema_error.join] if schema_error.any?
-
-    keys = %w[user password port connect-timeout ssl ssl-verify tmpdir cacert extensions]
-    opts = body['target'].select { |k, _| keys.include? k }
-    opts['protocol'] = 'winrm'
-    target = [Bolt::Target.new(body['target']['hostname'], opts)]
-    task = Bolt::Task.new(body['task'])
-    parameters = body['parameters'] || {}
-
-    # Since this will only be on one node we can just return the first result
-    results = @executor.run_task(target, task, parameters)
-    [200, results.first.to_json]
-  end
-
-  error 404 do
-    [404, "Could not find route #{request.path}"]
-  end
-
-  error 500 do
-    e = env['sinatra.error']
-    [500, "500: Unknown error: #{e.message}"]
-  end
-end

data/lib/bolt_ext/server_acl.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails/auth/rack'
-
-class TransportACL < Rails::Auth::ErrorPage::Middleware
-  class X509Matcher
-    def initialize(options)
-      @options = options.freeze
-    end
-
-    def match(env)
-      certificate = Rails::Auth::X509::Certificate.new(env['puma.peercert'])
-      # This can be extended fairly easily to search OpenSSL::X509::Certificate#extensions for subjectAltNames.
-      @options.all? { |name, value| certificate[name] == value }
-    end
-  end
-
-  def initialize(app, whitelist)
-    acls = []
-    whitelist.each do |entry|
-      acls << {
-        'resources' => [
-          {
-            'method' => 'ALL',
-            'path' => '/.*'
-          }
-        ],
-        'allow_x509_subject' => {
-          'cn' => entry
-        }
-      }
-    end
-    acl = Rails::Auth::ACL.new(acls, matchers: { allow_x509_subject: X509Matcher })
-    mid = Rails::Auth::ACL::Middleware.new(app, acl: acl)
-    super(mid, page_body: 'Access denied')
-  end
-end