empeira 0.1.0.pre.alpha.1

data/lib/empeira/addons/installation.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Addons
+    class Installation
+      attr_reader :name, :source, :ref
+
+      def initialize(name:, definition:, context:, runner:)
+        @name = name
+        @source, @ref = definition.values_at('git', 'ref')
+        @runner = runner
+        @project = context.project.path
+        @storage = Storage.new(base: cache_root(context), source: source, ref: ref)
+      rescue Empeira::Error, SystemCallError => e
+        failure(e)
+      end
+
+      def local
+        record = @storage.installed
+        return unless record
+
+        commit, path = record
+        [commit, path, Manifest.new(name: name, path: path)]
+      rescue Empeira::Error, SystemCallError => e
+        failure(e)
+      end
+
+      def synchronize(missing_only: false)
+        return local if missing_only && local
+
+        @storage.synchronize do
+          return local if missing_only && local
+
+          acquire
+          commit = resolve
+          materialize(commit)
+          @storage.publish(commit)
+          local
+        end
+      rescue Empeira::Error, SystemCallError => e
+        failure(e)
+      end
+
+      private
+
+      def cache_root(context)
+        root = Modules::GitSafety.canonical_path(context.locations.addons)
+        if root == @project || root.to_s.start_with?("#{@project}/")
+          raise Error.new(name, 'user addon cache must be outside the control repository')
+        end
+
+        root
+      end
+
+      def failure(error)
+        raise error if error.is_a?(Error)
+
+        message = error.is_a?(SystemCallError) ? 'cache is inaccessible or installation is incomplete' : error.message
+        raise Error.new(name, message), cause: nil
+      end
+
+      def acquire
+        git = Modules::HostGit.new(runner: @runner, project: @project, passive: true)
+        entries = [{ 'name' => 'addon', 'remote' => source }]
+        return git.prepare(entries, directory: @storage.root) if @storage.mirror.exist?
+
+        @storage.stage do |directory|
+          mirror = git.prepare(entries, directory: directory).fetch('addon')
+          File.rename(mirror, @storage.mirror)
+        end
+      end
+
+      def resolve
+        result = git('rev-parse', '--verify', '--end-of-options', "#{ref}^{commit}")
+        commit = result.stdout.strip
+        unless result.success? && commit.match?(Storage::COMMIT)
+          raise Error.new(name, 'Git reference is unknown or does not resolve to a commit'), cause: nil
+        end
+
+        commit
+      end
+
+      def materialize(commit)
+        final = @storage.revision(commit)
+        return Manifest.new(name: name, path: final) if final.exist?
+
+        @storage.stage do |directory|
+          checkout(commit, directory)
+          Manifest.new(name: name, path: directory)
+          File.rename(directory, final)
+        end
+      end
+
+      def checkout(commit, directory)
+        Tempfile.create('.index-', @storage.root) do |index|
+          File.unlink(index.path)
+          environment = { 'GIT_INDEX_FILE' => index.path }
+          tree = git('read-tree', commit, environment: environment)
+          raise Error.new(name, 'Git could not read the resolved commit') unless tree.success?
+
+          files = git('--work-tree', directory.to_s, 'checkout-index', '--all', "--prefix=#{directory}/",
+                      environment: environment)
+          raise Error.new(name, 'Git could not materialize the resolved commit') unless files.success?
+        end
+      end
+
+      def git(*arguments, environment: {})
+        # Authentication happens only in HostGit. Local export must not run
+        # user-configured smudge filters from .gitattributes or hooks.
+        environment = { 'GIT_CONFIG_GLOBAL' => File::NULL, 'GIT_CONFIG_NOSYSTEM' => '1',
+                        'GIT_CONFIG_COUNT' => '0', 'GIT_CONFIG_PARAMETERS' => nil, 'GIT_ATTR_NOSYSTEM' => '1' }
+                      .merge(environment)
+        @runner.run('git', arguments: ['-c', "core.hooksPath=#{File::NULL}", '-c', 'core.fsmonitor=false',
+                                       '--git-dir', @storage.mirror.to_s, *arguments],
+                           environment: environment, directory: @project, timeout: 300)
+      end
+    end
+  end
+end

data/lib/empeira/addons/manifest.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Addons
+    class Manifest
+      TOKEN = /\A[a-zA-Z0-9][a-zA-Z0-9_-]*\z/
+      OPTION = /\A--?[a-zA-Z0-9][a-zA-Z0-9-]*\z/
+      COMPLETIONS = %w[file directory enum none].freeze
+      attr_reader :data, :executable
+
+      def initialize(name:, path:)
+        @name = name
+        @root = Pathname(path).realpath
+        manifest = contained!('empeira-addon.yaml', 'manifest')
+        @data = Configuration::Document.new.read(manifest)
+        validate!
+        @executable = contained!(data.fetch('executable'), 'executable')
+        fail!('executable is not executable', 126) unless @executable.executable?
+        Immutable.deep_freeze(@data)
+      rescue ConfigurationError
+        fail!('manifest must contain valid, unique YAML data without aliases or object tags')
+      rescue SystemCallError
+        fail!('installation or manifest is missing or inaccessible')
+      end
+
+      private
+
+      def fail!(message, status = 1)
+        raise Error.new(@name, message, exit_status: status), cause: nil
+      end
+
+      def contained!(relative, label)
+        fail!("#{label} must be a relative path without traversal") unless relative?(relative)
+        path = @root.join(relative)
+        fail!("#{label} is missing", label == 'executable' ? 127 : 1) unless path.file?
+        fail!("#{label} escapes the addon directory") unless path.realpath.to_s.start_with?("#{@root}/")
+        path.realpath
+      end
+
+      def relative?(value)
+        value.is_a?(String) && !value.empty? && !value.match?(/[\p{Cc}\p{Cf}]/) &&
+          !Pathname(value).absolute? && !value.split('/').intersect?(%w[. ..])
+      end
+
+      def validate!
+        mapping!(data, %w[schema description executable commands], 'manifest', required: true)
+        unless data['schema'].is_a?(Integer) && data['schema'] == 1
+          fail!('unsupported manifest schema; expected integer 1')
+        end
+        text!(data['description'], 'description')
+        commands!(data['commands'])
+      end
+
+      def commands!(commands)
+        fail!('commands must be a mapping') unless commands.is_a?(Hash)
+        commands.each do |name, command|
+          fail!('invalid command name') unless name.is_a?(String) && name.match?(TOKEN)
+          command!(command)
+        end
+      end
+
+      def command!(command)
+        mapping!(command, %w[description arguments options], 'command')
+        text!(command['description'], 'command description')
+        arguments = command.fetch('arguments', [])
+        fail!('arguments must be an array') unless arguments.is_a?(Array)
+        arguments.each { |argument| argument!(argument) }
+        options!(command.fetch('options', {}))
+      end
+
+      def options!(options)
+        fail!('options must be a mapping') unless options.is_a?(Hash)
+        options.each do |name, value|
+          fail!('invalid option name') unless name.is_a?(String) && name.match?(OPTION)
+          completion!(value, option: true)
+        end
+      end
+
+      def argument!(argument)
+        completion!(argument)
+        name = argument['name']
+        fail!('argument requires a name') unless name.is_a?(String) && name.match?(TOKEN)
+      end
+
+      def completion!(value, option: false)
+        mapping!(value, option ? %w[description completion values] : %w[name completion values], 'completion')
+        text!(value['description'], 'option description') if option && value.key?('description')
+        fail!('unknown or missing completion type') unless COMPLETIONS.include?(value['completion'])
+        return enum!(value['values']) if value['completion'] == 'enum'
+
+        fail!('values are only supported for enum completion') if value.key?('values')
+      end
+
+      def enum!(values)
+        fail!('enum completion requires an array of values') unless values.is_a?(Array) && !values.empty?
+        values.each { |value| text!(value, 'enum value') }
+      end
+
+      def mapping!(value, fields, label, required: false)
+        return if value.is_a?(Hash) && (value.keys - fields).empty? && (!required || (fields - value.keys).empty?)
+
+        fail!("#{label} has missing or unsupported fields")
+      end
+
+      def text!(value, label)
+        return if value.is_a?(String) && !value.strip.empty? && !value.match?(/[\p{Cc}\p{Cf}]/)
+
+        fail!("#{label} must be a nonempty string without control characters")
+      end
+    end
+  end
+end

data/lib/empeira/addons/service.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Addons
+    class Service
+      def initialize(context:, runner:)
+        @context = context
+        @runner = runner
+      end
+
+      def names
+        @context.configuration.fetch('addons').keys
+      end
+
+      def installation(name)
+        definition = @context.configuration.fetch('addons')[name]
+        raise Error.new(name, 'not configured') unless definition
+
+        Installation.new(name: name, definition: definition, context: @context, runner: @runner)
+      end
+
+      def synchronize(missing_only: false)
+        errors = names.filter_map do |name|
+          installation(name).synchronize(missing_only: missing_only)
+          nil
+        rescue Error => e
+          e.message
+        end
+        raise Error.new('synchronization', errors.join("\n")) unless errors.empty?
+      end
+
+      def execute(name, arguments)
+        entry = installation(name)
+        _, path, manifest = entry.synchronize(missing_only: true)
+        environment = execution_environment(entry, path, [manifest.executable.to_s, *arguments])
+        @runner.replace(manifest.executable.to_s, arguments: arguments, environment: environment)
+      rescue Execution::LaunchError => e
+        raise Error.new(name, e.message, exit_status: e.exit_status), cause: nil
+      rescue ExecutionError => e
+        raise Error.new(name, e.message), cause: nil
+      end
+
+      private
+
+      def execution_environment(entry, path, arguments)
+        Environment.new(context: @context, name: entry.name, ref: entry.ref, path: path).export(arguments: arguments)
+      end
+    end
+  end
+end

data/lib/empeira/addons/storage.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tempfile'
+require 'digest'
+
+module Empeira
+  module Addons
+    class Storage
+      COMMIT = /\A(?:[0-9a-f]{40}|[0-9a-f]{64})\z/
+      attr_reader :root, :source_key
+
+      def initialize(base:, source:, ref:)
+        @base = Pathname(base)
+        @source_key = Digest::SHA256.hexdigest(source)
+        @root = @base.join(source_key)
+        @reference = root.join('refs', Digest::SHA256.hexdigest(ref))
+      end
+
+      def installed
+        return unless @reference.exist?
+
+        directories.each { |directory| owned!(directory, directory: true) }
+        owned!(@reference)
+        commit = @reference.read.strip
+        raise Empeira::Error, 'Installation reference is incomplete' unless commit.match?(COMMIT)
+
+        path = revision(commit)
+        owned!(path, directory: true)
+        [commit, path]
+      end
+
+      def revision(commit)
+        root.join('revisions', commit)
+      end
+
+      def synchronize
+        prepare
+        File.open(root.join('lock'), File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |file|
+          owned!(root.join('lock'))
+          file.flock(File::LOCK_EX)
+          yield
+        ensure
+          file.flock(File::LOCK_UN)
+        end
+      end
+
+      def publish(commit)
+        Tempfile.create('.reference-', @reference.dirname) do |file|
+          file.write("#{commit}\n")
+          file.flush
+          file.fsync
+          File.rename(file.path, @reference)
+        end
+      end
+
+      def stage
+        directory = Pathname(Dir.mktmpdir('.staging-', root))
+        yield directory
+      ensure
+        FileUtils.remove_entry(directory) if directory&.exist?
+      end
+
+      def mirror
+        root.join("#{source_key}.git")
+      end
+
+      private
+
+      def prepare
+        FileUtils.mkdir_p(@base.dirname, mode: 0o700)
+        directories.each do |path|
+          begin
+            Dir.mkdir(path, 0o700)
+          rescue Errno::EEXIST
+            # Stable private directories, following the existing agent cache convention.
+          end
+          owned!(path, directory: true)
+        end
+      end
+
+      def directories
+        [@base, root, root.join('refs'), root.join('revisions')]
+      end
+
+      def owned!(path, directory: false)
+        stat = path.lstat
+        valid = stat.uid == Process.uid && !stat.symlink? && stat.mode.nobits?(0o077)
+        valid &&= directory ? stat.directory? : stat.file? && stat.nlink == 1
+        raise Empeira::Error, 'Addon cache contains an unsafe, missing or foreign entry' unless valid
+      end
+    end
+  end
+end

data/lib/empeira/agent/acquisition.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Agent
+    # One source-resolution and acquisition path for both providers and both install methods.
+    class Acquisition
+      def initialize(context:, runtime:, progress: Progress.new)
+        @context = context
+        @runtime = runtime
+        @progress = progress
+      end
+
+      def with_package(requirements)
+        prepare(requirements)
+        request = identity
+        acquire = ->(directory) { fetch(directory, request) }
+        cache.with_artifact(request, enabled: @context.configuration.dig('agent', 'cache', 'enabled'),
+                                     acquire: acquire, pin: package_pin) do |artifact|
+          @progress.heartbeat('Installing the resolved local agent package...')
+          yield artifact
+        end
+      end
+
+      def preflight!(requirements)
+        return unless requirements.required?
+
+        @runtime.check_available!
+        return if @runtime.architecture == requirements.target.architecture
+
+        raise Error, 'Agent package helper architecture differs from the target; emulation is not enabled'
+      end
+
+      private
+
+      def cache
+        Cache.new(root: @context.locations.cache.join('agents'), temporary: @context.locations.temporary)
+      end
+
+      def prepare(requirements)
+        @requirements = requirements
+        @target = requirements.target
+        @source = requirements.repository
+      end
+
+      def identity
+        { 'package' => @requirements.agent.fetch('package'),
+          'requested_version' => @requirements.agent.fetch('version'),
+          'family' => @target.family, 'distribution' => @target.os, 'release' => @target.release,
+          'architecture' => @target.architecture, 'format' => @target.format, 'suffix' => @source['suffix'],
+          'source' => Cache.fingerprint(@source), 'method' => @requirements.agent.dig('install', 'method') }
+      end
+
+      def package_pin
+        return @source['sha256'] unless @source['url'].end_with?('.deb', '.rpm') &&
+                                        @requirements.agent.dig('install', 'method') == 'repository'
+
+        nil
+      end
+
+      def fetch(directory, request)
+        @authentication = Authentication.new(url: @source.fetch('url'), progress: @progress)
+        @download = Download.new(authentication: @authentication)
+        @progress.heartbeat('Resolving and acquiring the agent package...')
+        image = helper_image
+        @runtime.with_agent_helper(image: image.reference) do |resource|
+          resolver = native_source(resource, directory)
+          path, metadata, selected = acquire_package(resolver, directory, request)
+          describe(path, metadata, selected, resolver, request)
+        end
+      end
+
+      # rubocop:disable-next Metrics/AbcSize -- Keep native resolution, acquisition and identity verification together.
+      def acquire_package(resolver, directory, request)
+        direct = request.fetch('method') == 'package'
+        resolver.prepare_keys if direct
+        selected = direct ? { 'url' => @source.fetch('url'), 'sha256' => package_pin } : resolver.resolve
+        path = Pathname(directory).join("package.#{@target.format}")
+        @download.fetch(selected.fetch('url'), path, sha256: package_pin || selected['sha256'])
+        metadata = resolver.inspect_package(path)
+        if selected['version'] && metadata['version'] != selected['version']
+          raise Error, 'Agent package version differs from its native repository metadata'
+        end
+
+        [path, metadata, selected]
+      end
+
+      def describe(path, metadata, selected, resolver, request)
+        details = { 'schema' => 1, 'request' => request, 'url' => selected.fetch('url'),
+                    'sha256' => Digest::SHA256.file(path).hexdigest, 'size' => path.size,
+                    'public_keys' => resolver.public_keys,
+                    'verify_signatures' => @source.fetch('verify_signatures', true) }
+        Artifact.new(path: path, metadata: metadata.merge(details))
+      end
+
+      def native_source(resource, directory)
+        klass = @target.family == 'debian' ? Node::AgentRepository : Node::DnfAgentRepository
+        execute = ->(arguments) { @runtime.service_exec(resource, arguments, timeout: 300) }
+        copy = lambda do |source, destination, mode|
+          @runtime.copy_to(resource, source, destination)
+          raise Error, 'Cannot set agent helper file permissions' unless execute.call(['chmod', mode,
+                                                                                       destination]).success?
+        end
+        klass.new(source: @source, package: @requirements.agent.fetch('package'),
+                  version: @requirements.agent.fetch('version'), target: @target, execute: execute, copy: copy,
+                  download: @download, authentication: @authentication, directory: directory)
+      end
+
+      def helper_image
+        preflight!(@requirements)
+        request = Struct.new(:os, :version).new(@target.os, @target.release)
+        image = Node::Image.new(config: @context.configuration, request: request, architecture: @target.architecture)
+        @runtime.ensure_image(image.reference, recipe: image.recipe, files: image.build_files)
+        raise Error, 'Agent helper image architecture differs from the target' unless
+          @runtime.image_architecture(image.reference) == @target.architecture
+
+        image
+      end
+    end
+  end
+end

data/lib/empeira/agent/artifact.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Agent
+    Artifact = Data.define(:path, :metadata) do
+      def valid_identity?(request)
+        metadata.is_a?(Hash) && metadata['schema'] == 1 && metadata['request'] == request &&
+          valid_version?(request) && valid_transport? && valid_keys? && valid_architecture?(request)
+      end
+
+      private
+
+      def valid_version?(request)
+        metadata['version'].is_a?(String) &&
+          Versions.match?(metadata['version'], request.fetch('requested_version'), suffix: request['suffix'])
+      end
+
+      def valid_transport?
+        uri = URI(metadata['url'])
+        Configuration::AgentSchema.https_endpoint?(uri) &&
+          [true, false].include?(metadata['verify_signatures'])
+      rescue URI::InvalidURIError, TypeError, ArgumentError
+        false
+      end
+
+      def valid_keys?
+        keys = metadata['public_keys']
+        keys.is_a?(Array) && keys.all? { |key| key.is_a?(String) && !Base64.strict_decode64(key).empty? }
+      rescue ArgumentError
+        false
+      end
+
+      def valid_architecture?(request)
+        target = Target.new(os: request.fetch('distribution'), release: request.fetch('release'),
+                            architecture: request.fetch('architecture'))
+        generic = request.fetch('format') == 'deb' ? 'all' : 'noarch'
+        [target.native_architecture, generic].include?(metadata['architecture'])
+      end
+    end
+  end
+end

data/lib/empeira/agent/authentication.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'io/console'
+require 'uri'
+require 'base64'
+
+module Empeira
+  module Agent
+    class AuthenticationRequired < Error
+      attr_reader :url
+
+      def initialize(message = 'Agent repository requires authentication', url: nil)
+        super(message)
+        @url = url
+      end
+    end
+
+    class Authentication
+      def self.credentials
+        values = %w[EMPEIRA_AGENT_REPO_USERNAME EMPEIRA_AGENT_REPO_PASSWORD].map { |key| ENV.fetch(key, nil) }
+        return if values.all?(&:nil?)
+        return values if valid?(values)
+
+        raise ConfigurationError, 'Set both nonempty EMPEIRA_AGENT_REPO_USERNAME and ' \
+                                  'EMPEIRA_AGENT_REPO_PASSWORD for agent repository authentication'
+      end
+
+      def self.valid?(values)
+        values.all? { |value| value.is_a?(String) && !value.empty? && !value.match?(/[\x00-\x1f\x7f]/) } &&
+          !values.first.include?(':')
+      end
+
+      def initialize(url:, progress: Progress.new, input: $stdin, output: $stderr)
+        @origin = URI(url)
+        @credentials = self.class.credentials
+        @credential_source = @credentials ? 'ENV' : 'interactive'
+        @progress = progress
+        @input = input
+        @output = output
+      end
+
+      def credentials(url = @origin.to_s)
+        uri = URI(url)
+        @credentials if same_origin?(uri)
+      end
+
+      def attempt
+        yield
+      rescue AuthenticationRequired => e
+        require_retry!(e)
+        login_after(e)
+        begin
+          yield
+        rescue AuthenticationRequired => retry_error
+          raise Error, failure_message(retry_error), cause: nil
+        end
+      end
+
+      def redact(value)
+        secrets = [*@credentials, (Base64.strict_encode64(@credentials.join(':')) if @credentials)].compact
+        sorted = secrets.sort_by { |secret| -secret.length }
+        sorted.reduce(value.to_s) { |text, secret| text.gsub(secret, '[REDACTED]') }
+      end
+
+      private
+
+      def require_retry!(error)
+        if error.url && !same_origin?(URI(error.url))
+          raise Error, 'Agent authentication required at a different origin; source credentials were not forwarded.' \
+                       "\n#{error.message}",
+                cause: nil
+        end
+        raise Error, failure_message(error), cause: nil if @credentials
+      end
+
+      def login_after(error)
+        login
+      rescue Error => e
+        raise Error, "#{e.message}\n#{error.message}", cause: nil
+      end
+
+      def same_origin?(uri)
+        [uri.scheme, uri.host&.downcase, uri.port] == [@origin.scheme, @origin.host&.downcase, @origin.port]
+      end
+
+      def failure_message(error)
+        source = @credential_source == 'ENV' ? 'ENV credentials were not replaced' : 'interactive login was rejected'
+        "Agent repository authentication failed: #{source}.\n#{error.message}"
+      end
+
+      def login
+        unless @input.tty? && @output.tty?
+          raise Error, 'Agent repository requires authentication; set EMPEIRA_AGENT_REPO_USERNAME and ' \
+                       'EMPEIRA_AGENT_REPO_PASSWORD in noninteractive environments'
+        end
+
+        @progress.streaming { read_credentials }
+      end
+
+      def read_credentials
+        @output.puts("Agent repository requires authentication: #{@origin.host}")
+        @output.print('Log in? [y/N] ')
+        @output.flush
+        raise Error, 'Agent repository login cancelled' unless @input.gets&.strip&.match?(/\Ay(?:es)?\z/i)
+
+        username = read_value('Username: ')
+        password = read_value('Password: ', secret: true)
+        @output.puts
+        values = [username, password]
+        raise Error, 'Agent repository login cancelled or credentials are invalid' unless self.class.valid?(values)
+
+        @credentials = values
+      end
+
+      def read_value(prompt, secret: false)
+        @output.print(prompt)
+        @output.flush
+        (secret ? @input.noecho(&:gets) : @input.gets)&.chomp
+      end
+    end
+  end
+end