empeira 0.1.0.pre.alpha.1

data/lib/empeira/node/certificates.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    class Certificates
+      PUPPET = '/opt/puppetlabs/bin/puppet'
+      CA = '/opt/puppetlabs/bin/puppetserver'
+      RUBY = '/opt/puppetlabs/puppet/bin/ruby'
+
+      def initialize(runtime:, server:)
+        @runtime = runtime
+        @server = server
+      end
+
+      def enroll(node, record)
+        name = record.fetch('hostname')
+        raise Error, 'A CA identity already exists for this hostname; refusing to adopt it' if entry(name)
+
+        install_ca(node)
+        execute(node, [PUPPET, 'ssl', 'generate_request'], 'Node key preparation')
+        fingerprint = public_key(node, "/etc/puppetlabs/puppet/ssl/certificate_requests/#{name}.pem", request: true)
+        record['certificate_key'] = fingerprint
+        yield
+        execute(node, [PUPPET, 'ssl', 'submit_request'], 'Node certificate request')
+        request = entry(name)
+        unless request && fingerprint == ca_key(name, request: true)
+          raise Error, 'CA request does not match the Empeira-owned node key; signing refused'
+        end
+
+        execute(@server, [CA, 'ca', 'sign', '--certname', name], 'Node certificate signing')
+        execute(node, [PUPPET, 'ssl', 'download_cert'], 'Node certificate retrieval')
+      end
+
+      def clean(record)
+        certificate = entry(record.fetch('hostname'))
+        return unless certificate
+
+        request = certificate.fetch('state') == 'requested'
+        unless record['certificate_key'] && record['certificate_key'] == ca_key(record.fetch('hostname'),
+                                                                                request: request)
+          raise Error, 'CA identity differs from the recorded node key; certificate cleanup refused'
+        end
+
+        execute(@server, [CA, 'ca', 'clean', '--certname', record.fetch('hostname')], 'Node certificate cleanup')
+      end
+
+      def clean_retired(state, name)
+        retired = state.fetch('retired_certificates', {})
+        return unless retired.key?(name)
+
+        clean('hostname' => name, 'certificate_key' => retired.fetch(name))
+        retired.delete(name)
+        yield
+      end
+
+      private
+
+      def entry(name)
+        result = execute(@server, [CA, 'ca', 'list', '--all', '--format', 'json'], 'CA inventory')
+        data = JSON.parse(result.stdout)
+        data.values.flatten.find { |item| item.fetch('name') == name }
+      rescue JSON::ParserError, NoMethodError, KeyError
+        raise Error, 'Malformed CA inventory; certificate mutation refused', cause: nil
+      end
+
+      def ca_key(name, request:)
+        directory = request ? 'requests' : 'signed'
+        public_key(@server, "/etc/puppetlabs/puppetserver/ca/#{directory}/#{name}.pem", request: request)
+      end
+
+      def public_key(resource, path, request:)
+        type = request ? 'Request' : 'Certificate'
+        code = "require 'openssl'; require 'digest'; cert = OpenSSL::X509::#{type}.new(File.read(ARGV[0])); print Digest::SHA256.hexdigest(cert.public_key.to_der)"
+        execute(resource, [RUBY, '-e', code, path], 'Certificate identity verification').stdout
+      end
+
+      def install_ca(node)
+        Dir.mktmpdir('empeira-ca-') do |directory|
+          file = File.join(directory, 'ca.pem')
+          @runtime.copy_from(@server, '/etc/puppetlabs/puppet/ssl/certs/ca.pem', file)
+          execute(node, ['mkdir', '-p', '/etc/puppetlabs/puppet/ssl/certs'], 'Node CA directory')
+          @runtime.copy_to(node, file, '/etc/puppetlabs/puppet/ssl/certs/ca.pem')
+        end
+      end
+
+      def execute(resource, arguments, operation)
+        result = @runtime.service_exec(resource, arguments, timeout: 60)
+        unless result.success?
+          diagnostic = Execution::Diagnostics.native(result, operation: operation,
+                                                             tool: File.basename(arguments.first))
+          raise Error, "#{operation} failed; node state retained for diagnosis\n#{diagnostic}", cause: nil
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/empeira/node/command_mock.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+module Empeira
+  module Node
+    # A guest file resource with the same ownership labels and fingerprint contract as services.
+    class CommandMock < Services::Definition
+      attr_reader :command
+
+      def initialize(command:, workspace:, record:, definition:)
+        @command = command
+        super(key: "command-mock-#{command}", workspace: workspace, **definition.transform_keys(&:to_sym),
+              hostname: record.fetch('hostname'), provider: record.fetch('provider'))
+      end
+
+      def ownership_labels
+        super.merge('io.empeira.hostname' => options.fetch('hostname'),
+                    'io.empeira.provider' => options.fetch('provider'))
+      end
+
+      def file
+        Bootstrap::FileEntry.new(path: options.fetch('path'), content: content, mode: 0o755).freeze
+      end
+
+      def content
+        invocation = if options.fetch('mock_to') == 'echo'
+                       "printf '%s' #{Shellwords.escape(command)}\n" \
+                         "[ \"$#\" -eq 0 ] || printf ' %s' \"$@\"\nprintf '\\n'\n"
+                     else
+                       "#{Shellwords.escape(options.fetch('mock_to'))} \"$@\"\n"
+                     end
+        status = options.fetch('exit_code') == 'passthrough' ? '$?' : options.fetch('exit_code').to_s
+        "#!/bin/sh\n# #{JSON.generate(labels)}\n#{invocation}exit #{status}\n"
+      end
+    end
+  end
+end

data/lib/empeira/node/command_mocks.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    # Reconciles guest resources under the existing workspace transaction.
+    class CommandMocks
+      def initialize(context:, record:, execute:, persist:)
+        @context = context
+        @record = record
+        @execute = execute
+        @persist = persist
+      end
+
+      def reconcile
+        desired = @context.configuration.fetch('mocks').fetch('commands')
+        return false if desired.empty? && @record.fetch('command_mocks', {}).empty?
+
+        @entries = @record['command_mocks'] ||= {}
+        changed = prune(desired)
+        desired.each do |name, definition|
+          changed = apply(name, definition) || changed
+        end
+        changed
+      end
+
+      def self.valid_inventory?(entries)
+        return false unless entries.is_a?(Hash)
+
+        entries.all? { |path, entry| valid_entry?(path, entry) }
+      end
+
+      def self.valid_entry?(path, entry)
+        entry.is_a?(Hash) && (entry.keys - %w[name definition fingerprint previous]).empty? &&
+          valid_definition?(path, entry['name'], entry['definition']) &&
+          entry['fingerprint'] == Infrastructure::Definition.fingerprint(entry['definition']) &&
+          (!entry.key?('previous') || valid_definition?(path, entry['name'], entry['previous']))
+      end
+
+      def self.valid_definition?(path, name, definition)
+        Configuration::CommandMocks.validate!({ name => definition }, effective: true)
+        definition['path'] == path
+      rescue ConfigurationError
+        false
+      end
+
+      private
+
+      def resource(name, definition)
+        CommandMock.new(command: name, definition: definition, workspace: @context.workspace, record: @record)
+      end
+
+      def accepted(entry)
+        [entry.fetch('definition'), entry['previous']].compact.map do |definition|
+          resource(entry.fetch('name'), definition).file.content
+        end
+      end
+
+      def prune(desired)
+        changed = false
+        @entries.each_key do |path|
+          entry = @entries.fetch(path)
+          next if desired.dig(entry.fetch('name'), 'path') == path
+
+          changed = execute(path, nil, accepted(entry)) || changed
+          @entries.delete(path)
+          @persist.call
+        end
+        changed
+      end
+
+      def apply(name, definition)
+        path = definition.fetch('path')
+        entry = @entries[path]
+        return finish(entry) if entry && entry['definition'] == definition
+
+        # Finish interrupted writes before accepting another desired revision.
+        recovered = entry&.key?('previous') ? finish(entry) : false
+        entry = { 'name' => name, 'definition' => definition,
+                  'fingerprint' => Infrastructure::Definition.fingerprint(definition),
+                  **(entry ? { 'previous' => entry.fetch('definition') } : {}) }
+        @entries[path] = entry
+        @persist.call
+        finish(entry) || recovered
+      end
+
+      def finish(entry)
+        file = resource(entry.fetch('name'), entry.fetch('definition')).file
+        changed = execute(file.path, file.content, accepted(entry))
+        @persist.call if entry.delete('previous')
+        changed
+      end
+
+      def execute(path, content, accepted)
+        helper = File.read(File.expand_path('../../../resources/nodes/managed_file.rb', __dir__))
+        request = JSON.generate('path' => path, 'content' => content, 'accepted' => accepted, 'mode' => 0o755)
+        result = @execute.call([Certificates::RUBY, '-e', helper, request])
+        unless result.success?
+          unless @execute.call([Certificates::RUBY, '--version']).success?
+            raise Error,
+                  "Cannot run node agent Ruby at #{Certificates::RUBY}; " \
+                  'the agent installation or guest execution is unavailable. Inspect the node and its image.'
+          end
+
+          raise Providers::OwnershipError,
+                "Cannot reconcile command mock at #{path}: unsafe target, cleanup conflict or guest I/O failure"
+        end
+
+        JSON.parse(result.stdout).fetch('changed')
+      rescue JSON::ParserError, KeyError
+        raise Error, "Cannot verify command-mock reconciliation at #{path}"
+      end
+    end
+  end
+end

data/lib/empeira/node/container.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    # Container provider owns its workflow; the public service only routes logical providers.
+    # rubocop:disable-next Metrics/ClassLength -- Ordered provisioning and lifecycle share this provider boundary.
+    class Container < Interface
+      include ContainerLifecycle
+      include ContainerPreparation
+      include ContainerAgentBootstrap
+      include ContainerState
+      include ContainerGuest
+
+      attr_reader :context
+
+      def initialize(context:, runner:, backend:, build_info: BuildInfo.load, progress: Progress.new)
+        super(name: 'container', context: context, runner: runner, backend: backend)
+        @runtime = backend
+        @build_info = build_info
+        @progress = progress
+        @store = Infrastructure::Store.new(context: context)
+        @bootstrap_proxy = Network::BootstrapProxy.new(context: context, runtime: @runtime, store: @store)
+      end
+
+      def available?
+        @runtime.available?
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- Verify retired certificate ownership before the existing enrollment transaction.
+      def run(request)
+        mutate do
+          @progress.stage(10, 'Checking control plane and reserving hostname...')
+          server = ready_server
+          record, definition, image = reserve(request)
+          prepare_image(image, record)
+          resource = create_node(definition, record)
+          @progress.stage(65, 'Preparing node certificate...')
+          certificates = Certificates.new(runtime: @runtime, server: server)
+          certificates.clean_retired(@state, record.fetch('hostname')) { save } if @state['retired_certificates']
+          certificates.enroll(resource, record) { save }
+          reconcile_guest(resource, record)
+          complete_provisioning(record)
+          @progress.stage(85, 'Running Puppet agent...')
+          agent_run(resource, record)
+          lifecycle_result(record['hostname'], :running, changed: true)
+        end
+      end
+
+      def destroy_all(state:)
+        @state = state
+        @nodes = state.fetch('nodes', {})
+        @nodes.each { |name, record| remove_node(name, certificates: false) if record['provider'] == 'container' }
+      end
+
+      # rubocop:disable-next Naming/PredicateMethod -- Reconciliation mutates guest resources.
+      def reconcile_all(state:)
+        @state = state
+        @nodes = state.fetch('nodes', {})
+        container_records.values.map do |record|
+          resource = observed(record)
+          network_gateway.route(resource) if resource && resource['state'] == 'running'
+          next false unless record['provisioned'] && resource && resource['state'] == 'running'
+
+          reconcile_guest(resource, record)
+        end.any?
+      end
+
+      private
+
+      def complete_provisioning(record)
+        record['provisioned'] = true
+        save
+      end
+
+      def reconcile_command_mocks(resource, record)
+        CommandMocks.new(context: context, record: record, persist: method(:save),
+                         execute: ->(arguments) { @runtime.service_exec(resource, arguments) }).reconcile
+      end
+
+      def prepare_image(image, record)
+        @progress.stage(25, 'Preparing node image...')
+        @runtime.with_local_images do
+          @runtime.ensure_image(image.reference, recipe: image.recipe, files: image.build_files)
+          return if @runtime.image_architecture(image.reference) == record['architecture']
+        end
+
+        raise Error, 'Node image architecture differs from the runtime; emulation is not enabled'
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- Ordered bootstrap cleanup precedes normal guest policy.
+      def create_node(definition, record)
+        @progress.stage(45, 'Creating isolated container node...')
+        resource = @runtime.create_service(definition)
+        record['id'] = resource.fetch('id')
+        record['image_id'] = resource['image_id']
+        save
+        network_gateway.phase(record, bootstrap: true)
+        @runtime.start_service(resource)
+        resource = observed(record)
+        network_gateway.route(resource)
+        record['ssh_port'] = SSHEndpoint.port(resource) if record['ssh_transport'] == 'loopback'
+        save
+        apply_bootstrap(resource, record)
+        finish_network_bootstrap(record)
+        prepare_guest_environment(resource, record)
+        configure(resource, record)
+        resource
+      end
+
+      def apply_bootstrap(resource, record)
+        apply_agent_bootstrap(resource, record)
+      ensure
+        @bootstrap_proxy.cleanup(@state) if @state
+      end
+
+      def package_bootstrap(resource, record)
+        execute = ->(arguments) { @runtime.service_exec(resource, arguments, timeout: 300) }
+        requirements = package_requirements(record)
+        PackageBootstrap.new(config: context.configuration.dig('bootstrap', 'packages'),
+                             os: record.fetch('os'), execute: execute,
+                             copy: guest_copy(resource, execute),
+                             rpm_options: requirements.rpm_options,
+                             progress: @progress.method(:heartbeat))
+      end
+
+      def package_requirements(record, agent_required: false)
+        ::Empeira::VM::BootstrapRequirements.new(context: context, os: record.fetch('os'),
+                                                 version: record.fetch('version'), agent_required: agent_required,
+                                                 distribution_required: true,
+                                                 architecture: record.fetch('architecture'))
+      end
+    end
+  end
+end

data/lib/empeira/node/container_agent_bootstrap.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    # Runs the selected agent installation through container guest I/O.
+    module ContainerAgentBootstrap
+      private
+
+      # rubocop:disable-next Metrics/AbcSize -- The ordered guest transaction spans packages, APT and agent installation.
+      def apply_agent_bootstrap(resource, record)
+        execute = ->(arguments) { @runtime.service_exec(resource, arguments, timeout: 300) }
+        requirements = package_requirements(record, agent_required: true)
+        @bootstrap_proxy.start(@state, requirements, source: record.dig('definition', 'ip'))
+        family = PackageBootstrap::FAMILIES.fetch(record.fetch('os'))
+        configuration = family == 'debian' ? AptConfiguration : RpmConfiguration
+        configuration.new(os: record.fetch('os'), execute: execute).preserve do
+          packages = package_bootstrap(resource, record)
+          packages.run(proxy_url: @bootstrap_proxy.url) if packages.required?
+          install_agent(resource, record, requirements, execute)
+        end
+        ContainerBootstrap.new(runtime: @runtime).apply(resource: resource,
+                                                        bootstrap: Bootstrap.new(provider: 'container'))
+        record['empeira_fact'] = ExternalFact.fingerprint('container')
+        save
+        PackageSources.verify!(execute: execute)
+      end
+
+      def install_agent(resource, _record, requirements, execute)
+        installer = AgentInstaller.build(context: context, runtime: @runtime, requirements: requirements,
+                                         execute: execute, copy: guest_copy(resource, execute), progress: @progress)
+        installer.install(proxy_url: @bootstrap_proxy.url)
+      end
+
+      def guest_copy(resource, execute)
+        lambda do |source, destination, mode|
+          @runtime.copy_to(resource, source, destination)
+          raise Error, 'Cannot set agent file permissions' unless
+            execute.call(['chmod', mode, destination]).success?
+        end
+      end
+    end
+  end
+end

data/lib/empeira/node/container_bootstrap.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    # Containers use the already installed agent Ruby runtime, never Cloud-Init.
+    class ContainerBootstrap
+      INSTALL = <<~RUBY
+        require 'fileutils'
+        path, content, mode = ARGV
+        FileUtils.mkdir_p(File.dirname(path), mode: 0755)
+        File.write(path, content, mode: 'w', perm: mode.to_i(8))
+        File.chmod(mode.to_i(8), path)
+      RUBY
+      VERIFY = <<~RUBY
+        path, content, mode = ARGV
+        exit(File.binread(path) == content && (File.stat(path).mode & 0777) == mode.to_i(8) ? 0 : 1)
+      RUBY
+
+      def initialize(runtime:)
+        @runtime = runtime
+      end
+
+      def apply(resource:, bootstrap:)
+        bootstrap.files.each do |file|
+          run!(resource, INSTALL, file, 'install')
+          run!(resource, VERIFY, file, 'verify')
+        end
+      end
+
+      # rubocop:disable-next Naming/PredicateMethod -- Reconciliation mutates owned guest state and reports changes.
+      def reconcile(resource:, bootstrap:)
+        bootstrap.files.map do |file|
+          arguments = [Certificates::RUBY, '-e', VERIFY, file.path, file.content, format('%04o', file.mode)]
+          next false if @runtime.service_exec(resource, arguments).success?
+
+          run!(resource, INSTALL, file, 'install')
+          run!(resource, VERIFY, file, 'verify')
+          true
+        end.any?
+      end
+
+      private
+
+      def run!(resource, code, file, action)
+        arguments = [Certificates::RUBY, '-e', code, file.path, file.content, format('%04o', file.mode)]
+        return if @runtime.service_exec(resource, arguments).success?
+
+        raise Error, "Cannot #{action} required Empeira node bootstrap file #{file.path}; Puppet was not run"
+      end
+    end
+  end
+end

data/lib/empeira/node/container_guest.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    module ContainerGuest
+      private
+
+      def reconcile_guest(resource, record)
+        fact_changed = reconcile_provider_fact(resource, record)
+        environment_changed = prepare_guest_environment(resource, record)
+        reconcile_command_mocks(resource, record) || environment_changed || fact_changed
+      end
+
+      # rubocop:disable-next Naming/PredicateMethod -- Reconciliation mutates owned guest state and reports changes.
+      def reconcile_provider_fact(resource, record)
+        digest = ExternalFact.fingerprint('container')
+        return false if record['empeira_fact'] == digest
+
+        ContainerBootstrap.new(runtime: @runtime).reconcile(resource: resource,
+                                                            bootstrap: Bootstrap.new(provider: 'container'))
+        record['empeira_fact'] = digest
+        save
+        true
+      end
+
+      def prepare_guest_environment(resource, record)
+        proxy_changed = reconcile_runtime_proxy(resource, record)
+        reconcile_interactive_tools(resource, record) || proxy_changed
+      end
+
+      def reconcile_interactive_tools(resource, record)
+        InteractiveTools.new(record: record, persist: method(:save),
+                             execute: ->(arguments) { @runtime.service_exec(resource, arguments) }).reconcile
+      end
+
+      def reconcile_runtime_proxy(resource, record)
+        RuntimeProxy.new(context: context, record: record, persist: method(:save),
+                         execute: ->(arguments) { @runtime.service_exec(resource, arguments) }).reconcile
+      end
+    end
+  end
+end

data/lib/empeira/node/container_lifecycle.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Empeira
+  module Node
+    # rubocop:disable-next Metrics/ModuleLength -- Provider lifecycle operations share one ownership boundary.
+    module ContainerLifecycle
+      def start(name:)
+        mutate do
+          record = fetch(name)
+          require_provisioned!(record)
+          resource = observed(record)
+          raise Error, 'Node is missing; destroy its reservation before recreating it' unless resource
+
+          ready_server
+          changed = resource['state'] != 'running'
+          @runtime.start_service(resource) if changed
+          network_gateway.route(observed(record))
+          refresh_dns
+          changed = reconcile_guest(resource, record) || changed
+          lifecycle_result(record['hostname'], :running, changed: changed)
+        end
+      end
+
+      def stop(name:)
+        mutate do
+          resource = observed(fetch(name))
+          raise Error, 'Node is missing; inspect node list' unless resource
+
+          changed = resource['state'] == 'running'
+          @runtime.stop_service(resource) if changed
+          lifecycle_result(name.downcase, :stopped, changed: changed)
+        end
+      end
+
+      def destroy(name:)
+        Infrastructure::Cleanup.new(context: context, runner: @runner, runtime: @runtime,
+                                    progress: @progress).node(name)
+      end
+
+      def puppet(name:)
+        mutate(availability: false) do
+          record = fetch(name)
+          require_provisioned!(record)
+          resource = running(record)
+          ready_network!
+          @progress.stage(30, 'Running Puppet agent...')
+          agent_run(resource, record)
+        end
+      end
+
+      def shell(name:)
+        mutate do
+          resource = running(fetch(name))
+          raise Error, 'Node shell requires a local terminal' unless $stdin.tty? && $stdout.tty?
+
+          arguments = RuntimeProxy.command(context, ['env', 'HISTFILE=/dev/null', '/bin/bash', '-i'])
+          @runtime.stream_service(resource, arguments, interactive: true, user: '0')
+        end
+      end
+
+      def console(name:)
+        raise Error, "Container node #{name} has no serial VM console; use node shell or node exec"
+      end
+
+      def exec(name:, arguments:, timeout: nil, output: $stdout, error: $stderr)
+        DiagnosticCommand.validate!(arguments, timeout)
+        mutate do
+          Execution::Terminal.with_interrupts do
+            @progress.streaming do
+              @runtime.diagnostic_exec(running(fetch(name)), arguments, timeout: timeout, output: output, error: error)
+            end
+          end
+        end
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- Verify the owned endpoint before selecting a guest tunnel.
+      def ssh(name:, user: nil, identity: nil, port: nil)
+        mutate do
+          port = Configuration::SSHPreferences.port!(port || 22)
+          record = fetch(name)
+          resource = running(record)
+          unless record['ssh_host'] && record['ssh_port']
+            raise Error, 'Node has no managed SSH endpoint; destroy and recreate it with the current Empeira image'
+          end
+
+          proxy = if port != 22
+                    @runtime.ssh_proxy_command(resource, port: port)
+                  elsif record['ssh_transport'] == 'tunnel'
+                    @runtime.ssh_proxy_command(resource)
+                  end
+          record = record.merge('ssh_port' => port) if port != 22
+          UserSSH.new(runner: @runner, credentials: ssh_credentials(record.fetch('hostname')),
+                      proxy_command: proxy, home: context.locations.home).session(
+                        record, user: user, identity: identity
+                      )
+        end
+      end
+
+      def logs(name:)
+        load_state
+        check_runtime!
+        resource = observed(fetch(name))
+        raise Error, 'Node container is missing' unless resource
+
+        @runtime.stream_service(resource, [], logs: true)
+      end
+
+      private
+
+      def require_provisioned!(record)
+        return if record['provisioned']
+
+        raise Error, 'Node bootstrap is incomplete; inspect logs or shell, then destroy and recreate the node'
+      end
+
+      def remove_node(name, certificates: true)
+        record = @nodes[name]
+        return unless record
+
+        observed(record)
+        if certificates && record['certificate_key']
+          Certificates.new(runtime: @runtime, server: ready_server).clean(record)
+        end
+        @runtime.remove_service(definition(record), expected_id: record['id'])
+        ssh_credentials(name).cleanup
+        @nodes.delete(name)
+        save
+        true
+      end
+
+      def agent_run(resource, record)
+        require_provisioned!(record)
+        reconcile_provider_fact(resource, record)
+        result = @progress.streaming do
+          @runtime.stream_service(resource, RuntimeProxy.command(context, PuppetCommand.arguments))
+        end
+        record['last_puppet_exit'] = result.exit_status
+        save
+        return result if [0, 2].include?(result.exit_status)
+
+        raise Error, "Puppet agent failed (exit #{result.exit_status}); node retained for shell/logs and another run"
+      end
+    end
+  end
+end