ace-lab 0.4.0 → 0.5.1

data/lib/ace/lab/organisms/protected_service_client.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require_relative "protected_service_listener"
+
+module Ace
+  module Lab
+    module Organisms
+      # Worker-side short claim exchange. No local journal/executor fallback;
+      # outcome/recovery remains the existing protected authority service API.
+      class ProtectedServiceClient
+        def initialize(mapping_id:, service_id:, deployment: Ace::Assign::Authority::Deployment.load,
+          kernel: Ace::Runtime::Molecules::ProtectedLinux.new, wire: Ace::Runtime::Molecules::ProtectedSocket)
+          @deployment, @kernel, @wire, @mapping_id = deployment, kernel, wire, mapping_id
+          @map = deployment.mapping(mapping_id)
+          project = deployment.project(@map.fetch("project_id"))
+          @selection = project.fetch("service_receivers").fetch(service_id)
+          @credentials = project.fetch("peer_credentials").fetch(@selection.fetch("executor_uid").to_s)
+          @scratch = project.fetch("peer_credentials").fetch(@map.fetch("worker_uid").to_s).fetch("scratch_root")
+        end
+
+        def preview_workspace_prune(intent:)
+          intent = Atoms::ProtectedWorkspacePrunePreview.intent!(JSON.parse(JSON.generate(intent)))
+          bytes = JSON.generate(intent)
+          deadline = @wire.deadline(5)
+          @deployment.verify!(@mapping_id, kernel: @kernel)
+          caller = @kernel.capture(Process.pid)
+          unless caller.values_at("uid", "gid", "groups") == @map.values_at("worker_uid", "worker_gid", "worker_groups")
+            raise SecurityError, "preview caller differs"
+          end
+          codec = Ace::Assign::Authority::TransferCodec.new(root: @scratch)
+          descriptor = codec.descriptor([bytes], purpose: :service_input)
+          path = @selection.fetch("socket_path")
+          before = @wire.socket_identity(path)
+          raise SecurityError, "preview receiver owner differs" unless before.last == @selection.fetch("executor_uid")
+          @wire.connect(path, deadline: deadline) do |socket|
+            peer = @kernel.peer(socket)
+            unless peer.values_at("uid", "gid", "groups") == [@selection.fetch("executor_uid"), @credentials.fetch("gid"), @credentials.fetch("groups")] &&
+                @wire.socket_identity(path) == before
+              raise SecurityError, "preview receiver peer changed"
+            end
+            @wire.write(socket, {"version" => 1, "kind" => "preview_workspace_prune", "transfer" => descriptor},
+              deadline: deadline, limit: ProtectedServiceListener::LIMIT)
+            codec.send(socket, parts: [bytes], descriptor: descriptor, purpose: :service_input, deadline: deadline)
+            socket.shutdown(Socket::SHUT_WR)
+            guarded = Ace::Runtime::Molecules::ProtectedSocket::Ingress.new(socket)
+            result = @wire.read(guarded, deadline: deadline, limit: ProtectedServiceListener::LIMIT)
+            eof!(guarded, deadline)
+            raise SecurityError, "preview result unavailable" unless result.is_a?(Hash) && result.key?("maintenance_context")
+            Atoms::ProtectedWorkspacePrunePreview.context!(result.fetch("maintenance_context"))
+            Atoms::ProtectedWorkspacePrunePreview.result!(result, intent: intent, context: result.fetch("maintenance_context"))
+          end
+        end
+
+        def submit(submission:, input_bytes:, mutation_id:)
+          unless submission.is_a?(Hash) && submission.keys.sort == ProtectedServiceReceiver::SUBMISSION.sort &&
+              input_bytes.is_a?(String) && mutation_id.is_a?(String) && mutation_id.match?(Ace::Assign::Molecules::JournalMutation::ID)
+            raise ArgumentError, "receiver submission is invalid"
+          end
+          submission = JSON.parse(JSON.generate(submission))
+          bytes = input_bytes.dup.freeze
+          mutation_id = mutation_id.dup.freeze
+          deadline = @wire.deadline(5)
+          @deployment.verify!(@mapping_id, kernel: @kernel)
+          caller = @kernel.capture(Process.pid)
+          unless caller.values_at("uid", "gid", "groups") == @map.values_at("worker_uid", "worker_gid", "worker_groups")
+            raise SecurityError, "receiver caller differs"
+          end
+          codec = Ace::Assign::Authority::TransferCodec.new(root: @scratch)
+          descriptor = codec.descriptor([bytes], purpose: :service_input)
+          path = @selection.fetch("socket_path")
+          before = @wire.socket_identity(path)
+          raise SecurityError, "receiver endpoint owner differs" unless before.last == @selection.fetch("executor_uid")
+          @wire.connect(path, deadline: deadline) do |socket|
+            peer = @kernel.peer(socket)
+            unless peer.values_at("uid", "gid", "groups") == [@selection.fetch("executor_uid"), @credentials.fetch("gid"), @credentials.fetch("groups")] &&
+                @wire.socket_identity(path) == before
+              raise SecurityError, "receiver endpoint peer changed"
+            end
+            @wire.write(socket, {"version" => 1, "submission" => submission,
+              "mutation_id" => mutation_id, "transfer" => descriptor}, deadline: deadline, limit: ProtectedServiceListener::LIMIT)
+            codec.send(socket, parts: [bytes], descriptor: descriptor, purpose: :service_input, deadline: deadline)
+            socket.shutdown(Socket::SHUT_WR)
+            guarded = Ace::Runtime::Molecules::ProtectedSocket::Ingress.new(socket)
+            reply = @wire.read(guarded, deadline: deadline, limit: ProtectedServiceListener::LIMIT)
+            eof!(guarded, deadline)
+            validate_reply!(reply, submission.fetch("request_id"))
+            reply
+          end
+        rescue Ace::Runtime::RuntimeUnavailableError, IOError, SystemCallError, Timeout::Error
+          {"version" => 1, "type" => "service_claim_unconfirmed", "required_action" => "inspect_canonical_service_status"}
+        end
+
+        private
+
+        def eof!(socket, deadline)
+          loop do
+            remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            unless remaining.positive? && IO.select([socket], nil, nil, remaining)
+              raise Ace::Runtime::RuntimeUnavailableError, "receiver reply EOF deadline expired"
+            end
+            byte = socket.read_nonblock(1, exception: false)
+            next if byte == :wait_readable
+            raise SecurityError, "receiver reply has trailing bytes" unless byte.nil?
+            return
+          end
+        end
+
+        def validate_reply!(reply, request_id)
+          unless reply.is_a?(Hash) && reply["version"].is_a?(Integer) && reply["version"] == 1
+            raise SecurityError, "receiver claim reply is invalid"
+          end
+          if reply["type"] == "service_claim_refused"
+            raise SecurityError, "receiver claim refusal is invalid" unless reply.keys.sort == %w[code type version] && %w[busy unavailable].include?(reply["code"])
+            return
+          end
+          data = reply["data"]
+          unless reply.keys.sort == %w[data type version] && reply["type"] == "service_claim_accepted" &&
+              data.is_a?(Hash) && (data.keys - %w[request_id generation journal_commit state claim replayed]).empty? &&
+              data["request_id"] == request_id && data["generation"].is_a?(Integer) && data["generation"].positive? &&
+              data["journal_commit"].is_a?(String) && data["journal_commit"].match?(/\A(?:[0-9a-f]{40}|[0-9a-f]{64})\z/) &&
+              %w[uncertain succeeded failed failed-settled].include?(data["state"]) && [true, false].include?(data["replayed"]) &&
+              (!data.key?("claim") || %w[created retained].include?(data["claim"]))
+            raise SecurityError, "receiver canonical claim identity differs"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/lab/organisms/protected_service_listener.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "socket"
+require_relative "protected_service_worker"
+
+module Ace
+  module Lab
+    module Organisms
+      # Fixed receiver ingress. The short connection observes canonical claim
+      # acceptance; it does not own or cancel the admitted execution lifetime.
+      class ProtectedServiceListener
+        LIMIT = 16_384
+        REQUEST_KEYS = %w[mutation_id submission transfer version].freeze
+
+        def initialize(mapping_id:, service_id:, deployment: Ace::Assign::Authority::Deployment.load,
+          kernel: Ace::Runtime::Molecules::ProtectedLinux.new, receiver: nil,
+          wire: Ace::Runtime::Molecules::ProtectedSocket)
+          @deployment, @kernel, @wire = deployment, kernel, wire
+          @map = deployment.mapping(mapping_id)
+          project = deployment.project(@map.fetch("project_id"))
+          @selection = project.fetch("service_receivers").fetch(service_id)
+          @credentials = project.fetch("peer_credentials").fetch(@selection.fetch("executor_uid").to_s)
+          @path = @selection.fetch("socket_path")
+          @receiver = receiver || ProtectedServiceReceiver.new(
+            mapping_id: mapping_id, service_id: service_id, deployment: deployment, kernel: kernel)
+          @worker = ProtectedServiceWorker.new(receiver: @receiver)
+          @stopping = false
+        end
+
+        def serve
+          own_principal!
+          parent = File.dirname(@path)
+          @wire.root_path!(parent, directory: true, owner: @selection.fetch("executor_uid"))
+          # The installed receiver must actually be able to publish a socket
+          # accessible to its mapped worker. No ambient ACL/root fixup exists.
+          unless @credentials.fetch("gid") == @map.fetch("worker_gid") || @credentials.fetch("groups").include?(@map.fetch("worker_gid"))
+            raise SecurityError, "receiver socket group is unavailable"
+          end
+          @lock = File.open("#{@path}.lock", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600)
+          stat = @lock.stat
+          unless stat.file? && stat.uid == Process.uid && stat.nlink == 1 && (stat.mode & 0o7777) == 0o600
+            raise SecurityError, "receiver listener lock differs"
+          end
+          unless @lock.flock(File::LOCK_EX | File::LOCK_NB)
+            raise Ace::Assign::AttemptErrors::Conflict, "receiver listener already active"
+          end
+          remove_stale_endpoint!
+          @listener = UNIXServer.new(@path)
+          File.chown(nil, @map.fetch("worker_gid"), @path)
+          File.chmod(0o660, @path)
+          @identity = @wire.socket_identity(@path)
+          until @stopping
+            begin
+              socket = @listener.accept
+              receive(socket)
+            ensure
+              socket&.close
+              socket = nil
+            end
+          end
+        rescue IOError, Errno::EBADF
+          raise unless @stopping
+        ensure
+          # Never release listener exclusion while an admitted worker exists.
+          # Actual operation transport supplies its separate bounded lifetime.
+          @worker.close(timeout: 1) until @worker.close(timeout: 0)
+          @listener&.close
+          remove_owned_endpoint!
+          @lock&.close
+        end
+
+        def stop
+          @stopping = true
+          @worker.close(timeout: 0)
+          @listener&.close
+        end
+
+        private
+
+        def receive(socket)
+          deadline = @wire.deadline(5)
+          peer = @kernel.peer(socket)
+          unless peer.values_at("uid", "gid", "groups") == @map.values_at("worker_uid", "worker_gid", "worker_groups")
+            raise SecurityError, "receiver worker peer differs"
+          end
+          guarded = Ace::Runtime::Molecules::ProtectedSocket::Ingress.new(socket)
+          request = @wire.read(guarded, deadline: deadline, limit: LIMIT)
+          if request.is_a?(Hash) && request["kind"] == "preview_workspace_prune"
+            unless request.keys.sort == %w[kind transfer version] && request["version"].is_a?(Integer) && request["version"] == 1
+              raise ArgumentError, "receiver preview header differs"
+            end
+            codec = Ace::Assign::Authority::TransferCodec.new(root: @selection.fetch("staging_root"))
+            bytes = codec.receive(guarded, descriptor: request.fetch("transfer"), purpose: :service_input, deadline: deadline) do |input|
+              raise ArgumentError, "preview requires one intent part" unless input.count == 1
+              input.bytes
+            end
+            intent = JSON.parse(bytes, create_additions: false, max_nesting: 16, allow_nan: false,
+              allow_comments: false, allow_duplicate_key: false)
+            intent = Atoms::ProtectedWorkspacePrunePreview.intent!(intent)
+            result = @receiver.preview_workspace_prune(intent: intent, peer: peer, deadline: deadline)
+            @wire.write(socket, result, deadline: deadline, limit: LIMIT)
+            return
+          end
+          strict_request!(request)
+          codec = Ace::Assign::Authority::TransferCodec.new(root: @selection.fetch("staging_root"))
+          bytes = codec.receive(guarded, descriptor: request.fetch("transfer"), purpose: :service_input, deadline: deadline) { |input| input.bytes }
+          completed = Queue.new
+          @worker.start(submission: request.fetch("submission"), peer: peer,
+            input_bytes: bytes, mutation_id: request.fetch("mutation_id"), deadline: deadline,
+            on_claim: lambda do |identity|
+              begin
+                @wire.write(socket, {"version" => 1, "type" => "service_claim_accepted", "data" => identity},
+                  deadline: deadline, limit: LIMIT)
+              rescue Ace::Runtime::RuntimeUnavailableError, IOError, SystemCallError
+                # Only bounded transport failure is swallowed. Original work
+                # continues and its accepted identity is observed via status.
+                nil
+              ensure
+                completed << true
+              end
+            end)
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          completed.pop(timeout: remaining) if remaining.positive?
+        rescue Ace::Assign::AttemptErrors::Conflict
+          refuse(socket, "busy", deadline)
+        rescue SecurityError, ArgumentError, KeyError, TypeError, Ace::Assign::Error,
+          Ace::Runtime::RuntimeUnavailableError, IOError, SystemCallError, Timeout::Error
+          refuse(socket, "unavailable", deadline)
+        end
+
+        def strict_request!(request)
+          unless request.is_a?(Hash) && request.keys.sort == REQUEST_KEYS &&
+              request["version"].is_a?(Integer) && request["version"] == 1 &&
+              request["transfer"].is_a?(Hash) && request["transfer"]["version"].is_a?(Integer) && request["transfer"]["version"] == 1 &&
+              request["mutation_id"].is_a?(String) && request["mutation_id"].match?(Ace::Assign::Molecules::JournalMutation::ID) &&
+              request["submission"].is_a?(Hash) && request["submission"].keys.sort == ProtectedServiceReceiver::SUBMISSION.sort
+            raise ArgumentError, "receiver request is invalid"
+          end
+          walk = [[request, 0]]
+          until walk.empty?
+            value, depth = walk.pop
+            raise ArgumentError, "receiver request depth exceeds bound" if depth > 16
+            children = value.is_a?(Hash) ? value.values : (value.is_a?(Array) ? value : [])
+            children.each { |child| walk << [child, depth + 1] }
+          end
+        end
+
+        def refuse(socket, code, deadline)
+          @wire.write(socket, {"version" => 1, "type" => "service_claim_refused", "code" => code}, deadline: deadline, limit: LIMIT)
+        rescue Ace::Runtime::RuntimeUnavailableError, IOError, SystemCallError
+          nil
+        end
+
+        def own_principal!
+          me = @kernel.capture(Process.pid)
+          unless me.values_at("uid", "gid", "groups") == [@selection.fetch("executor_uid"), @credentials.fetch("gid"), @credentials.fetch("groups")]
+            raise SecurityError, "receiver listener principal differs"
+          end
+          @kernel.live!(me)
+        end
+
+        def remove_stale_endpoint!
+          stat = File.lstat(@path)
+          unless stat.socket? && stat.uid == Process.uid
+            raise SecurityError, "receiver endpoint differs"
+          end
+          File.unlink(@path)
+        rescue Errno::ENOENT
+          nil
+        end
+
+        def remove_owned_endpoint!
+          File.unlink(@path) if @identity && @wire.socket_identity(@path) == @identity
+        rescue Errno::ENOENT
+          nil
+        end
+
+      end
+    end
+  end
+end

data/lib/ace/lab/organisms/protected_service_publication.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+require "securerandom"
+require "rubygems/package"
+require "ace/hitl/providers/lab"
+require_relative "../molecules/protected_publication_publisher"
+
+module Ace
+  module Lab
+    module Organisms
+      class ProtectedServiceReceiver
+        # One invocation is retained by the existing receiver worker, not a
+        # second effect queue. Only canonical continuation can authorize a push.
+        def begin_publication!(admitted:, materialized:, deadline:)
+          raise SecurityError, "publication receiver already owns an invocation" if @publication
+          @publication_hitl ||= Ace::Hitl::Providers::Lab.boundary_client(
+            socket_path: Ace::Hitl::Providers::Lab::DEFAULT_SOCKET_PATH,
+            policy: Ace::Hitl::Providers::Lab.grants_policy(grants_path: Ace::Hitl::Providers::Lab::DEFAULT_GRANTS_PATH))
+          params = admitted.fetch(:params)
+          started = admitted.fetch(:started)
+          unless started.fetch("executor_process_binding") == @kernel.capture(Process.pid)
+            raise SecurityError, "publication original receiver differs"
+          end
+          @publication = admitted.merge(materialized: materialized, handles: [], challenge: nil,
+            ask_id: nil, disposition: "dispatch_started")
+          hold_publication_artifact!(@publication)
+          verified = publication_registry!(@publication, deadline)
+          return complete_publication!(@publication, verified, deadline) if %w[succeeded failed].include?(verified.fetch("classification"))
+          unless verified.fetch("classification") == "absent"
+            @publication[:disposition] = "uncertain"
+            return publication_status("uncertain", verified.fetch("code"))
+          end
+          publication_push!(@publication, nil, deadline)
+        rescue StandardError, SecurityError
+          publication_status("uncertain", "publication_admission_unavailable")
+        end
+
+        def tick_publication!(request_id:, deadline:)
+          entry = @publication
+          return {"request_id" => request_id, "state" => "publication-blocked", "code" => "original-executor-unavailable"} unless entry && entry.dig(:params, "request_id") == request_id
+          check_publication!(entry)
+          status = publication_call!(entry, "service_status", {}, deadline).data
+          if %w[succeeded failed].include?(status["state"])
+            close_publication!(entry)
+            return projection(status)
+          end
+          if entry[:disposition] == "uncertain"
+            verified = publication_registry!(entry, deadline)
+            return complete_publication!(entry, verified, deadline) if verified["classification"] == "succeeded"
+            return publication_status("uncertain", "publication_outcome_unconfirmed")
+          end
+          unless status["dispatch_phase"] == "otp_pending" && status["publication_challenge_digest"] == entry.fetch(:challenge).fetch("publication_challenge_digest")
+            return publication_status("uncertain", "publication_challenge_changed")
+          end
+          publication_ask!(entry, deadline) unless entry[:ask_created]
+          ask = @publication_hitl.read(entry.fetch(:ask_id), deadline: deadline)
+          check_publication_ask!(entry, ask)
+          if %w[cancelled consumed].include?(ask["state"])
+            entry[:ask_id] = nil
+            entry[:ask_created] = false
+            entry[:ask_otp] = nil
+            return publication_status("publication-pending", "otp-answer-unavailable")
+          end
+          if ask.fetch("otp").fetch("expires_at") <= Time.now.to_i
+            @publication_hitl.cancel(entry.fetch(:ask_id), reason: "otp-answer-expired", deadline: deadline)
+            return publication_status("publication-pending", "otp-answer-expired")
+          end
+          return publication_status("publication-pending", "otp-answer-pending") unless ask["state"] == "answer-delivered"
+          return publication_status("publication-pending", "otp-tick-budget") if publication_remaining!(deadline) < 1.5
+          answer = @publication_hitl.consume(entry.fetch(:ask_id), timeout: 1, operation: "publish", deadline: deadline)
+          otp = answer["answer"]
+          unless otp.is_a?(String) && otp.match?(/\A[0-9]{6}\z/) && ask.fetch("otp").fetch("expires_at") > Time.now.to_i
+            return publication_status("publication-pending", "otp-answer-unavailable")
+          end
+          check_publication!(entry)
+          return publication_status("publication-pending", "otp-tick-budget") if publication_remaining!(deadline) < 1
+          status = publication_call!(entry, "service_status", {}, deadline).data
+          return publication_status("publication-pending", "otp-answer-expired") unless ask.fetch("otp").fetch("expires_at") > Time.now.to_i
+          entry[:disposition] = "uncertain" # A lost issuing reply is never retried.
+          issued = publication_call!(entry, "publication_continue", {
+            "expected_generation" => status.fetch("generation"), "input_digest" => entry.fetch(:params).fetch("input_digest"),
+            "challenge_digest" => entry.fetch(:challenge).fetch("publication_challenge_digest")}, deadline,
+            mutation_id: Digest::SHA256.hexdigest("publication-continue:#{entry.fetch(:challenge).fetch('publication_challenge_digest')}")).data
+          return publication_status("uncertain", "publication_issuing_unconfirmed") unless issued["invocation"] == "permitted" && issued["dispatch_phase"] == "issuing"
+          entry[:started] = issued
+          publication_push!(entry, otp, deadline)
+        rescue StandardError, SecurityError
+          publication_status("uncertain", "publication_tick_unavailable")
+        ensure
+          otp&.clear if otp.is_a?(String) && !otp.frozen?
+          answer&.delete("answer")
+        end
+
+        def publication_active?
+          !@publication.nil?
+        end
+
+        # Called only after the existing worker has stopped its bounded tick.
+        # Durable uncertain state and candidate bytes remain for read-only recovery.
+        def release_publication_lifetime!
+          @publication&.fetch(:handles)&.each { |_path, handle, _identity| handle.close unless handle.closed? }
+          @publication = nil
+        end
+
+        private
+
+        def hold_publication_artifact!(entry)
+          root = entry.fetch(:materialized).fetch("directory")
+          publication = entry.fetch(:envelope).fetch("input").fetch("publication")
+          components = publication.fetch("artifact_relative_path").split("/")
+          paths = [root]
+          components.each { |component| paths << File.join(paths.last, component) }
+          paths.each_with_index do |path, index|
+            handle = File.open(path, File::RDONLY | File::NOFOLLOW | File::NONBLOCK)
+            stat = handle.stat
+            valid = index == paths.size - 1 ? stat.file? && stat.size.between?(1, 64 * 1024 * 1024) : stat.directory?
+            unless valid && stat.uid == Process.uid && (stat.mode & 0o022).zero?
+              handle.close
+              raise SecurityError, "publication staging protection differs"
+            end
+            entry.fetch(:handles) << [path, handle, publication_identity(stat)]
+          end
+          entry[:artifact] = paths.last
+          spec = Gem::Package.new(paths.last).spec
+          unless spec.name == publication.fetch("gem_name") && spec.version.to_s == publication.fetch("version")
+            raise SecurityError, "publication artifact metadata differs"
+          end
+          entry[:platform] = spec.platform.to_s
+          check_publication!(entry)
+        end
+
+        def check_publication!(entry)
+          @kernel.live!(entry.fetch(:started).fetch("executor_process_binding"))
+          raise SecurityError, "publication receiver birth differs" unless @kernel.capture(Process.pid) == entry.fetch(:started).fetch("executor_process_binding")
+          entry.fetch(:handles).each do |path, handle, expected|
+            unless publication_identity(handle.stat) == expected && publication_identity(File.lstat(path)) == expected
+              raise SecurityError, "publication held staging changed"
+            end
+          end
+          artifact = entry.fetch(:handles).last.fetch(1)
+          artifact.rewind
+          digest = Digest::SHA256.new
+          while (chunk = artifact.read(65_536))
+            digest.update(chunk)
+          end
+          unless digest.hexdigest == entry.fetch(:envelope).fetch("input").fetch("target").fetch("artifact_digest")
+            raise SecurityError, "publication artifact bytes differ"
+          end
+        end
+
+        def publication_identity(stat)
+          stat.directory? ? [stat.dev, stat.ino, stat.uid, stat.gid, stat.mode] :
+            [stat.dev, stat.ino, stat.uid, stat.gid, stat.mode, stat.size, stat.mtime, stat.ctime]
+        end
+
+        def publication_registry!(entry, deadline)
+          check_publication!(entry)
+          @publication_publisher.verify!(publication: entry.fetch(:envelope).fetch("input").fetch("publication"),
+            artifact_digest: entry.fetch(:params).fetch("target").fetch("artifact_digest"), platform: entry.fetch(:platform), deadline: deadline)
+        end
+
+        def publication_push!(entry, otp, deadline)
+          check_publication!(entry)
+          entry[:disposition] = "uncertain"
+          result = @publication_publisher.push!(operation: entry.fetch(:operation),
+            publication: entry.fetch(:envelope).fetch("input").fetch("publication"), artifact: entry.fetch(:artifact),
+            artifact_digest: entry.fetch(:params).fetch("target").fetch("artifact_digest"), otp: otp, deadline: deadline)
+          check_publication!(entry)
+          verified = publication_registry!(entry, deadline)
+          return complete_publication!(entry, verified, deadline) if %w[succeeded failed].include?(verified["classification"])
+          if %w[otp-required otp-rejected otp-expired].include?(result["classification"]) && verified["classification"] == "absent"
+            return retain_publication_challenge!(entry, result, deadline)
+          end
+          return complete_publication!(entry, result, deadline) if result["classification"] == "failed" && verified["classification"] == "absent"
+          publication_status("uncertain", "publication_outcome_unconfirmed")
+        end
+
+        def retain_publication_challenge!(entry, result, deadline)
+          input = entry.fetch(:envelope).fetch("input")
+          params = entry.fetch(:params)
+          previous = entry[:challenge]&.fetch("publication_challenge_digest")
+          need = params.slice("request_id", "input_digest", "candidate_generation", "head").merge(
+            "schema" => "ace.publication-otp-need/v1", "classification" => result.fetch("classification"),
+            "claim_binding" => entry.fetch(:claim).fetch("claim_binding"),
+            "registry" => input.fetch("publication").fetch("registry"), "gem_name" => input.fetch("publication").fetch("gem_name"),
+            "version" => input.fetch("publication").fetch("version"), "artifact_digest" => input.fetch("target").fetch("artifact_digest"),
+            "dispatch_event_digest" => entry.fetch(:dispatch_event_digest), "previous_challenge_digest" => previous)
+          bytes = JSON.generate(need)
+          status = publication_call!(entry, "service_status", {}, deadline).data
+          entry[:challenge] = publication_call!(entry, "publication_challenge", {
+            "input_digest" => params.fetch("input_digest"), "expected_generation" => status.fetch("generation"),
+            "receipt_sha256" => Digest::SHA256.hexdigest(bytes)}, deadline,
+            mutation_id: Digest::SHA256.hexdigest("publication-challenge:#{Digest::SHA256.hexdigest(bytes)}"),
+            upload_parts: [bytes], purpose: :receipt_artifacts).data
+          entry[:disposition] = "otp_pending"
+          entry[:ask_id] = nil
+          entry[:ask_created] = false
+          entry[:ask_otp] = nil
+          publication_ask!(entry, deadline)
+          publication_status("publication-pending", result.fetch("classification"))
+        end
+
+        def publication_ask!(entry, deadline)
+          entry[:ask_id] ||= "publication-#{SecureRandom.hex(12)}"
+          binding = entry.fetch(:params).slice("assignment_id", "attempt_id", "request_id", "input_digest", "candidate_generation", "head").merge(
+            "schema" => "ace.hitl-publication-binding/v1", "mapping_id" => @mapping_id,
+            "claim_binding" => entry.fetch(:claim).fetch("claim_binding"),
+            "challenge_digest" => entry.fetch(:challenge).fetch("publication_challenge_digest"))
+          entry[:ask_binding] = immutable(binding)
+          entry[:ask_otp] ||= immutable({"operation" => "publish", "input_digest" => binding.fetch("input_digest"),
+            "result_ref" => entry.fetch(:challenge).fetch("publication_challenge_ref").fetch("ref"), "expires_at" => Time.now.to_i + 900})
+          @publication_hitl.create(id: entry.fetch(:ask_id), assignment: binding.fetch("assignment_id"), attempt: binding.fetch("attempt_id"),
+            project: @map.fetch("project_id"), harness: "lab-admin", plan: "Scoped RubyGems publication", kind: "otp",
+            question: "Provide the RubyGems OTP for this authorized artifact publication.", ace_hitl_id: entry.fetch(:ask_id),
+            publication_binding: binding, otp: entry.fetch(:ask_otp), deadline: deadline)
+          entry[:ask_created] = true
+        end
+
+        def check_publication_ask!(entry, ask)
+          unless ask["id"] == entry.fetch(:ask_id) && ask["publication_binding"] == entry.fetch(:ask_binding) &&
+              ask["publication_requester_peer"] == entry.fetch(:started).fetch("executor_process_binding") &&
+              ask.dig("otp", "input_digest") == entry.fetch(:params).fetch("input_digest") &&
+              ask.dig("otp", "result_ref") == entry.fetch(:challenge).fetch("publication_challenge_ref").fetch("ref")
+            raise SecurityError, "publication HITL retained request differs"
+          end
+        end
+
+        def publication_call!(entry, operation, extra, deadline, **options)
+          params = entry.fetch(:params).slice("assignment_id", "attempt_id", "candidate_generation", "head", "request_id")
+          params["claim_binding"] = entry.fetch(:claim).fetch("claim_binding") unless operation == "service_status"
+          @client.call(operation, params.merge(extra), timeout: publication_remaining!(deadline), **options)
+        end
+
+        def complete_publication!(entry, result, deadline)
+          check_publication!(entry)
+          input = entry.fetch(:envelope).fetch("input")
+          evidence = JSON.generate(input.merge("schema" => "ace.publication-result/v1", "classification" => result.fetch("classification"),
+            "code" => result.fetch("code"), "request_id" => entry.fetch(:params).fetch("request_id"),
+            "input_digest" => entry.fetch(:params).fetch("input_digest"), "claim_binding" => entry.fetch(:claim).fetch("claim_binding")))
+          receipt = entry.fetch(:envelope).fetch("request").slice(*Ace::Assign::Molecules::EvidenceJournal::TERMINAL_BINDING_FIELDS).merge(
+            "outcome" => result.fetch("classification") == "succeeded" ? "succeeded" : "failed",
+            "evidence" => [{"ref" => "publication-result", "sha256" => Digest::SHA256.hexdigest(evidence)}])
+          bytes = JSON.generate(receipt)
+          completed = publication_call!(entry, "complete_service", {"receipt_sha256" => Digest::SHA256.hexdigest(bytes)}, deadline,
+            mutation_id: Digest::SHA256.hexdigest("publication-complete:#{entry.fetch(:mutation_id)}"), upload_parts: [bytes, evidence], purpose: :receipt_artifacts).data
+          return publication_status("uncertain", "publication_completion_unconfirmed") unless %w[succeeded failed].include?(completed["state"])
+          close_publication!(entry)
+          projection(completed)
+        end
+
+        def close_publication!(entry)
+          entry.fetch(:handles).each { |_path, handle, _identity| handle.close }
+          cleanup_staging(entry.fetch(:materialized).fetch("directory"), entry.fetch(:staging_identity))
+          @publication = nil
+        end
+
+        def publication_remaining!(deadline)
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          raise IOError, "publication tick deadline expired" unless remaining.positive?
+          remaining
+        end
+
+        def publication_status(state, code)
+          {"request_id" => @publication&.dig(:params, "request_id"), "state" => state, "code" => code}.freeze
+        end
+      end
+    end
+  end
+end