@meistrari/agent-core 0.2.0-experimental.0 → 0.2.0-experimental.2

package/src/worker-runtime-client/frame-processor.ts

@@ -1,5 +1,5 @@
 import type { Logger } from '../logger'
-import type { CommandAckEnvelope, RuntimeStateEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
+import type { CommandAckEnvelope, DurableEventEnvelope, EphemeralEventEnvelope, RuntimeStateEnvelope, SupervisorToControlPlaneEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
 import type {
     ConnectionAuthorityStore,
     DurableEventSink,
@@ -9,6 +9,7 @@ import type {
     SessionSandboxRef,
     SupervisorConnection,
 } from './ports'
+import { Buffer } from 'node:buffer'
 import {
     supervisorBootstrapRejectionReceiptCloseCode,
     supervisorBootstrapRejectionReceiptCloseReason,
@@ -20,7 +21,9 @@ import { SupervisorRpcRequestManager } from './rpc-request-manager'
 export class SupervisorFrameProcessor {
     private readonly readiness = Promise.withResolvers<RuntimeStateEnvelope>()
     private readonly rpc: SupervisorRpcRequestManager
-    private processing = Promise.resolve()
+    private readonly inbox: ReturnType<typeof createOrderedSupervisorInbox>
+    private readonly persistenceGate = Promise.withResolvers<void>()
+    private journalSourceId: string | undefined
     private stopped = false
     private supervisorLivenessObserved = false
     private readinessSettled = false
@@ -39,8 +42,11 @@ export class SupervisorFrameProcessor {
         logger: Logger
         onCommandAck: (ack: CommandAckEnvelope) => void
         onFatal: (error: unknown) => void
+        maxFrames?: number
+        maxBytes?: number
     }) {
         void this.readiness.promise.catch(() => undefined)
+        void this.persistenceGate.promise.catch(() => undefined)
         this.rpc = new SupervisorRpcRequestManager({
             ref: dependencies.ref,
             generation: dependencies.generation,
@@ -49,14 +55,21 @@ export class SupervisorFrameProcessor {
             dispatcher: dependencies.rpcDispatcher,
             connection: dependencies.connection,
         })
+        this.inbox = createOrderedSupervisorInbox({
+            maxFrames: dependencies.maxFrames,
+            maxBytes: dependencies.maxBytes,
+            handle: (raw, _receivedAt, signal) => this.process(frameText(raw), signal),
+            onFailure: (error) => {
+                this.stop(error)
+                dependencies.onFatal(error)
+            },
+        })
     }
 
     receive(data: string | ArrayBuffer | Uint8Array): void {
         if (this.stopped)
             return
-        this.processing = this.processing.then(async () => await this.process(frameText(data))).catch((error) => {
-            this.dependencies.onFatal(error)
-        })
+        this.inbox.push(data)
     }
 
     async waitForReadiness(input: { signal: AbortSignal }): Promise<RuntimeStateEnvelope> {
@@ -73,6 +86,15 @@ export class SupervisorFrameProcessor {
         }
     }
 
+    /**
+     * Durable events replayed right after `runtime.state` wait here until the control plane has
+     * accepted this connection (authority and journal identity). Nothing is persisted or ACKed
+     * for a readiness the store later refuses.
+     */
+    confirmReadiness(): void {
+        this.persistenceGate.resolve()
+    }
+
     takeSupervisorLivenessObservation(): boolean {
         const observed = this.supervisorLivenessObserved
         this.supervisorLivenessObserved = false
@@ -84,69 +106,81 @@ export class SupervisorFrameProcessor {
             return
         this.stopped = true
         this.rpc.stop(reason)
-        this.readiness.reject(reason ?? new DOMException('Supervisor connection closed.', 'AbortError'))
+        void this.inbox.close()
+        const rejection = reason ?? new DOMException('Supervisor connection closed.', 'AbortError')
+        this.readiness.reject(rejection)
+        this.persistenceGate.reject(rejection)
     }
 
     async drain(): Promise<void> {
-        await this.processing
+        try {
+            await this.inbox.drain()
+        }
+        finally {
+            await this.rpc.drain()
+        }
     }
 
-    private async process(text: string): Promise<void> {
+    private async process(text: string, signal: AbortSignal): Promise<void> {
         if (this.stopped)
             return
-        const decoded = decodeSupervisorFrame(text)
-        if (decoded.kind !== 'ok')
-            throw new Error(decoded.detail)
-        const frame = decoded.frame
-        if (frame.kind === 'runtime.state') {
-            if (this.readinessSettled)
-                throw new Error('Supervisor emitted more than one bootstrap outcome.')
-            this.readinessSettled = true
-            this.readiness.resolve(frame)
-            return
-        }
-        if (frame.kind === 'command.ack') {
-            if (!this.readinessSettled && frame.status === 'rejected') {
+        await routeSupervisorFrame(text, new Date(), signal, {
+            runtimeState: (frame) => {
+                if (this.readinessSettled)
+                    throw new Error('Supervisor emitted more than one bootstrap outcome.')
                 this.readinessSettled = true
-                this.readiness.reject(new BootstrapRejectedAckError(frame))
-            }
-            this.dependencies.onCommandAck(frame)
-            return
-        }
-        if (frame.kind === 'rpc.request') {
-            this.rpc.handle(frame)
-            return
-        }
-        if (frame.kind === 'rpc.cancel') {
-            this.rpc.cancel(frame)
-            return
+                this.journalSourceId = frame.sourceId
+                this.readiness.resolve(frame)
+            },
+            commandAck: (frame) => {
+                if (!this.readinessSettled && frame.status === 'rejected') {
+                    this.readinessSettled = true
+                    this.readiness.reject(new BootstrapRejectedAckError(frame))
+                }
+                this.dependencies.onCommandAck(frame)
+            },
+            rpcRequest: frame => this.rpc.handle(frame),
+            rpcCancel: frame => this.rpc.cancel(frame),
+            ephemeral: (frame) => {
+                if (frame.body.type === 'supervisor.heartbeat') {
+                    this.supervisorLivenessObserved = true
+                    return
+                }
+                if (!isEphemeralWrappedAgentEvent(frame.body))
+                    throw new Error('Supervisor emitted an invalid ephemeral event body.')
+                this.dependencies.ephemeralEvents.publish({
+                    sessionId: this.dependencies.ref.sessionId,
+                    occurredAt: frame.occurredAt,
+                    body: frame.body,
+                })
+            },
+            persistDurable: async (frame, signal) => {
+                await this.waitForConfirmedReadiness(signal)
+                const result = await this.dependencies.durableEvents.record({
+                    ...this.dependencies.ref,
+                    runtimeConnectionId: this.dependencies.runtimeConnectionId,
+                    journalSourceId: this.journalSourceId,
+                    event: frame,
+                })
+                if (!('highWaterMark' in result))
+                    throw new Error(`Durable supervisor event was rejected as ${result.status}.`)
+                return result.highWaterMark
+            },
+            send: async encoded => this.dependencies.connection.send(encoded),
+        })
+    }
+
+    private async waitForConfirmedReadiness(signal: AbortSignal): Promise<void> {
+        signal.throwIfAborted()
+        const aborted = Promise.withResolvers<never>()
+        const onAbort = () => aborted.reject(signal.reason)
+        signal.addEventListener('abort', onAbort, { once: true })
+        try {
+            await Promise.race([this.persistenceGate.promise, aborted.promise])
         }
-        if (frame.delivery_semantics === 'ephemeral') {
-            if (frame.body.type === 'supervisor.heartbeat') {
-                this.supervisorLivenessObserved = true
-                return
-            }
-            if (!isEphemeralWrappedAgentEvent(frame.body))
-                throw new Error('Supervisor emitted an invalid ephemeral event body.')
-            this.dependencies.ephemeralEvents.publish({
-                sessionId: this.dependencies.ref.sessionId,
-                occurredAt: frame.occurredAt,
-                body: frame.body,
-            })
-            return
+        finally {
+            signal.removeEventListener('abort', onAbort)
         }
-
-        const result = await this.dependencies.durableEvents.record({
-            ...this.dependencies.ref,
-            runtimeConnectionId: this.dependencies.runtimeConnectionId,
-            event: frame,
-        })
-        if (!('highWaterMark' in result))
-            throw new Error(`Durable supervisor event was rejected as ${result.status}.`)
-        await this.dependencies.connection.send(encodeWireFrame({
-            kind: 'event.ack',
-            highWaterMark: result.highWaterMark,
-        }))
     }
 
     async acknowledgeBootstrapRejection(ack: Extract<CommandAckEnvelope, { status: 'rejected' }>): Promise<void> {
@@ -176,3 +210,153 @@ function frameText(data: string | ArrayBuffer | Uint8Array): string {
         return data
     return new TextDecoder().decode(data)
 }
+
+export type SupervisorFrameBytes = string | Uint8Array | ArrayBuffer
+interface QueuedFrame { raw: SupervisorFrameBytes, receivedAt: Date, bytes: number }
+
+/** Socket callbacks cannot await SQL. Bound retained frames and stop at the first failure. */
+export function createOrderedSupervisorInbox(options: {
+    handle: (raw: SupervisorFrameBytes, receivedAt: Date, signal: AbortSignal) => Promise<void>
+    onFailure: (error: unknown) => void
+    maxFrames?: number
+    maxBytes?: number
+}) {
+    const maxFrames = options.maxFrames ?? 128
+    const maxBytes = options.maxBytes ?? 4 * 1024 * 1024
+    if (!Number.isSafeInteger(maxFrames) || maxFrames < 1 || !Number.isSafeInteger(maxBytes) || maxBytes < 1)
+        throw new Error('invalid_supervisor_inbox_limits')
+    const stop = new AbortController()
+    const queue: QueuedFrame[] = []
+    let retainedFrames = 0
+    let retainedBytes = 0
+    let running: Promise<void> | undefined
+    let closed = false
+    let failed = false
+    let failure: unknown
+
+    function discard() {
+        for (const frame of queue) {
+            retainedFrames--
+            retainedBytes -= frame.bytes
+        }
+        queue.length = 0
+    }
+    function fail(error: unknown) {
+        if (closed)
+            return
+        closed = true
+        failed = true
+        failure = error
+        discard()
+        stop.abort(error)
+        // The owner closes the socket and stops command dispatch; reconnect uses SQL's cursor.
+        try {
+            options.onFailure(error)
+        }
+        catch (callbackError) {
+            failure = new AggregateError([error, callbackError], 'supervisor_inbox_failure_callback')
+        }
+    }
+    async function process() {
+        while (!closed && queue.length) {
+            const frame = queue.shift()!
+            try {
+                await options.handle(frame.raw, frame.receivedAt, stop.signal)
+            }
+            catch (error) {
+                fail(error)
+            }
+            finally {
+                retainedFrames--
+                retainedBytes -= frame.bytes
+            }
+        }
+    }
+    function schedule() {
+        running = Promise.resolve().then(process).finally(() => {
+            running = undefined
+            if (!closed && queue.length)
+                schedule()
+        })
+    }
+    return {
+        push(raw: SupervisorFrameBytes, receivedAt = new Date()) {
+            if (closed)
+                return false
+            const bytes = typeof raw === 'string' ? Buffer.byteLength(raw) : raw.byteLength
+            if (retainedFrames + 1 > maxFrames || retainedBytes + bytes > maxBytes) {
+                fail(new Error('supervisor_inbox_overflow'))
+                return false
+            }
+            // Binary buffers may be reused by the transport after the callback returns.
+            queue.push({ raw: typeof raw === 'string' ? raw : raw instanceof Uint8Array ? new Uint8Array(raw) : raw.slice(0), receivedAt: new Date(receivedAt), bytes })
+            retainedFrames++
+            retainedBytes += bytes
+            if (!running)
+                schedule()
+            return true
+        },
+        async drain() {
+            let pending = running
+            while (pending) {
+                await pending
+                pending = running
+            }
+            if (failed)
+                throw failure
+        },
+        async close() {
+            closed = true
+            discard()
+            stop.abort()
+            if (running)
+                await running
+        },
+    }
+}
+
+/** Product callbacks own authorization and persistence; routing and ACK ordering stay here. */
+export interface SupervisorFrameHandlers {
+    beforeFrame?: (frame: SupervisorToControlPlaneEnvelope) => void | Promise<void>
+    invalidFrame?: (detail: string) => Error
+    runtimeState: (frame: RuntimeStateEnvelope, signal: AbortSignal) => void | Promise<void>
+    commandAck: (frame: CommandAckEnvelope, signal: AbortSignal) => void | Promise<void>
+    rpcRequest: (frame: Extract<SupervisorToControlPlaneEnvelope, { kind: 'rpc.request' }>, signal: AbortSignal) => void | Promise<void>
+    rpcCancel: (frame: Extract<SupervisorToControlPlaneEnvelope, { kind: 'rpc.cancel' }>) => void | Promise<void>
+    ephemeral: (frame: EphemeralEventEnvelope, receivedAt: Date) => void | Promise<void>
+    persistDurable: (frame: DurableEventEnvelope, signal: AbortSignal) => Promise<number>
+    send: (encoded: string) => Promise<void>
+}
+
+export async function routeSupervisorFrame(raw: SupervisorFrameBytes, receivedAt: Date, signal: AbortSignal, handlers: SupervisorFrameHandlers): Promise<void> {
+    signal.throwIfAborted()
+    const decoded = decodeSupervisorFrame(frameText(raw))
+    if (decoded.kind !== 'ok')
+        throw handlers.invalidFrame?.(decoded.detail) ?? new Error(decoded.detail)
+    const frame = decoded.frame
+    await handlers.beforeFrame?.(frame)
+    signal.throwIfAborted()
+    switch (frame.kind) {
+        case 'runtime.state':
+            await handlers.runtimeState(frame, signal)
+            return
+        case 'command.ack':
+            await handlers.commandAck(frame, signal)
+            return
+        case 'rpc.request':
+            await handlers.rpcRequest(frame, signal)
+            return
+        case 'rpc.cancel':
+            await handlers.rpcCancel(frame)
+            return
+        case 'event': {
+            if (frame.delivery_semantics === 'ephemeral') {
+                await handlers.ephemeral(frame, receivedAt)
+                return
+            }
+            const highWaterMark = await handlers.persistDurable(frame, signal)
+            signal.throwIfAborted()
+            await handlers.send(encodeWireFrame({ kind: 'event.ack', highWaterMark }))
+        }
+    }
+}

package/src/worker-runtime-client/index.ts

@@ -1,12 +1,16 @@
-export { SessionCommandPump } from './command-pump'
+export { CommandAckTimeoutError, SessionCommandPump } from './command-pump'
+export { drainSupervisorConnection, openSupervisorConnectionBeforeDeadline, sendSupervisorBootstrap } from './connection-attempt'
 export { ControlAuthorityConfigurationError, createControlAuthoritySigner } from './control-authority-signer'
 export { createE2bSupervisorConnectionOpener, openE2bSupervisorConnection } from './e2b-supervisor-connection'
+export { createOrderedSupervisorInbox, routeSupervisorFrame } from './frame-processor'
+export type { SupervisorFrameBytes, SupervisorFrameHandlers } from './frame-processor'
 export { reconcileConnectionLeases } from './lease-reconciler'
 export type {
     BootstrapProvider,
     CommandSource,
     ConnectionAuthorityStore,
     ConnectionClaim,
+    ConnectionReadinessRejection,
     ControlAuthoritySigner,
     DurableEventResult,
     DurableEventSink,
@@ -23,5 +27,7 @@ export { createPostgresNotificationListener } from './postgres-notification-list
 export type { PostgresNotificationListener } from './postgres-notification-listener'
 export { createRpcDispatcher } from './rpc-dispatcher'
 export type { SupervisorRpcHandler } from './rpc-dispatcher'
-export { createSandboxConnectionRuntime } from './sandbox-connection-runtime'
+export { SupervisorRpcRequestManager } from './rpc-request-manager'
+export type { PreparedSupervisorRpcOptions, PrepareSupervisorRpc } from './rpc-request-manager'
+export { createSandboxConnectionRuntime, runConnectionAttempts } from './sandbox-connection-runtime'
 export { decryptToken, encryptToken, TokenDecryptionError } from './token-crypto'

package/src/worker-runtime-client/ports.ts

@@ -34,19 +34,50 @@ export interface ConnectionAuthorityStore {
         generation: number
         runtimeConnectionAttemptId: string
     }) => Promise<boolean>
+    /**
+     * `journalSourceId` is the supervisor journal identity announced by `runtime.state`
+     * (always present under `commandTransport: 'wss-v1'`). Stores must persist it on the first
+     * readiness of a session sandbox and answer a `ConnectionReadinessRejection` when a later
+     * readiness announces a different identity: a replaced journal restarts command and event
+     * sequences, so resuming from the previous cursor and ACK floor would skip commands and
+     * discard events as duplicates. `false` still means the claim lost its authority.
+     */
     markConnectionReady: (input: {
         sessionSandboxId: string
         generation: number
         runtimeConnectionId: string
         lastReceivedCommandSeq: number
         agentRunSnapshot: SupervisorAgentRunSnapshot
-    }) => Promise<boolean>
+        journalSourceId?: string
+    }) => Promise<boolean | ConnectionReadinessRejection>
     markConnectionReconnecting: (input: { sessionSandboxId: string, generation: number }) => Promise<void>
     releaseConnection: (input: { sessionSandboxId: string, generation: number, reason: string }) => Promise<void>
     markConnectionRetryExhausted: (input: { sessionSandboxId: string, generation: number, replicaId: string }) => Promise<boolean>
 }
 
+/** A readiness the store refuses deterministically; the session fails instead of retrying. */
+export interface ConnectionReadinessRejection {
+    ready: false
+    code: string
+    detail?: string
+}
+
 export interface CommandSource {
+    /** wss-v1 selects durable pending work; no high-water cursor implies receipt. */
+    pending?: (input: { sessionId: string, runtimeConnectionId: string }) => Promise<CommandEnvelope | undefined>
+    received?: (input: { sessionId: string, runtimeConnectionId: string, commandId: string, commandSeq: number }) => Promise<void>
+    /**
+     * wss-v1 persists a terminal supervisor rejection (invalid body, identity conflict) before the
+     * pump selects again. A rejected command must never be offered by `pending` afterwards.
+     */
+    rejected?: (input: {
+        sessionId: string
+        runtimeConnectionId: string
+        commandId: string
+        commandSeq: number
+        errorCode: string
+        detail?: string
+    }) => Promise<void>
     initial: (input: { sessionId: string }) => Promise<CommandEnvelope | undefined>
     next: (input: { sessionId: string, runtimeConnectionId: string, afterSequence: number }) => Promise<CommandEnvelope | undefined>
 }
@@ -60,7 +91,8 @@ export interface DurableEventSink {
     // Atomically persist the event, advance the ACK floor, and project its command outcome.
     // Only supervisor.command.applied marks a command applied; received ACKs never do.
     // Replayed events must not duplicate projections, including after reconnect.
-    record: (input: SessionSandboxRef & { runtimeConnectionId: string, event: DurableEventEnvelope }) => Promise<DurableEventResult>
+    // `journalSourceId` identifies the supervisor journal that numbered the event (wss-v1 only).
+    record: (input: SessionSandboxRef & { runtimeConnectionId: string, journalSourceId?: string, event: DurableEventEnvelope }) => Promise<DurableEventResult>
 }
 
 export interface EphemeralEventSink {