@meistrari/agent-core 0.2.0-experimental.0 → 0.2.0-experimental.1

package/src/worker-runtime-client/rpc-request-manager.ts

@@ -2,137 +2,182 @@ import type { SupervisorRpcCancelEnvelope, SupervisorRpcRequestEnvelope, Supervi
 import type { ConnectionAuthorityStore, RpcDispatcher, SessionSandboxRef, SupervisorConnection } from './ports'
 import { isAppError } from '../errors'
 
-const maximumConcurrentRequests = 32
+/** Preparation is awaited in frame order; the returned external operation runs concurrently. */
+export type PrepareSupervisorRpc = (request: SupervisorRpcRequestEnvelope, signal: AbortSignal) => Promise<() => Promise<unknown>>
+interface RpcError { code: string, message: string, retryable: boolean }
+export interface PreparedSupervisorRpcOptions {
+    prepare: PrepareSupervisorRpc
+    send: (response: SupervisorRpcResponseEnvelope) => Promise<void>
+    onFailure: (error: unknown) => void
+    maximumConcurrentRequests?: number
+    maximumDeadlineMs?: number
+    capacityError?: RpcError
+    formatError?: (error: unknown, aborted: boolean) => RpcError | null
+}
+interface LegacyDependencies {
+    ref: SessionSandboxRef
+    generation: number
+    runtimeConnectionAttemptId: string
+    authorityStore: ConnectionAuthorityStore
+    dispatcher: RpcDispatcher
+    connection: SupervisorConnection
+}
+interface ActiveRequest {
+    stop: AbortController
+    cancelled: boolean
+    done: ReturnType<typeof Promise.withResolvers<void>>
+}
 
+/** One lifecycle implementation for both legacy authority ports and ordered preparation ports. */
 export class SupervisorRpcRequestManager {
-    private readonly active = new Map<string, AbortController>()
+    private readonly active = new Map<string, ActiveRequest>()
     private stopped = false
+    private readonly options: PreparedSupervisorRpcOptions
+    private readonly legacyConnection?: SupervisorConnection
 
-    constructor(private readonly dependencies: {
-        ref: SessionSandboxRef
-        generation: number
-        runtimeConnectionAttemptId: string
-        authorityStore: ConnectionAuthorityStore
-        dispatcher: RpcDispatcher
-        connection: SupervisorConnection
-    }) {}
+    constructor(dependencies: PreparedSupervisorRpcOptions | LegacyDependencies) {
+        this.options = 'prepare' in dependencies ? dependencies : legacyOptions(dependencies)
+        this.legacyConnection = 'prepare' in dependencies ? undefined : dependencies.connection
+        for (const limit of [this.options.maximumConcurrentRequests ?? 32, this.options.maximumDeadlineMs ?? 30000]) {
+            if (!Number.isSafeInteger(limit) || limit < 1)
+                throw new RangeError('RPC limits must be positive safe integers')
+        }
+    }
 
+    /** Compatibility entrypoint; the frame processor need not await external I/O. */
     handle(request: SupervisorRpcRequestEnvelope): void {
-        if (this.stopped)
-            return
-        if (this.active.has(request.requestId)) {
-            this.dependencies.connection.close(1008, 'Supervisor RPC request ID was reused.')
-            this.stop(new Error('Supervisor RPC request ID was reused.'))
+        void this.request(request, new AbortController().signal).catch((error) => {
+            if (this.legacyConnection && error instanceof Error && error.message === 'duplicate_active_rpc_request') {
+                this.legacyConnection.close(1008, 'Supervisor RPC request ID was reused.')
+                this.stop(error)
+            }
+            else {
+                this.failed(error)
+            }
+        })
+    }
+
+    async request(request: SupervisorRpcRequestEnvelope, parent: AbortSignal): Promise<void> {
+        if (this.stopped || parent.aborted)
             return
-        }
-        if (this.active.size >= maximumConcurrentRequests) {
-            this.sendFailure({
-                requestId: request.requestId,
-                code: 'agent-core.rpc-capacity-exceeded',
-                message: 'Supervisor RPC capacity is unavailable.',
-                retryable: true,
-            })
+        if (this.active.has(request.requestId))
+            throw new Error('duplicate_active_rpc_request')
+        if (this.active.size >= (this.options.maximumConcurrentRequests ?? 32)) {
+            await this.sendFailure(request.requestId, this.options.capacityError ?? { code: 'rpc_capacity_exceeded', message: 'rpc_capacity_exceeded', retryable: true })
             return
         }
-        const controller = new AbortController()
-        const timeout = setTimeout(() => {
-            controller.abort(new DOMException('Supervisor RPC deadline elapsed.', 'TimeoutError'))
-        }, request.deadlineMs)
+        const entry: ActiveRequest = { stop: new AbortController(), cancelled: false, done: Promise.withResolvers<void>() }
+        this.active.set(request.requestId, entry)
+        const deadline = new AbortController()
+        const timeout = setTimeout(() => deadline.abort(new DOMException('Supervisor RPC deadline elapsed.', 'TimeoutError')), Math.min(request.deadlineMs, this.options.maximumDeadlineMs ?? 30000))
         timeout.unref?.()
-        this.active.set(request.requestId, controller)
-        void this.execute({ request, controller }).finally(() => {
+        const signal = AbortSignal.any([parent, entry.stop.signal, deadline.signal])
+        const finish = () => {
             clearTimeout(timeout)
-            if (this.active.get(request.requestId) === controller)
-                this.active.delete(request.requestId)
-        })
+            this.active.delete(request.requestId)
+            entry.done.resolve()
+        }
+        const respondFailure = async (error: unknown) => {
+            if (this.stopped || parent.aborted || entry.cancelled)
+                return
+            const code = signal.aborted ? 'rpc_deadline_exceeded' : 'rpc_failed'
+            const failure = this.options.formatError
+                ? this.options.formatError(error, signal.aborted)
+                : { code, message: code, retryable: signal.aborted }
+            if (failure)
+                await this.sendFailure(request.requestId, failure)
+        }
+        let execute: () => Promise<unknown>
+        try {
+            execute = await this.options.prepare(request, signal)
+            signal.throwIfAborted()
+        }
+        catch (error) {
+            try {
+                await respondFailure(error)
+            }
+            finally {
+                finish()
+            }
+            return
+        }
+        void (async () => {
+            let value: unknown
+            try {
+                value = await execute()
+                signal.throwIfAborted()
+            }
+            catch (error) {
+                await respondFailure(error)
+                return
+            }
+            if (!this.stopped && !entry.cancelled && !parent.aborted)
+                await this.options.send({ kind: 'rpc.response', requestId: request.requestId, result: { status: 'ok', value } })
+        })().catch(error => this.failed(error)).finally(finish)
     }
 
-    cancel(frame: SupervisorRpcCancelEnvelope): void {
-        this.active.get(frame.requestId)?.abort(new DOMException('Supervisor RPC was cancelled.', 'AbortError'))
+    cancel(frame: SupervisorRpcCancelEnvelope | string): void {
+        const entry = this.active.get(typeof frame === 'string' ? frame : frame.requestId)
+        if (entry) {
+            entry.cancelled = true
+            entry.stop.abort(new DOMException('Supervisor RPC was cancelled.', 'AbortError'))
+        }
     }
 
     stop(reason: unknown = new DOMException('Supervisor connection closed.', 'AbortError')): void {
-        if (this.stopped)
-            return
         this.stopped = true
-        for (const controller of this.active.values())
-            controller.abort(reason)
-        this.active.clear()
+        for (const entry of this.active.values()) {
+            entry.cancelled = true
+            entry.stop.abort(reason)
+        }
+    }
+
+    async drain(): Promise<void> {
+        await Promise.all([...this.active.values()].map(entry => entry.done.promise))
     }
 
-    private async execute(input: {
-        request: SupervisorRpcRequestEnvelope
-        controller: AbortController
-    }): Promise<void> {
+    async close(): Promise<void> {
+        this.stop()
+        await this.drain()
+    }
+
+    private async sendFailure(requestId: string, error: RpcError): Promise<void> {
+        if (!this.stopped)
+            await this.options.send({ kind: 'rpc.response', requestId, result: { status: 'error', error } })
+    }
+
+    private failed(error: unknown): void {
         try {
-            const authorized = await this.dependencies.authorityStore.confirmConnectionAuthority({
-                sessionSandboxId: this.dependencies.ref.sessionSandboxId,
-                generation: this.dependencies.generation,
-                runtimeConnectionAttemptId: this.dependencies.runtimeConnectionAttemptId,
-            })
-            if (!authorized)
-                throw new RpcAuthorityLostError()
-            const value = await this.dependencies.dispatcher.handle({
-                ...this.dependencies.ref,
-                request: input.request,
-                signal: input.controller.signal,
-            })
-            if (input.controller.signal.aborted || this.stopped)
-                return
-            await this.dependencies.connection.send(JSON.stringify({
-                kind: 'rpc.response',
-                requestId: input.request.requestId,
-                result: { status: 'ok', value },
-            } satisfies SupervisorRpcResponseEnvelope))
+            this.options.onFailure(error)
         }
-        catch (error) {
-            if (input.controller.signal.aborted || this.stopped)
-                return
-            if (error instanceof RpcAuthorityLostError) {
-                this.sendFailure({
-                    requestId: input.request.requestId,
-                    code: 'agent-core.connection-authority-lost',
-                    message: 'Supervisor connection authority is no longer current.',
-                    retryable: true,
-                })
-                return
-            }
-            if (isAppError(error)) {
-                this.sendFailure({
-                    requestId: input.request.requestId,
-                    code: error.code,
-                    message: error.publicMessage,
-                    retryable: error.retryable,
-                })
-                return
-            }
-            this.sendFailure({
-                requestId: input.request.requestId,
-                code: 'agent-core.rpc-failed',
-                message: 'Supervisor RPC failed.',
-                retryable: true,
-            })
+        catch {
+            this.stop(error)
         }
     }
+}
 
-    private sendFailure(input: {
-        requestId: string
-        code: string
-        message: string
-        retryable: boolean
-    }): void {
-        if (this.stopped)
-            return
-        void Promise.resolve(this.dependencies.connection.send(JSON.stringify({
-            kind: 'rpc.response',
-            requestId: input.requestId,
-            result: {
-                status: 'error',
-                error: { code: input.code, message: input.message, retryable: input.retryable },
-            },
-        } satisfies SupervisorRpcResponseEnvelope))).catch(() => {
-            this.dependencies.connection.close(1011, 'Supervisor RPC response delivery failed.')
-        })
+function legacyOptions(dependencies: LegacyDependencies): PreparedSupervisorRpcOptions {
+    return {
+        maximumConcurrentRequests: 32,
+        maximumDeadlineMs: Number.MAX_SAFE_INTEGER,
+        prepare: async (request, signal) => {
+            const authorized = await dependencies.authorityStore.confirmConnectionAuthority({ sessionSandboxId: dependencies.ref.sessionSandboxId, generation: dependencies.generation, runtimeConnectionAttemptId: dependencies.runtimeConnectionAttemptId })
+            if (!authorized)
+                throw new RpcAuthorityLostError()
+            return () => dependencies.dispatcher.handle({ ...dependencies.ref, request, signal })
+        },
+        send: async (response) => { await dependencies.connection.send(JSON.stringify(response)) },
+        onFailure: () => { dependencies.connection.close(1011, 'Supervisor RPC response delivery failed.') },
+        capacityError: { code: 'agent-core.rpc-capacity-exceeded', message: 'Supervisor RPC capacity is unavailable.', retryable: true },
+        formatError: (error, aborted) => {
+            if (aborted)
+                return null
+            if (error instanceof RpcAuthorityLostError)
+                return { code: 'agent-core.connection-authority-lost', message: 'Supervisor connection authority is no longer current.', retryable: true }
+            if (isAppError(error))
+                return { code: error.code, message: error.publicMessage, retryable: error.retryable }
+            return { code: 'agent-core.rpc-failed', message: 'Supervisor RPC failed.', retryable: true }
+        },
     }
 }
 

package/src/worker-runtime-client/sandbox-connection-runtime.ts

@@ -151,34 +151,41 @@ class OwnedConnection {
             this.claimed?.(claim.sessionId)
             let establishmentDeadline = performance.now() + this.dependencies.timings.establishmentBudgetMs
             let backoff = this.dependencies.timings.retryInitialDelayMs
-            while (!this.controller.signal.aborted) {
-                const attempt = new SessionSandboxConnectionAttempt({
-                    claim,
-                    establishmentDeadline,
-                    replicaId: this.dependencies.replicaId,
-                    trafficTokenEncryptionKey: this.dependencies.trafficTokenEncryptionKey,
-                    signControlAuthority: this.dependencies.signControlAuthority,
-                    openSupervisorConnection: this.dependencies.openSupervisorConnection,
-                    authorityStore: this.dependencies.authorityStore,
-                    commands: this.dependencies.commands,
-                    durableEvents: this.dependencies.durableEvents,
-                    ephemeralEvents: this.dependencies.ephemeralEvents,
-                    rpc: this.dependencies.rpc,
-                    bootstrap: this.dependencies.bootstrap,
-                    failures: this.dependencies.failures,
-                    timings: this.dependencies.timings,
-                    logger: this.dependencies.logger,
-                    parentSignal: this.controller.signal,
-                })
-                this.attempt = attempt
-                try {
-                    await attempt.run()
-                }
-                catch (error) {
-                    attempt.stop(error)
-                    await attempt.drain()
+            const exhaustedBudget = await runConnectionAttempts<boolean>({
+                signal: this.controller.signal,
+                attempt: async () => {
+                    const attempt = new SessionSandboxConnectionAttempt({
+                        claim,
+                        establishmentDeadline,
+                        replicaId: this.dependencies.replicaId,
+                        trafficTokenEncryptionKey: this.dependencies.trafficTokenEncryptionKey,
+                        signControlAuthority: this.dependencies.signControlAuthority,
+                        openSupervisorConnection: this.dependencies.openSupervisorConnection,
+                        authorityStore: this.dependencies.authorityStore,
+                        commands: this.dependencies.commands,
+                        durableEvents: this.dependencies.durableEvents,
+                        ephemeralEvents: this.dependencies.ephemeralEvents,
+                        rpc: this.dependencies.rpc,
+                        bootstrap: this.dependencies.bootstrap,
+                        failures: this.dependencies.failures,
+                        timings: this.dependencies.timings,
+                        logger: this.dependencies.logger,
+                        parentSignal: this.controller.signal,
+                    })
+                    this.attempt = attempt
+                    try {
+                        await attempt.run()
+                        return { delayMs: 0 }
+                    }
+                    catch (error) {
+                        attempt.stop(error)
+                        await attempt.drain()
+                        throw error
+                    }
+                },
+                onFailure: async (error) => {
                     if (this.controller.signal.aborted)
-                        return
+                        return { done: false }
                     if (error instanceof DeterministicEstablishmentError) {
                         if (!error.reported) {
                             await this.dependencies.failures.failDeterministicEstablishment({
@@ -190,10 +197,10 @@ class OwnedConnection {
                                 detail: error.detail,
                             })
                         }
-                        return
+                        return { done: false }
                     }
                     if (error instanceof ConnectionAuthorityLostError)
-                        return
+                        return { done: false }
                     if (error instanceof TokenDecryptionError) {
                         await this.dependencies.failures.markSessionFailed({
                             sessionId: claim.sessionId,
@@ -201,7 +208,7 @@ class OwnedConnection {
                             providerSandboxId: claim.providerSandboxId,
                             code: 'agent-core.traffic-token-decryption-failed',
                         })
-                        return
+                        return { done: false }
                     }
                     if (error instanceof ConnectionClosedError && error.ready) {
                         establishmentDeadline = performance.now() + this.dependencies.timings.establishmentBudgetMs
@@ -212,12 +219,13 @@ class OwnedConnection {
                         generation: claim.runtimeConnectionGeneration,
                     })
                     if (performance.now() + backoff >= establishmentDeadline)
-                        break
-                    await abortableDelay(backoff, this.controller.signal)
+                        return { done: true }
+                    const delayMs = backoff
                     backoff = Math.min(backoff * 2, this.dependencies.timings.retryMaxDelayMs)
-                }
-            }
-            if (!this.controller.signal.aborted) {
+                    return { delayMs }
+                },
+            })
+            if (exhaustedBudget && !this.controller.signal.aborted) {
                 const exhausted = await this.dependencies.authorityStore.markConnectionRetryExhausted({
                     sessionSandboxId: claim.sessionSandboxId,
                     generation: claim.runtimeConnectionGeneration,
@@ -299,3 +307,32 @@ function validatedTimings(timings: SandboxConnectionRuntimeTimings): SandboxConn
         throw new RangeError('retryInitialDelayMs must not exceed retryMaxDelayMs.')
     return timings
 }
+
+export type ConnectionAttemptOutcome<T> = { done: T } | { delayMs: number }
+
+/** Retry scheduling only. Authority, failure classification and budgets belong to callers. */
+export async function runConnectionAttempts<T>(options: {
+    signal: AbortSignal
+    attempt: (attemptNumber: number) => Promise<ConnectionAttemptOutcome<T>>
+    onFailure: (error: unknown, attemptNumber: number) => Promise<ConnectionAttemptOutcome<T>>
+}): Promise<T> {
+    let attemptNumber = 0
+    while (true) {
+        options.signal.throwIfAborted()
+        let outcome: ConnectionAttemptOutcome<T>
+        try {
+            outcome = await options.attempt(attemptNumber)
+        }
+        catch (error) {
+            options.signal.throwIfAborted()
+            outcome = await options.onFailure(error, attemptNumber)
+        }
+        options.signal.throwIfAborted()
+        if ('done' in outcome)
+            return outcome.done
+        if (!Number.isSafeInteger(outcome.delayMs) || outcome.delayMs < 0)
+            throw new RangeError('Connection retry delay must be a nonnegative safe integer.')
+        await abortableDelay(outcome.delayMs, options.signal)
+        attemptNumber++
+    }
+}