@meistrari/agent-core 0.0.0 → 0.1.0

package/src/worker-runtime-client/frame-processor.ts

@@ -0,0 +1,178 @@
+import type { Logger } from '../logger'
+import type { CommandAckEnvelope, RuntimeStateEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
+import type {
+    ConnectionAuthorityStore,
+    DurableEventSink,
+    EphemeralEventSink,
+    RpcDispatcher,
+    SessionFailureSink,
+    SessionSandboxRef,
+    SupervisorConnection,
+} from './ports'
+import {
+    supervisorBootstrapRejectionReceiptCloseCode,
+    supervisorBootstrapRejectionReceiptCloseReason,
+} from '../supervisor-protocol/bootstrap-rejection-receipt'
+import { isEphemeralWrappedAgentEvent } from '../supervisor-protocol/durability'
+import { decodeSupervisorFrame, encodeWireFrame } from '../supervisor-protocol/wire-codec'
+import { SupervisorRpcRequestManager } from './rpc-request-manager'
+
+export class SupervisorFrameProcessor {
+    private readonly readiness = Promise.withResolvers<RuntimeStateEnvelope>()
+    private readonly rpc: SupervisorRpcRequestManager
+    private processing = Promise.resolve()
+    private stopped = false
+    private supervisorLivenessObserved = false
+    private readinessSettled = false
+
+    constructor(private readonly dependencies: {
+        ref: SessionSandboxRef
+        generation: number
+        runtimeConnectionId: string
+        runtimeConnectionAttemptId: string
+        connection: SupervisorConnection
+        authorityStore: ConnectionAuthorityStore
+        durableEvents: DurableEventSink
+        ephemeralEvents: EphemeralEventSink
+        rpcDispatcher: RpcDispatcher
+        failures: SessionFailureSink
+        logger: Logger
+        onCommandAck: (ack: CommandAckEnvelope) => void
+        onFatal: (error: unknown) => void
+    }) {
+        void this.readiness.promise.catch(() => undefined)
+        this.rpc = new SupervisorRpcRequestManager({
+            ref: dependencies.ref,
+            generation: dependencies.generation,
+            runtimeConnectionAttemptId: dependencies.runtimeConnectionAttemptId,
+            authorityStore: dependencies.authorityStore,
+            dispatcher: dependencies.rpcDispatcher,
+            connection: dependencies.connection,
+        })
+    }
+
+    receive(data: string | ArrayBuffer | Uint8Array): void {
+        if (this.stopped)
+            return
+        this.processing = this.processing.then(async () => await this.process(frameText(data))).catch((error) => {
+            this.dependencies.onFatal(error)
+        })
+    }
+
+    async waitForReadiness(input: { signal: AbortSignal }): Promise<RuntimeStateEnvelope> {
+        if (input.signal.aborted)
+            throw input.signal.reason
+        const aborted = Promise.withResolvers<never>()
+        const onAbort = () => aborted.reject(input.signal.reason)
+        input.signal.addEventListener('abort', onAbort, { once: true })
+        try {
+            return await Promise.race([this.readiness.promise, aborted.promise])
+        }
+        finally {
+            input.signal.removeEventListener('abort', onAbort)
+        }
+    }
+
+    takeSupervisorLivenessObservation(): boolean {
+        const observed = this.supervisorLivenessObserved
+        this.supervisorLivenessObserved = false
+        return observed
+    }
+
+    stop(reason?: unknown): void {
+        if (this.stopped)
+            return
+        this.stopped = true
+        this.rpc.stop(reason)
+        this.readiness.reject(reason ?? new DOMException('Supervisor connection closed.', 'AbortError'))
+    }
+
+    async drain(): Promise<void> {
+        await this.processing
+    }
+
+    private async process(text: string): Promise<void> {
+        if (this.stopped)
+            return
+        const decoded = decodeSupervisorFrame(text)
+        if (decoded.kind !== 'ok')
+            throw new Error(decoded.detail)
+        const frame = decoded.frame
+        if (frame.kind === 'runtime.state') {
+            if (this.readinessSettled)
+                throw new Error('Supervisor emitted more than one bootstrap outcome.')
+            this.readinessSettled = true
+            this.readiness.resolve(frame)
+            return
+        }
+        if (frame.kind === 'command.ack') {
+            if (!this.readinessSettled && frame.status === 'rejected') {
+                this.readinessSettled = true
+                this.readiness.reject(new BootstrapRejectedAckError(frame))
+            }
+            this.dependencies.onCommandAck(frame)
+            return
+        }
+        if (frame.kind === 'rpc.request') {
+            this.rpc.handle(frame)
+            return
+        }
+        if (frame.kind === 'rpc.cancel') {
+            this.rpc.cancel(frame)
+            return
+        }
+        if (frame.delivery_semantics === 'ephemeral') {
+            if (frame.body.type === 'supervisor.heartbeat') {
+                this.supervisorLivenessObserved = true
+                return
+            }
+            if (!isEphemeralWrappedAgentEvent(frame.body))
+                throw new Error('Supervisor emitted an invalid ephemeral event body.')
+            this.dependencies.ephemeralEvents.publish({
+                sessionId: this.dependencies.ref.sessionId,
+                occurredAt: frame.occurredAt,
+                body: frame.body,
+            })
+            return
+        }
+
+        const result = await this.dependencies.durableEvents.record({
+            ...this.dependencies.ref,
+            runtimeConnectionId: this.dependencies.runtimeConnectionId,
+            event: frame,
+        })
+        if (!('highWaterMark' in result))
+            throw new Error(`Durable supervisor event was rejected as ${result.status}.`)
+        await this.dependencies.connection.send(encodeWireFrame({
+            kind: 'event.ack',
+            highWaterMark: result.highWaterMark,
+        }))
+    }
+
+    async acknowledgeBootstrapRejection(ack: Extract<CommandAckEnvelope, { status: 'rejected' }>): Promise<void> {
+        await this.dependencies.failures.failDeterministicEstablishment({
+            ...this.dependencies.ref,
+            generation: this.dependencies.generation,
+            code: ack.errorCode,
+            detail: ack.detail,
+        })
+        this.dependencies.connection.close(
+            supervisorBootstrapRejectionReceiptCloseCode,
+            supervisorBootstrapRejectionReceiptCloseReason,
+        )
+    }
+}
+
+export class BootstrapRejectedAckError extends Error {
+    override readonly name = 'BootstrapRejectedAckError'
+
+    constructor(readonly ack: Extract<CommandAckEnvelope, { status: 'rejected' }>) {
+        super(ack.detail ?? ack.errorCode)
+    }
+}
+
+function frameText(data: string | ArrayBuffer | Uint8Array): string {
+    if (typeof data === 'string')
+        return data
+    return new TextDecoder().decode(data)
+}

package/src/worker-runtime-client/index.ts

@@ -0,0 +1,27 @@
+export { SessionCommandPump } from './command-pump'
+export { ControlAuthorityConfigurationError, createControlAuthoritySigner } from './control-authority-signer'
+export { createE2bSupervisorConnectionOpener, openE2bSupervisorConnection } from './e2b-supervisor-connection'
+export { reconcileConnectionLeases } from './lease-reconciler'
+export type {
+    BootstrapProvider,
+    CommandSource,
+    ConnectionAuthorityStore,
+    ConnectionClaim,
+    ControlAuthoritySigner,
+    DurableEventResult,
+    DurableEventSink,
+    EphemeralEventSink,
+    RpcDispatcher,
+    RuntimeNotifications,
+    SandboxConnectionRuntimeTimings,
+    SessionFailureSink,
+    SessionSandboxRef,
+    SupervisorConnection,
+    SupervisorConnectionOpener,
+} from './ports'
+export { createPostgresNotificationListener } from './postgres-notification-listener'
+export type { PostgresNotificationListener } from './postgres-notification-listener'
+export { createRpcDispatcher } from './rpc-dispatcher'
+export type { SupervisorRpcHandler } from './rpc-dispatcher'
+export { createSandboxConnectionRuntime } from './sandbox-connection-runtime'
+export { decryptToken, encryptToken, TokenDecryptionError } from './token-crypto'

package/src/worker-runtime-client/lease-reconciler.ts

@@ -0,0 +1,14 @@
+import type { ConnectionAuthorityStore } from './ports'
+
+export async function reconcileConnectionLeases(input: {
+    authorityStore: ConnectionAuthorityStore
+    limit?: number
+}): Promise<number> {
+    const candidates = await input.authorityStore.listConnectionCandidates({ limit: input.limit ?? 100 })
+    if (candidates.length > 0) {
+        await input.authorityStore.notifyConnectionDemand({
+            sessionSandboxIds: candidates.map(candidate => candidate.sessionSandboxId),
+        })
+    }
+    return candidates.length
+}

package/src/worker-runtime-client/ports.ts

@@ -0,0 +1,137 @@
+import type { EphemeralCredentials, SessionBootstrapBody } from '../supervisor-protocol/bootstrap'
+import type { WireCommandBody } from '../supervisor-protocol/command-body'
+import type { CommandEnvelope } from '../supervisor-protocol/envelopes/control-plane-to-supervisor'
+import type { DurableEventEnvelope, EphemeralEventEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
+import type { SupervisorRpcRequestEnvelope } from '../supervisor-protocol/rpc'
+import type { SupervisorAgentRunSnapshot } from '../supervisor-protocol/supervisor-agent-run-snapshot'
+
+export interface SessionSandboxRef {
+    sessionId: string
+    sessionSandboxId: string
+    providerSandboxId: string
+}
+
+export type ConnectionClaim = SessionSandboxRef & {
+    runtimeConnectionGeneration: number
+    runtimeConnectionId: string
+    trafficAccessTokenEncrypted: string
+    leaseExpiresAt: Date
+}
+
+export interface ConnectionAuthorityStore {
+    listConnectionCandidates: (input: { limit: number }) => Promise<{ sessionSandboxId: string }[]>
+    notifyConnectionDemand: (input: { sessionSandboxIds: string[] }) => Promise<void>
+    claimConnection: (input: { sessionSandboxId: string, replicaId: string, leaseDurationMs: number }) => Promise<ConnectionClaim | undefined>
+    renewConnectionLease: (input: {
+        sessionSandboxId: string
+        replicaId: string
+        generation: number
+        leaseDurationMs: number
+        supervisorLivenessObserved: boolean
+    }) => Promise<{ renewed: boolean, leaseExpiresAt?: Date, hasOutstandingCommand: boolean }>
+    confirmConnectionAuthority: (input: {
+        sessionSandboxId: string
+        generation: number
+        runtimeConnectionAttemptId: string
+    }) => Promise<boolean>
+    markConnectionReady: (input: {
+        sessionSandboxId: string
+        generation: number
+        runtimeConnectionId: string
+        lastReceivedCommandSeq: number
+        agentRunSnapshot: SupervisorAgentRunSnapshot
+    }) => Promise<boolean>
+    markConnectionReconnecting: (input: { sessionSandboxId: string, generation: number }) => Promise<void>
+    releaseConnection: (input: { sessionSandboxId: string, generation: number, reason: string }) => Promise<void>
+    markConnectionRetryExhausted: (input: { sessionSandboxId: string, generation: number, replicaId: string }) => Promise<boolean>
+    projectAppliedCommand: (input: { sessionSandboxId: string, runtimeConnectionId: string, commandSeq: number }) => Promise<void>
+}
+
+export interface CommandSource {
+    initial: (input: { sessionId: string }) => Promise<CommandEnvelope | undefined>
+    next: (input: { sessionId: string, runtimeConnectionId: string, afterSequence: number }) => Promise<CommandEnvelope | undefined>
+}
+
+export type DurableEventResult
+    = | { status: 'recorded' | 'duplicate', highWaterMark: number }
+        | { status: 'gap' | 'stale_owner' }
+
+export interface DurableEventSink {
+    getAckFloor: (input: { sessionSandboxId: string }) => Promise<number>
+    record: (input: SessionSandboxRef & { runtimeConnectionId: string, event: DurableEventEnvelope }) => Promise<DurableEventResult>
+}
+
+export interface EphemeralEventSink {
+    publish: (input: { sessionId: string, occurredAt: string, body: EphemeralEventEnvelope['body'] }) => void
+}
+
+export interface RpcDispatcher {
+    handle: (input: SessionSandboxRef & { request: SupervisorRpcRequestEnvelope, signal: AbortSignal }) => Promise<unknown>
+}
+
+export interface SupervisorConnection {
+    send: (data: string) => void | Promise<void>
+    close: (code?: number, reason?: string) => void
+}
+
+export type SupervisorConnectionOpener = (input: {
+    providerSandboxId: string
+    trafficAccessToken: string
+    sessionId: string
+    sessionSandboxId: string
+    runtimeConnectionAttemptId: string
+    authorityAssertion: string
+    connectionGeneration: number
+    eventAckFloor: number
+    signal: AbortSignal
+    onMessage: (data: string | ArrayBuffer | Uint8Array) => void
+    onClose: (input: { code: number, reason: string }) => void
+}) => Promise<SupervisorConnection>
+
+export type ControlAuthoritySigner = (claims: {
+    providerSandboxId: string
+    sessionId: string
+    sessionSandboxId: string
+    connectionGeneration: number
+    runtimeConnectionAttemptId: string
+    eventAckFloor: number
+}) => Promise<string>
+
+export interface BootstrapProvider {
+    loadBootstrapBody: (input: { sessionId: string }) => Promise<Omit<SessionBootstrapBody, 'sessionId' | 'initialCommand'>>
+    issueEphemeralSecrets: (input: { sessionId: string, eventAckFloor: number, signal: AbortSignal }) => Promise<{
+        gitToken?: { token: string, expiresAt: string }
+        credentials?: EphemeralCredentials
+    }>
+}
+
+export interface SessionFailureSink {
+    markSessionFailed: (input: SessionSandboxRef & { code: string, detail?: string }) => Promise<void>
+    failDeterministicEstablishment: (input: SessionSandboxRef & { generation: number, code: string, detail?: string }) => Promise<void>
+    markEstablishmentExhausted: (input: SessionSandboxRef & { generation: number }) => Promise<void>
+}
+
+export interface RuntimeNotifications {
+    start: (input: {
+        onConnectionDemand: (sessionSandboxId: string) => void
+        onCommandsReady: (sessionId: string) => void
+        onOwnerChanged: (input: { sessionSandboxId: string, generation: number }) => void
+    }) => Promise<void>
+    stop: () => Promise<void>
+}
+
+export interface SandboxConnectionRuntimeTimings {
+    leaseDurationMs: number
+    leaseRenewIntervalMs: number
+    establishmentBudgetMs: number
+    dialTimeoutMs: number
+    retryInitialDelayMs: number
+    retryMaxDelayMs: number
+    candidateLimit: number
+}
+
+export interface BootstrapCommand {
+    commandId: string
+    commandSeq: number
+    body: WireCommandBody
+}

package/src/worker-runtime-client/postgres-notification-listener.ts

@@ -0,0 +1,91 @@
+import type { Client as PostgresClient } from 'pg'
+import type { Logger } from '../logger'
+
+export interface PostgresNotificationListener {
+    start: () => Promise<void>
+    stop: () => Promise<void>
+}
+
+export function createPostgresNotificationListener(input: {
+    connectionString: string
+    channels: readonly string[]
+    onNotification: (input: { channel: string, payload: string | undefined }) => void
+    onDisconnected?: () => void
+    onReady?: () => void
+    logger: Logger
+    reconnectDelayMs?: number
+}): PostgresNotificationListener {
+    const controller = new AbortController()
+    let task: Promise<void> | undefined
+    let client: PostgresClient | undefined
+    const firstReady = Promise.withResolvers<void>()
+    let connectedOnce = false
+    const reconnectDelayMs = input.reconnectDelayMs ?? 1_000
+
+    return {
+        async start() {
+            if (task)
+                return
+            task = run().catch((error) => {
+                if (!connectedOnce)
+                    firstReady.reject(error)
+                throw error
+            })
+            await firstReady.promise
+        },
+        async stop() {
+            if (!controller.signal.aborted)
+                controller.abort(new DOMException('PostgreSQL notification listener stopped.', 'AbortError'))
+            await client?.end().catch(() => undefined)
+            await task?.catch(() => undefined)
+        },
+    }
+
+    async function run(): Promise<void> {
+        const { Client, escapeIdentifier } = await import('pg')
+        while (!controller.signal.aborted) {
+            const candidate = new Client({ connectionString: input.connectionString })
+            client = candidate
+            try {
+                candidate.on('notification', notification => input.onNotification({
+                    channel: notification.channel,
+                    payload: notification.payload,
+                }))
+                candidate.on('error', error => input.logger.warn({ err: error }, 'PostgreSQL notification listener error'))
+                await candidate.connect()
+                for (const channel of input.channels)
+                    await candidate.query(`LISTEN ${escapeIdentifier(channel)}`)
+                connectedOnce = true
+                firstReady.resolve()
+                input.onReady?.()
+                await waitForEnd({ client: candidate, signal: controller.signal })
+            }
+            catch (error) {
+                if (!controller.signal.aborted)
+                    input.logger.warn({ err: error }, 'PostgreSQL notification listener disconnected')
+            }
+            finally {
+                input.onDisconnected?.()
+                if (client === candidate)
+                    client = undefined
+                await candidate.end().catch(() => undefined)
+            }
+            if (!controller.signal.aborted)
+                await Bun.sleep(reconnectDelayMs)
+        }
+        if (!connectedOnce)
+            firstReady.reject(controller.signal.reason ?? new Error('PostgreSQL notification listener stopped before connecting.'))
+    }
+}
+
+async function waitForEnd(input: { client: PostgresClient, signal: AbortSignal }): Promise<void> {
+    await new Promise<void>((resolve) => {
+        const finish = () => {
+            input.client.removeListener('end', finish)
+            input.signal.removeEventListener('abort', finish)
+            resolve()
+        }
+        input.client.once('end', finish)
+        input.signal.addEventListener('abort', finish, { once: true })
+    })
+}

package/src/worker-runtime-client/rpc-dispatcher.ts

@@ -0,0 +1,27 @@
+import type { SupervisorRpcMethod, SupervisorRpcRequestEnvelope } from '../supervisor-protocol/rpc'
+import type { RpcDispatcher, SessionSandboxRef } from './ports'
+
+export type SupervisorRpcHandler = (input: SessionSandboxRef & {
+    request: SupervisorRpcRequestEnvelope
+    signal: AbortSignal
+}) => Promise<unknown>
+
+export function createRpcDispatcher(input: {
+    handlers: ReadonlyMap<SupervisorRpcMethod, SupervisorRpcHandler> | Readonly<Record<string, SupervisorRpcHandler>>
+    requiredMethods: readonly string[]
+}): RpcDispatcher {
+    const handlers = input.handlers instanceof Map
+        ? new Map(input.handlers)
+        : new Map(Object.entries(input.handlers) as Array<[SupervisorRpcMethod, SupervisorRpcHandler]>)
+    const missing = input.requiredMethods.filter(method => !handlers.has(method as SupervisorRpcMethod))
+    if (missing.length > 0)
+        throw new Error(`Missing required supervisor RPC handlers: ${missing.join(', ')}.`)
+    return {
+        async handle(request) {
+            const handler = handlers.get(request.request.method)
+            if (!handler)
+                throw new Error(`Supervisor RPC method is not registered: ${request.request.method}.`)
+            return await handler(request)
+        },
+    }
+}

package/src/worker-runtime-client/rpc-request-manager.ts

@@ -0,0 +1,141 @@
+import type { SupervisorRpcCancelEnvelope, SupervisorRpcRequestEnvelope, SupervisorRpcResponseEnvelope } from '../supervisor-protocol/rpc'
+import type { ConnectionAuthorityStore, RpcDispatcher, SessionSandboxRef, SupervisorConnection } from './ports'
+import { isAppError } from '../errors'
+
+const maximumConcurrentRequests = 32
+
+export class SupervisorRpcRequestManager {
+    private readonly active = new Map<string, AbortController>()
+    private stopped = false
+
+    constructor(private readonly dependencies: {
+        ref: SessionSandboxRef
+        generation: number
+        runtimeConnectionAttemptId: string
+        authorityStore: ConnectionAuthorityStore
+        dispatcher: RpcDispatcher
+        connection: SupervisorConnection
+    }) {}
+
+    handle(request: SupervisorRpcRequestEnvelope): void {
+        if (this.stopped)
+            return
+        if (this.active.has(request.requestId)) {
+            this.dependencies.connection.close(1008, 'Supervisor RPC request ID was reused.')
+            this.stop(new Error('Supervisor RPC request ID was reused.'))
+            return
+        }
+        if (this.active.size >= maximumConcurrentRequests) {
+            this.sendFailure({
+                requestId: request.requestId,
+                code: 'agent-core.rpc-capacity-exceeded',
+                message: 'Supervisor RPC capacity is unavailable.',
+                retryable: true,
+            })
+            return
+        }
+        const controller = new AbortController()
+        const timeout = setTimeout(() => {
+            controller.abort(new DOMException('Supervisor RPC deadline elapsed.', 'TimeoutError'))
+        }, request.deadlineMs)
+        timeout.unref?.()
+        this.active.set(request.requestId, controller)
+        void this.execute({ request, controller }).finally(() => {
+            clearTimeout(timeout)
+            if (this.active.get(request.requestId) === controller)
+                this.active.delete(request.requestId)
+        })
+    }
+
+    cancel(frame: SupervisorRpcCancelEnvelope): void {
+        this.active.get(frame.requestId)?.abort(new DOMException('Supervisor RPC was cancelled.', 'AbortError'))
+    }
+
+    stop(reason: unknown = new DOMException('Supervisor connection closed.', 'AbortError')): void {
+        if (this.stopped)
+            return
+        this.stopped = true
+        for (const controller of this.active.values())
+            controller.abort(reason)
+        this.active.clear()
+    }
+
+    private async execute(input: {
+        request: SupervisorRpcRequestEnvelope
+        controller: AbortController
+    }): Promise<void> {
+        try {
+            const authorized = await this.dependencies.authorityStore.confirmConnectionAuthority({
+                sessionSandboxId: this.dependencies.ref.sessionSandboxId,
+                generation: this.dependencies.generation,
+                runtimeConnectionAttemptId: this.dependencies.runtimeConnectionAttemptId,
+            })
+            if (!authorized)
+                throw new RpcAuthorityLostError()
+            const value = await this.dependencies.dispatcher.handle({
+                ...this.dependencies.ref,
+                request: input.request,
+                signal: input.controller.signal,
+            })
+            if (input.controller.signal.aborted || this.stopped)
+                return
+            await this.dependencies.connection.send(JSON.stringify({
+                kind: 'rpc.response',
+                requestId: input.request.requestId,
+                result: { status: 'ok', value },
+            } satisfies SupervisorRpcResponseEnvelope))
+        }
+        catch (error) {
+            if (input.controller.signal.aborted || this.stopped)
+                return
+            if (error instanceof RpcAuthorityLostError) {
+                this.sendFailure({
+                    requestId: input.request.requestId,
+                    code: 'agent-core.connection-authority-lost',
+                    message: 'Supervisor connection authority is no longer current.',
+                    retryable: true,
+                })
+                return
+            }
+            if (isAppError(error)) {
+                this.sendFailure({
+                    requestId: input.request.requestId,
+                    code: error.code,
+                    message: error.publicMessage,
+                    retryable: error.retryable,
+                })
+                return
+            }
+            this.sendFailure({
+                requestId: input.request.requestId,
+                code: 'agent-core.rpc-failed',
+                message: 'Supervisor RPC failed.',
+                retryable: true,
+            })
+        }
+    }
+
+    private sendFailure(input: {
+        requestId: string
+        code: string
+        message: string
+        retryable: boolean
+    }): void {
+        if (this.stopped)
+            return
+        void Promise.resolve(this.dependencies.connection.send(JSON.stringify({
+            kind: 'rpc.response',
+            requestId: input.requestId,
+            result: {
+                status: 'error',
+                error: { code: input.code, message: input.message, retryable: input.retryable },
+            },
+        } satisfies SupervisorRpcResponseEnvelope))).catch(() => {
+            this.dependencies.connection.close(1011, 'Supervisor RPC response delivery failed.')
+        })
+    }
+}
+
+class RpcAuthorityLostError extends Error {
+    override readonly name = 'RpcAuthorityLostError'
+}