@meistrari/agent-core 0.1.13 → 0.2.0-experimental.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/docs/credential-isolation-loopback-2026-09-14.md +30 -0
- package/package.json +1 -1
- package/src/agents/claude/claude-event-mapper.ts +29 -6
- package/src/agents/claude/claude-run.ts +70 -2
- package/src/agents/codex/codex-run.ts +12 -2
- package/src/protocol/agent-command.ts +2 -0
- package/src/protocol/agent-usage.ts +1 -0
- package/src/provenance.gen.ts +3 -3
- package/src/steps/contract.ts +4 -1
- package/src/steps/projector.ts +17 -1
- package/src/supervisor/agent-provider-factory.ts +60 -5
- package/src/supervisor/bootstrap-binder.ts +3 -2
- package/src/supervisor/index.ts +1 -1
- package/src/supervisor/persistence/supervisor-store.ts +22 -8
- package/src/supervisor/runtime-handler.ts +30 -2
- package/src/supervisor-protocol/bootstrap.ts +4 -1
- package/src/supervisor-protocol/command-body.ts +4 -1
- package/src/supervisor-protocol/envelopes/supervisor-to-control-plane.ts +1 -0
- package/src/testing/in-memory-runtime-control-plane.ts +118 -12
- package/src/testing/index.ts +1 -1
- package/src/worker-runtime-client/command-pump.ts +85 -10
- package/src/worker-runtime-client/connection-attempt.ts +95 -20
- package/src/worker-runtime-client/frame-processor.ts +242 -58
- package/src/worker-runtime-client/index.ts +8 -2
- package/src/worker-runtime-client/ports.ts +34 -2
- package/src/worker-runtime-client/rpc-request-manager.ts +152 -107
- package/src/worker-runtime-client/sandbox-connection-runtime.ts +72 -35
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { Logger } from '../logger'
|
|
2
|
+
import type { SessionBootstrapBody } from '../supervisor-protocol/bootstrap'
|
|
3
|
+
import type { SessionBootstrapEnvelope } from '../supervisor-protocol/envelopes/control-plane-to-supervisor'
|
|
2
4
|
import type { CommandAckEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
|
|
3
5
|
import type {
|
|
4
6
|
BootstrapProvider,
|
|
@@ -16,6 +18,7 @@ import type {
|
|
|
16
18
|
} from './ports'
|
|
17
19
|
import { randomUUID } from 'node:crypto'
|
|
18
20
|
import { wireCommandBodySchema } from '../supervisor-protocol/command-body'
|
|
21
|
+
import { sessionBootstrapEnvelopeSchema } from '../supervisor-protocol/envelopes/control-plane-to-supervisor'
|
|
19
22
|
import { encodeWireFrame } from '../supervisor-protocol/wire-codec'
|
|
20
23
|
import { SessionCommandPump } from './command-pump'
|
|
21
24
|
import { BootstrapRejectedAckError, SupervisorFrameProcessor } from './frame-processor'
|
|
@@ -85,7 +88,7 @@ export class SessionSandboxConnectionAttempt {
|
|
|
85
88
|
const queuedFrames: Array<string | ArrayBuffer | Uint8Array> = []
|
|
86
89
|
const closed = Promise.withResolvers<{ code: number, reason: string }>()
|
|
87
90
|
const authorityAssertion = await authorityAssertionPromise
|
|
88
|
-
const connection = await
|
|
91
|
+
const connection = await openSupervisorConnectionBeforeDeadline(this.dependencies.openSupervisorConnection, {
|
|
89
92
|
providerSandboxId: claim.providerSandboxId,
|
|
90
93
|
trafficAccessToken,
|
|
91
94
|
sessionId: claim.sessionId,
|
|
@@ -97,7 +100,7 @@ export class SessionSandboxConnectionAttempt {
|
|
|
97
100
|
signal: this.controller.signal,
|
|
98
101
|
onMessage: frame => this.processor ? this.processor.receive(frame) : queuedFrames.push(frame),
|
|
99
102
|
onClose: close => closed.resolve(close),
|
|
100
|
-
}
|
|
103
|
+
}, Math.min(this.dependencies.establishmentDeadline, performance.now() + this.dependencies.timings.dialTimeoutMs), () => new ConnectionClosedError({
|
|
101
104
|
ready: false,
|
|
102
105
|
detail: 'Supervisor connection dial timed out.',
|
|
103
106
|
}))
|
|
@@ -136,30 +139,25 @@ export class SessionSandboxConnectionAttempt {
|
|
|
136
139
|
bootstrapBodyPromise,
|
|
137
140
|
secretsPromise,
|
|
138
141
|
])
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
'Session command 1 has an invalid body.'
|
|
146
|
-
|
|
142
|
+
let initialCommandBody: SessionBootstrapBody['initialCommand']
|
|
143
|
+
if (body.commandTransport !== 'wss-v1') {
|
|
144
|
+
if (!initialCommand || initialCommand.commandSeq !== 1)
|
|
145
|
+
throw new DeterministicEstablishmentError('agent-core.initial-command-invalid', 'Session command 1 is missing or invalid.')
|
|
146
|
+
const parsed = wireCommandBodySchema.safeParse(initialCommand.body)
|
|
147
|
+
if (!parsed.success)
|
|
148
|
+
throw new DeterministicEstablishmentError('agent-core.initial-command-invalid', 'Session command 1 has an invalid body.')
|
|
149
|
+
initialCommandBody = { commandId: initialCommand.commandId, commandSeq: 1, body: parsed.data }
|
|
147
150
|
}
|
|
148
|
-
|
|
149
|
-
await connection.send(encodeWireFrame({
|
|
151
|
+
await sendSupervisorBootstrap(connection, {
|
|
150
152
|
kind: 'session.bootstrap',
|
|
151
153
|
connectionGeneration: claim.runtimeConnectionGeneration,
|
|
152
154
|
body: {
|
|
153
155
|
...body,
|
|
154
156
|
sessionId: claim.sessionId,
|
|
155
|
-
initialCommand: {
|
|
156
|
-
commandId: initialCommand.commandId,
|
|
157
|
-
commandSeq: initialCommand.commandSeq,
|
|
158
|
-
body: initialBody,
|
|
159
|
-
},
|
|
157
|
+
...(initialCommandBody ? { initialCommand: initialCommandBody } : {}),
|
|
160
158
|
},
|
|
161
159
|
...secrets,
|
|
162
|
-
})
|
|
160
|
+
}, this.controller.signal)
|
|
163
161
|
|
|
164
162
|
const readiness = await beforeDeadline(Promise.race([
|
|
165
163
|
processor.waitForReadiness({ signal: this.controller.signal }),
|
|
@@ -176,15 +174,24 @@ export class SessionSandboxConnectionAttempt {
|
|
|
176
174
|
}
|
|
177
175
|
throw error
|
|
178
176
|
})
|
|
177
|
+
const wssV1 = body.commandTransport === 'wss-v1'
|
|
178
|
+
if (wssV1 && readiness.sourceId === undefined)
|
|
179
|
+
throw new DeterministicEstablishmentError('agent-core.journal-identity-missing', 'Supervisor did not announce its journal identity.')
|
|
180
|
+
// The store pins the journal identity before any command is dispatched or event persisted;
|
|
181
|
+
// a replaced journal is refused here instead of being resumed from a stale cursor.
|
|
179
182
|
const markedReady = await this.dependencies.authorityStore.markConnectionReady({
|
|
180
183
|
sessionSandboxId: claim.sessionSandboxId,
|
|
181
184
|
generation: claim.runtimeConnectionGeneration,
|
|
182
185
|
runtimeConnectionId: claim.runtimeConnectionId,
|
|
183
186
|
lastReceivedCommandSeq: readiness.lastReceivedCommandSeq,
|
|
184
187
|
agentRunSnapshot: readiness.agentRun,
|
|
188
|
+
...(readiness.sourceId === undefined ? {} : { journalSourceId: readiness.sourceId }),
|
|
185
189
|
})
|
|
186
|
-
if (
|
|
190
|
+
if (markedReady === false)
|
|
187
191
|
throw new ConnectionAuthorityLostError()
|
|
192
|
+
if (markedReady !== true)
|
|
193
|
+
throw new DeterministicEstablishmentError(markedReady.code, markedReady.detail)
|
|
194
|
+
processor.confirmReadiness()
|
|
188
195
|
this.ready = true
|
|
189
196
|
|
|
190
197
|
const pump = new SessionCommandPump({
|
|
@@ -192,11 +199,17 @@ export class SessionSandboxConnectionAttempt {
|
|
|
192
199
|
sessionSandboxId: claim.sessionSandboxId,
|
|
193
200
|
runtimeConnectionId: claim.runtimeConnectionId,
|
|
194
201
|
afterSequence: readiness.lastReceivedCommandSeq,
|
|
202
|
+
individualReceipts: wssV1,
|
|
195
203
|
commands: this.dependencies.commands,
|
|
196
204
|
authorityStore: this.dependencies.authorityStore,
|
|
197
205
|
connection,
|
|
198
206
|
logger: this.dependencies.logger,
|
|
199
207
|
onRejected: async (ack) => {
|
|
208
|
+
if (wssV1) {
|
|
209
|
+
// The pump already persisted the terminal rejection; the session keeps running.
|
|
210
|
+
this.dependencies.logger.warn({ commandSeq: ack.commandSeq, errorCode: ack.errorCode }, 'supervisor rejected a command; it will not be redispatched')
|
|
211
|
+
return
|
|
212
|
+
}
|
|
200
213
|
await this.dependencies.failures.markSessionFailed({
|
|
201
214
|
sessionId: claim.sessionId,
|
|
202
215
|
sessionSandboxId: claim.sessionSandboxId,
|
|
@@ -233,7 +246,10 @@ export class SessionSandboxConnectionAttempt {
|
|
|
233
246
|
}
|
|
234
247
|
|
|
235
248
|
async drain(): Promise<void> {
|
|
236
|
-
await
|
|
249
|
+
await drainSupervisorConnection([
|
|
250
|
+
() => this.processor?.drain(),
|
|
251
|
+
() => this.commandPump?.close(),
|
|
252
|
+
])
|
|
237
253
|
}
|
|
238
254
|
|
|
239
255
|
private async renewLease(): Promise<never> {
|
|
@@ -347,3 +363,62 @@ async function beforeDeadline<T>(
|
|
|
347
363
|
signal.removeEventListener('abort', abort!)
|
|
348
364
|
}
|
|
349
365
|
}
|
|
366
|
+
|
|
367
|
+
/** A cancelled/timed-out dial must never leave a late socket alive. */
|
|
368
|
+
export async function openSupervisorConnectionBeforeDeadline(
|
|
369
|
+
open: SupervisorConnectionOpener,
|
|
370
|
+
input: Parameters<SupervisorConnectionOpener>[0],
|
|
371
|
+
deadline: number,
|
|
372
|
+
timeoutError: () => Error,
|
|
373
|
+
): Promise<SupervisorConnection> {
|
|
374
|
+
input.signal.throwIfAborted()
|
|
375
|
+
const stop = new AbortController()
|
|
376
|
+
const signal = AbortSignal.any([input.signal, stop.signal])
|
|
377
|
+
let opened: SupervisorConnection | undefined
|
|
378
|
+
let retired = false
|
|
379
|
+
const retire = () => {
|
|
380
|
+
if (opened && !retired) {
|
|
381
|
+
retired = true
|
|
382
|
+
opened.close(1001, 'Connection dial cancelled.')
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
const task = Promise.resolve().then(() => {
|
|
386
|
+
signal.throwIfAborted()
|
|
387
|
+
return open({ ...input, signal })
|
|
388
|
+
}).then((connection) => {
|
|
389
|
+
opened = connection
|
|
390
|
+
if (signal.aborted) {
|
|
391
|
+
retire()
|
|
392
|
+
throw signal.reason
|
|
393
|
+
}
|
|
394
|
+
return connection
|
|
395
|
+
})
|
|
396
|
+
try {
|
|
397
|
+
return await beforeDeadline(task, deadline, signal, timeoutError)
|
|
398
|
+
}
|
|
399
|
+
catch (error) {
|
|
400
|
+
stop.abort(error)
|
|
401
|
+
retire()
|
|
402
|
+
// Observe late rejection even if the deadline elapsed before beforeDeadline raced it.
|
|
403
|
+
void task.catch(() => undefined)
|
|
404
|
+
throw error
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Release ownership only after every pending connection task has settled. */
|
|
409
|
+
export async function drainSupervisorConnection(tasks: readonly (() => void | Promise<void>)[]): Promise<void> {
|
|
410
|
+
const results = await Promise.allSettled(tasks.map(task => Promise.resolve().then(task)))
|
|
411
|
+
const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : [])
|
|
412
|
+
if (errors.length === 1)
|
|
413
|
+
throw errors[0]
|
|
414
|
+
if (errors.length > 1)
|
|
415
|
+
throw new AggregateError(errors, 'Supervisor connection drain failed.')
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** Validate the wire envelope and reject a bootstrap whose owner was cancelled while loading it. */
|
|
419
|
+
export async function sendSupervisorBootstrap(connection: SupervisorConnection, envelope: SessionBootstrapEnvelope, signal: AbortSignal): Promise<void> {
|
|
420
|
+
signal.throwIfAborted()
|
|
421
|
+
const encoded = encodeWireFrame(sessionBootstrapEnvelopeSchema.parse(envelope))
|
|
422
|
+
signal.throwIfAborted()
|
|
423
|
+
await connection.send(encoded)
|
|
424
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Logger } from '../logger'
|
|
2
|
-
import type { CommandAckEnvelope, RuntimeStateEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
|
|
2
|
+
import type { CommandAckEnvelope, DurableEventEnvelope, EphemeralEventEnvelope, RuntimeStateEnvelope, SupervisorToControlPlaneEnvelope } from '../supervisor-protocol/envelopes/supervisor-to-control-plane'
|
|
3
3
|
import type {
|
|
4
4
|
ConnectionAuthorityStore,
|
|
5
5
|
DurableEventSink,
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
SessionSandboxRef,
|
|
10
10
|
SupervisorConnection,
|
|
11
11
|
} from './ports'
|
|
12
|
+
import { Buffer } from 'node:buffer'
|
|
12
13
|
import {
|
|
13
14
|
supervisorBootstrapRejectionReceiptCloseCode,
|
|
14
15
|
supervisorBootstrapRejectionReceiptCloseReason,
|
|
@@ -20,7 +21,9 @@ import { SupervisorRpcRequestManager } from './rpc-request-manager'
|
|
|
20
21
|
export class SupervisorFrameProcessor {
|
|
21
22
|
private readonly readiness = Promise.withResolvers<RuntimeStateEnvelope>()
|
|
22
23
|
private readonly rpc: SupervisorRpcRequestManager
|
|
23
|
-
private
|
|
24
|
+
private readonly inbox: ReturnType<typeof createOrderedSupervisorInbox>
|
|
25
|
+
private readonly persistenceGate = Promise.withResolvers<void>()
|
|
26
|
+
private journalSourceId: string | undefined
|
|
24
27
|
private stopped = false
|
|
25
28
|
private supervisorLivenessObserved = false
|
|
26
29
|
private readinessSettled = false
|
|
@@ -39,8 +42,11 @@ export class SupervisorFrameProcessor {
|
|
|
39
42
|
logger: Logger
|
|
40
43
|
onCommandAck: (ack: CommandAckEnvelope) => void
|
|
41
44
|
onFatal: (error: unknown) => void
|
|
45
|
+
maxFrames?: number
|
|
46
|
+
maxBytes?: number
|
|
42
47
|
}) {
|
|
43
48
|
void this.readiness.promise.catch(() => undefined)
|
|
49
|
+
void this.persistenceGate.promise.catch(() => undefined)
|
|
44
50
|
this.rpc = new SupervisorRpcRequestManager({
|
|
45
51
|
ref: dependencies.ref,
|
|
46
52
|
generation: dependencies.generation,
|
|
@@ -49,14 +55,21 @@ export class SupervisorFrameProcessor {
|
|
|
49
55
|
dispatcher: dependencies.rpcDispatcher,
|
|
50
56
|
connection: dependencies.connection,
|
|
51
57
|
})
|
|
58
|
+
this.inbox = createOrderedSupervisorInbox({
|
|
59
|
+
maxFrames: dependencies.maxFrames,
|
|
60
|
+
maxBytes: dependencies.maxBytes,
|
|
61
|
+
handle: (raw, _receivedAt, signal) => this.process(frameText(raw), signal),
|
|
62
|
+
onFailure: (error) => {
|
|
63
|
+
this.stop(error)
|
|
64
|
+
dependencies.onFatal(error)
|
|
65
|
+
},
|
|
66
|
+
})
|
|
52
67
|
}
|
|
53
68
|
|
|
54
69
|
receive(data: string | ArrayBuffer | Uint8Array): void {
|
|
55
70
|
if (this.stopped)
|
|
56
71
|
return
|
|
57
|
-
this.
|
|
58
|
-
this.dependencies.onFatal(error)
|
|
59
|
-
})
|
|
72
|
+
this.inbox.push(data)
|
|
60
73
|
}
|
|
61
74
|
|
|
62
75
|
async waitForReadiness(input: { signal: AbortSignal }): Promise<RuntimeStateEnvelope> {
|
|
@@ -73,6 +86,15 @@ export class SupervisorFrameProcessor {
|
|
|
73
86
|
}
|
|
74
87
|
}
|
|
75
88
|
|
|
89
|
+
/**
|
|
90
|
+
* Durable events replayed right after `runtime.state` wait here until the control plane has
|
|
91
|
+
* accepted this connection (authority and journal identity). Nothing is persisted or ACKed
|
|
92
|
+
* for a readiness the store later refuses.
|
|
93
|
+
*/
|
|
94
|
+
confirmReadiness(): void {
|
|
95
|
+
this.persistenceGate.resolve()
|
|
96
|
+
}
|
|
97
|
+
|
|
76
98
|
takeSupervisorLivenessObservation(): boolean {
|
|
77
99
|
const observed = this.supervisorLivenessObserved
|
|
78
100
|
this.supervisorLivenessObserved = false
|
|
@@ -84,69 +106,81 @@ export class SupervisorFrameProcessor {
|
|
|
84
106
|
return
|
|
85
107
|
this.stopped = true
|
|
86
108
|
this.rpc.stop(reason)
|
|
87
|
-
this.
|
|
109
|
+
void this.inbox.close()
|
|
110
|
+
const rejection = reason ?? new DOMException('Supervisor connection closed.', 'AbortError')
|
|
111
|
+
this.readiness.reject(rejection)
|
|
112
|
+
this.persistenceGate.reject(rejection)
|
|
88
113
|
}
|
|
89
114
|
|
|
90
115
|
async drain(): Promise<void> {
|
|
91
|
-
|
|
116
|
+
try {
|
|
117
|
+
await this.inbox.drain()
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
await this.rpc.drain()
|
|
121
|
+
}
|
|
92
122
|
}
|
|
93
123
|
|
|
94
|
-
private async process(text: string): Promise<void> {
|
|
124
|
+
private async process(text: string, signal: AbortSignal): Promise<void> {
|
|
95
125
|
if (this.stopped)
|
|
96
126
|
return
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (frame.kind === 'runtime.state') {
|
|
102
|
-
if (this.readinessSettled)
|
|
103
|
-
throw new Error('Supervisor emitted more than one bootstrap outcome.')
|
|
104
|
-
this.readinessSettled = true
|
|
105
|
-
this.readiness.resolve(frame)
|
|
106
|
-
return
|
|
107
|
-
}
|
|
108
|
-
if (frame.kind === 'command.ack') {
|
|
109
|
-
if (!this.readinessSettled && frame.status === 'rejected') {
|
|
127
|
+
await routeSupervisorFrame(text, new Date(), signal, {
|
|
128
|
+
runtimeState: (frame) => {
|
|
129
|
+
if (this.readinessSettled)
|
|
130
|
+
throw new Error('Supervisor emitted more than one bootstrap outcome.')
|
|
110
131
|
this.readinessSettled = true
|
|
111
|
-
this.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
this.rpc.
|
|
122
|
-
|
|
132
|
+
this.journalSourceId = frame.sourceId
|
|
133
|
+
this.readiness.resolve(frame)
|
|
134
|
+
},
|
|
135
|
+
commandAck: (frame) => {
|
|
136
|
+
if (!this.readinessSettled && frame.status === 'rejected') {
|
|
137
|
+
this.readinessSettled = true
|
|
138
|
+
this.readiness.reject(new BootstrapRejectedAckError(frame))
|
|
139
|
+
}
|
|
140
|
+
this.dependencies.onCommandAck(frame)
|
|
141
|
+
},
|
|
142
|
+
rpcRequest: frame => this.rpc.handle(frame),
|
|
143
|
+
rpcCancel: frame => this.rpc.cancel(frame),
|
|
144
|
+
ephemeral: (frame) => {
|
|
145
|
+
if (frame.body.type === 'supervisor.heartbeat') {
|
|
146
|
+
this.supervisorLivenessObserved = true
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
if (!isEphemeralWrappedAgentEvent(frame.body))
|
|
150
|
+
throw new Error('Supervisor emitted an invalid ephemeral event body.')
|
|
151
|
+
this.dependencies.ephemeralEvents.publish({
|
|
152
|
+
sessionId: this.dependencies.ref.sessionId,
|
|
153
|
+
occurredAt: frame.occurredAt,
|
|
154
|
+
body: frame.body,
|
|
155
|
+
})
|
|
156
|
+
},
|
|
157
|
+
persistDurable: async (frame, signal) => {
|
|
158
|
+
await this.waitForConfirmedReadiness(signal)
|
|
159
|
+
const result = await this.dependencies.durableEvents.record({
|
|
160
|
+
...this.dependencies.ref,
|
|
161
|
+
runtimeConnectionId: this.dependencies.runtimeConnectionId,
|
|
162
|
+
journalSourceId: this.journalSourceId,
|
|
163
|
+
event: frame,
|
|
164
|
+
})
|
|
165
|
+
if (!('highWaterMark' in result))
|
|
166
|
+
throw new Error(`Durable supervisor event was rejected as ${result.status}.`)
|
|
167
|
+
return result.highWaterMark
|
|
168
|
+
},
|
|
169
|
+
send: async encoded => this.dependencies.connection.send(encoded),
|
|
170
|
+
})
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private async waitForConfirmedReadiness(signal: AbortSignal): Promise<void> {
|
|
174
|
+
signal.throwIfAborted()
|
|
175
|
+
const aborted = Promise.withResolvers<never>()
|
|
176
|
+
const onAbort = () => aborted.reject(signal.reason)
|
|
177
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
178
|
+
try {
|
|
179
|
+
await Promise.race([this.persistenceGate.promise, aborted.promise])
|
|
123
180
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
this.supervisorLivenessObserved = true
|
|
127
|
-
return
|
|
128
|
-
}
|
|
129
|
-
if (!isEphemeralWrappedAgentEvent(frame.body))
|
|
130
|
-
throw new Error('Supervisor emitted an invalid ephemeral event body.')
|
|
131
|
-
this.dependencies.ephemeralEvents.publish({
|
|
132
|
-
sessionId: this.dependencies.ref.sessionId,
|
|
133
|
-
occurredAt: frame.occurredAt,
|
|
134
|
-
body: frame.body,
|
|
135
|
-
})
|
|
136
|
-
return
|
|
181
|
+
finally {
|
|
182
|
+
signal.removeEventListener('abort', onAbort)
|
|
137
183
|
}
|
|
138
|
-
|
|
139
|
-
const result = await this.dependencies.durableEvents.record({
|
|
140
|
-
...this.dependencies.ref,
|
|
141
|
-
runtimeConnectionId: this.dependencies.runtimeConnectionId,
|
|
142
|
-
event: frame,
|
|
143
|
-
})
|
|
144
|
-
if (!('highWaterMark' in result))
|
|
145
|
-
throw new Error(`Durable supervisor event was rejected as ${result.status}.`)
|
|
146
|
-
await this.dependencies.connection.send(encodeWireFrame({
|
|
147
|
-
kind: 'event.ack',
|
|
148
|
-
highWaterMark: result.highWaterMark,
|
|
149
|
-
}))
|
|
150
184
|
}
|
|
151
185
|
|
|
152
186
|
async acknowledgeBootstrapRejection(ack: Extract<CommandAckEnvelope, { status: 'rejected' }>): Promise<void> {
|
|
@@ -176,3 +210,153 @@ function frameText(data: string | ArrayBuffer | Uint8Array): string {
|
|
|
176
210
|
return data
|
|
177
211
|
return new TextDecoder().decode(data)
|
|
178
212
|
}
|
|
213
|
+
|
|
214
|
+
export type SupervisorFrameBytes = string | Uint8Array | ArrayBuffer
|
|
215
|
+
interface QueuedFrame { raw: SupervisorFrameBytes, receivedAt: Date, bytes: number }
|
|
216
|
+
|
|
217
|
+
/** Socket callbacks cannot await SQL. Bound retained frames and stop at the first failure. */
|
|
218
|
+
export function createOrderedSupervisorInbox(options: {
|
|
219
|
+
handle: (raw: SupervisorFrameBytes, receivedAt: Date, signal: AbortSignal) => Promise<void>
|
|
220
|
+
onFailure: (error: unknown) => void
|
|
221
|
+
maxFrames?: number
|
|
222
|
+
maxBytes?: number
|
|
223
|
+
}) {
|
|
224
|
+
const maxFrames = options.maxFrames ?? 128
|
|
225
|
+
const maxBytes = options.maxBytes ?? 4 * 1024 * 1024
|
|
226
|
+
if (!Number.isSafeInteger(maxFrames) || maxFrames < 1 || !Number.isSafeInteger(maxBytes) || maxBytes < 1)
|
|
227
|
+
throw new Error('invalid_supervisor_inbox_limits')
|
|
228
|
+
const stop = new AbortController()
|
|
229
|
+
const queue: QueuedFrame[] = []
|
|
230
|
+
let retainedFrames = 0
|
|
231
|
+
let retainedBytes = 0
|
|
232
|
+
let running: Promise<void> | undefined
|
|
233
|
+
let closed = false
|
|
234
|
+
let failed = false
|
|
235
|
+
let failure: unknown
|
|
236
|
+
|
|
237
|
+
function discard() {
|
|
238
|
+
for (const frame of queue) {
|
|
239
|
+
retainedFrames--
|
|
240
|
+
retainedBytes -= frame.bytes
|
|
241
|
+
}
|
|
242
|
+
queue.length = 0
|
|
243
|
+
}
|
|
244
|
+
function fail(error: unknown) {
|
|
245
|
+
if (closed)
|
|
246
|
+
return
|
|
247
|
+
closed = true
|
|
248
|
+
failed = true
|
|
249
|
+
failure = error
|
|
250
|
+
discard()
|
|
251
|
+
stop.abort(error)
|
|
252
|
+
// The owner closes the socket and stops command dispatch; reconnect uses SQL's cursor.
|
|
253
|
+
try {
|
|
254
|
+
options.onFailure(error)
|
|
255
|
+
}
|
|
256
|
+
catch (callbackError) {
|
|
257
|
+
failure = new AggregateError([error, callbackError], 'supervisor_inbox_failure_callback')
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
async function process() {
|
|
261
|
+
while (!closed && queue.length) {
|
|
262
|
+
const frame = queue.shift()!
|
|
263
|
+
try {
|
|
264
|
+
await options.handle(frame.raw, frame.receivedAt, stop.signal)
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
fail(error)
|
|
268
|
+
}
|
|
269
|
+
finally {
|
|
270
|
+
retainedFrames--
|
|
271
|
+
retainedBytes -= frame.bytes
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
function schedule() {
|
|
276
|
+
running = Promise.resolve().then(process).finally(() => {
|
|
277
|
+
running = undefined
|
|
278
|
+
if (!closed && queue.length)
|
|
279
|
+
schedule()
|
|
280
|
+
})
|
|
281
|
+
}
|
|
282
|
+
return {
|
|
283
|
+
push(raw: SupervisorFrameBytes, receivedAt = new Date()) {
|
|
284
|
+
if (closed)
|
|
285
|
+
return false
|
|
286
|
+
const bytes = typeof raw === 'string' ? Buffer.byteLength(raw) : raw.byteLength
|
|
287
|
+
if (retainedFrames + 1 > maxFrames || retainedBytes + bytes > maxBytes) {
|
|
288
|
+
fail(new Error('supervisor_inbox_overflow'))
|
|
289
|
+
return false
|
|
290
|
+
}
|
|
291
|
+
// Binary buffers may be reused by the transport after the callback returns.
|
|
292
|
+
queue.push({ raw: typeof raw === 'string' ? raw : raw instanceof Uint8Array ? new Uint8Array(raw) : raw.slice(0), receivedAt: new Date(receivedAt), bytes })
|
|
293
|
+
retainedFrames++
|
|
294
|
+
retainedBytes += bytes
|
|
295
|
+
if (!running)
|
|
296
|
+
schedule()
|
|
297
|
+
return true
|
|
298
|
+
},
|
|
299
|
+
async drain() {
|
|
300
|
+
let pending = running
|
|
301
|
+
while (pending) {
|
|
302
|
+
await pending
|
|
303
|
+
pending = running
|
|
304
|
+
}
|
|
305
|
+
if (failed)
|
|
306
|
+
throw failure
|
|
307
|
+
},
|
|
308
|
+
async close() {
|
|
309
|
+
closed = true
|
|
310
|
+
discard()
|
|
311
|
+
stop.abort()
|
|
312
|
+
if (running)
|
|
313
|
+
await running
|
|
314
|
+
},
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Product callbacks own authorization and persistence; routing and ACK ordering stay here. */
|
|
319
|
+
export interface SupervisorFrameHandlers {
|
|
320
|
+
beforeFrame?: (frame: SupervisorToControlPlaneEnvelope) => void | Promise<void>
|
|
321
|
+
invalidFrame?: (detail: string) => Error
|
|
322
|
+
runtimeState: (frame: RuntimeStateEnvelope, signal: AbortSignal) => void | Promise<void>
|
|
323
|
+
commandAck: (frame: CommandAckEnvelope, signal: AbortSignal) => void | Promise<void>
|
|
324
|
+
rpcRequest: (frame: Extract<SupervisorToControlPlaneEnvelope, { kind: 'rpc.request' }>, signal: AbortSignal) => void | Promise<void>
|
|
325
|
+
rpcCancel: (frame: Extract<SupervisorToControlPlaneEnvelope, { kind: 'rpc.cancel' }>) => void | Promise<void>
|
|
326
|
+
ephemeral: (frame: EphemeralEventEnvelope, receivedAt: Date) => void | Promise<void>
|
|
327
|
+
persistDurable: (frame: DurableEventEnvelope, signal: AbortSignal) => Promise<number>
|
|
328
|
+
send: (encoded: string) => Promise<void>
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export async function routeSupervisorFrame(raw: SupervisorFrameBytes, receivedAt: Date, signal: AbortSignal, handlers: SupervisorFrameHandlers): Promise<void> {
|
|
332
|
+
signal.throwIfAborted()
|
|
333
|
+
const decoded = decodeSupervisorFrame(frameText(raw))
|
|
334
|
+
if (decoded.kind !== 'ok')
|
|
335
|
+
throw handlers.invalidFrame?.(decoded.detail) ?? new Error(decoded.detail)
|
|
336
|
+
const frame = decoded.frame
|
|
337
|
+
await handlers.beforeFrame?.(frame)
|
|
338
|
+
signal.throwIfAborted()
|
|
339
|
+
switch (frame.kind) {
|
|
340
|
+
case 'runtime.state':
|
|
341
|
+
await handlers.runtimeState(frame, signal)
|
|
342
|
+
return
|
|
343
|
+
case 'command.ack':
|
|
344
|
+
await handlers.commandAck(frame, signal)
|
|
345
|
+
return
|
|
346
|
+
case 'rpc.request':
|
|
347
|
+
await handlers.rpcRequest(frame, signal)
|
|
348
|
+
return
|
|
349
|
+
case 'rpc.cancel':
|
|
350
|
+
await handlers.rpcCancel(frame)
|
|
351
|
+
return
|
|
352
|
+
case 'event': {
|
|
353
|
+
if (frame.delivery_semantics === 'ephemeral') {
|
|
354
|
+
await handlers.ephemeral(frame, receivedAt)
|
|
355
|
+
return
|
|
356
|
+
}
|
|
357
|
+
const highWaterMark = await handlers.persistDurable(frame, signal)
|
|
358
|
+
signal.throwIfAborted()
|
|
359
|
+
await handlers.send(encodeWireFrame({ kind: 'event.ack', highWaterMark }))
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
@@ -1,12 +1,16 @@
|
|
|
1
|
-
export { SessionCommandPump } from './command-pump'
|
|
1
|
+
export { CommandAckTimeoutError, SessionCommandPump } from './command-pump'
|
|
2
|
+
export { drainSupervisorConnection, openSupervisorConnectionBeforeDeadline, sendSupervisorBootstrap } from './connection-attempt'
|
|
2
3
|
export { ControlAuthorityConfigurationError, createControlAuthoritySigner } from './control-authority-signer'
|
|
3
4
|
export { createE2bSupervisorConnectionOpener, openE2bSupervisorConnection } from './e2b-supervisor-connection'
|
|
5
|
+
export { createOrderedSupervisorInbox, routeSupervisorFrame } from './frame-processor'
|
|
6
|
+
export type { SupervisorFrameBytes, SupervisorFrameHandlers } from './frame-processor'
|
|
4
7
|
export { reconcileConnectionLeases } from './lease-reconciler'
|
|
5
8
|
export type {
|
|
6
9
|
BootstrapProvider,
|
|
7
10
|
CommandSource,
|
|
8
11
|
ConnectionAuthorityStore,
|
|
9
12
|
ConnectionClaim,
|
|
13
|
+
ConnectionReadinessRejection,
|
|
10
14
|
ControlAuthoritySigner,
|
|
11
15
|
DurableEventResult,
|
|
12
16
|
DurableEventSink,
|
|
@@ -23,5 +27,7 @@ export { createPostgresNotificationListener } from './postgres-notification-list
|
|
|
23
27
|
export type { PostgresNotificationListener } from './postgres-notification-listener'
|
|
24
28
|
export { createRpcDispatcher } from './rpc-dispatcher'
|
|
25
29
|
export type { SupervisorRpcHandler } from './rpc-dispatcher'
|
|
26
|
-
export {
|
|
30
|
+
export { SupervisorRpcRequestManager } from './rpc-request-manager'
|
|
31
|
+
export type { PreparedSupervisorRpcOptions, PrepareSupervisorRpc } from './rpc-request-manager'
|
|
32
|
+
export { createSandboxConnectionRuntime, runConnectionAttempts } from './sandbox-connection-runtime'
|
|
27
33
|
export { decryptToken, encryptToken, TokenDecryptionError } from './token-crypto'
|
|
@@ -34,19 +34,50 @@ export interface ConnectionAuthorityStore {
|
|
|
34
34
|
generation: number
|
|
35
35
|
runtimeConnectionAttemptId: string
|
|
36
36
|
}) => Promise<boolean>
|
|
37
|
+
/**
|
|
38
|
+
* `journalSourceId` is the supervisor journal identity announced by `runtime.state`
|
|
39
|
+
* (always present under `commandTransport: 'wss-v1'`). Stores must persist it on the first
|
|
40
|
+
* readiness of a session sandbox and answer a `ConnectionReadinessRejection` when a later
|
|
41
|
+
* readiness announces a different identity: a replaced journal restarts command and event
|
|
42
|
+
* sequences, so resuming from the previous cursor and ACK floor would skip commands and
|
|
43
|
+
* discard events as duplicates. `false` still means the claim lost its authority.
|
|
44
|
+
*/
|
|
37
45
|
markConnectionReady: (input: {
|
|
38
46
|
sessionSandboxId: string
|
|
39
47
|
generation: number
|
|
40
48
|
runtimeConnectionId: string
|
|
41
49
|
lastReceivedCommandSeq: number
|
|
42
50
|
agentRunSnapshot: SupervisorAgentRunSnapshot
|
|
43
|
-
|
|
51
|
+
journalSourceId?: string
|
|
52
|
+
}) => Promise<boolean | ConnectionReadinessRejection>
|
|
44
53
|
markConnectionReconnecting: (input: { sessionSandboxId: string, generation: number }) => Promise<void>
|
|
45
54
|
releaseConnection: (input: { sessionSandboxId: string, generation: number, reason: string }) => Promise<void>
|
|
46
55
|
markConnectionRetryExhausted: (input: { sessionSandboxId: string, generation: number, replicaId: string }) => Promise<boolean>
|
|
47
56
|
}
|
|
48
57
|
|
|
58
|
+
/** A readiness the store refuses deterministically; the session fails instead of retrying. */
|
|
59
|
+
export interface ConnectionReadinessRejection {
|
|
60
|
+
ready: false
|
|
61
|
+
code: string
|
|
62
|
+
detail?: string
|
|
63
|
+
}
|
|
64
|
+
|
|
49
65
|
export interface CommandSource {
|
|
66
|
+
/** wss-v1 selects durable pending work; no high-water cursor implies receipt. */
|
|
67
|
+
pending?: (input: { sessionId: string, runtimeConnectionId: string }) => Promise<CommandEnvelope | undefined>
|
|
68
|
+
received?: (input: { sessionId: string, runtimeConnectionId: string, commandId: string, commandSeq: number }) => Promise<void>
|
|
69
|
+
/**
|
|
70
|
+
* wss-v1 persists a terminal supervisor rejection (invalid body, identity conflict) before the
|
|
71
|
+
* pump selects again. A rejected command must never be offered by `pending` afterwards.
|
|
72
|
+
*/
|
|
73
|
+
rejected?: (input: {
|
|
74
|
+
sessionId: string
|
|
75
|
+
runtimeConnectionId: string
|
|
76
|
+
commandId: string
|
|
77
|
+
commandSeq: number
|
|
78
|
+
errorCode: string
|
|
79
|
+
detail?: string
|
|
80
|
+
}) => Promise<void>
|
|
50
81
|
initial: (input: { sessionId: string }) => Promise<CommandEnvelope | undefined>
|
|
51
82
|
next: (input: { sessionId: string, runtimeConnectionId: string, afterSequence: number }) => Promise<CommandEnvelope | undefined>
|
|
52
83
|
}
|
|
@@ -60,7 +91,8 @@ export interface DurableEventSink {
|
|
|
60
91
|
// Atomically persist the event, advance the ACK floor, and project its command outcome.
|
|
61
92
|
// Only supervisor.command.applied marks a command applied; received ACKs never do.
|
|
62
93
|
// Replayed events must not duplicate projections, including after reconnect.
|
|
63
|
-
|
|
94
|
+
// `journalSourceId` identifies the supervisor journal that numbered the event (wss-v1 only).
|
|
95
|
+
record: (input: SessionSandboxRef & { runtimeConnectionId: string, journalSourceId?: string, event: DurableEventEnvelope }) => Promise<DurableEventResult>
|
|
64
96
|
}
|
|
65
97
|
|
|
66
98
|
export interface EphemeralEventSink {
|