@mono-agent/agent-app 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/app.js +4 -2
- package/dist/app.js.map +1 -1
- package/dist/background-runtime.d.ts +32 -0
- package/dist/background-runtime.d.ts.map +1 -1
- package/dist/background-runtime.js +396 -36
- package/dist/background-runtime.js.map +1 -1
- package/dist/background-snapshot-key.d.ts +2 -0
- package/dist/background-snapshot-key.d.ts.map +1 -1
- package/dist/background-snapshot-key.js +32 -1
- package/dist/background-snapshot-key.js.map +1 -1
- package/dist/channels.d.ts.map +1 -1
- package/dist/channels.js +41 -12
- package/dist/channels.js.map +1 -1
- package/dist/cli.d.ts +8 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +61 -0
- package/dist/cli.js.map +1 -1
- package/dist/configured-agent.d.ts.map +1 -1
- package/dist/configured-agent.js +16 -6
- package/dist/configured-agent.js.map +1 -1
- package/dist/continuation-service.d.ts.map +1 -1
- package/dist/continuation-service.js +453 -32
- package/dist/continuation-service.js.map +1 -1
- package/dist/continuation-store.d.ts +41 -2
- package/dist/continuation-store.d.ts.map +1 -1
- package/dist/continuation-store.js +643 -33
- package/dist/continuation-store.js.map +1 -1
- package/dist/continuations.d.ts +29 -4
- package/dist/continuations.d.ts.map +1 -1
- package/dist/continuations.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +99 -10
- package/dist/doctor.js.map +1 -1
- package/dist/launchd.d.ts +2 -0
- package/dist/launchd.d.ts.map +1 -1
- package/dist/launchd.js +46 -17
- package/dist/launchd.js.map +1 -1
- package/dist/managed-runtime-packages.js +1 -1
- package/dist/managed-runtime-packages.js.map +1 -1
- package/dist/memory-command.d.ts +6 -0
- package/dist/memory-command.d.ts.map +1 -1
- package/dist/memory-command.js +363 -3
- package/dist/memory-command.js.map +1 -1
- package/dist/notify-runtime.d.ts +26 -0
- package/dist/notify-runtime.d.ts.map +1 -0
- package/dist/notify-runtime.js +27 -0
- package/dist/notify-runtime.js.map +1 -0
- package/dist/notify-tool.d.ts +51 -0
- package/dist/notify-tool.d.ts.map +1 -0
- package/dist/notify-tool.js +182 -0
- package/dist/notify-tool.js.map +1 -0
- package/dist/recipes/base.d.ts +15 -0
- package/dist/recipes/base.d.ts.map +1 -0
- package/dist/recipes/base.js +51 -0
- package/dist/recipes/base.js.map +1 -0
- package/dist/recipes/catalog.d.ts +4 -0
- package/dist/recipes/catalog.d.ts.map +1 -0
- package/dist/recipes/catalog.js +525 -0
- package/dist/recipes/catalog.js.map +1 -0
- package/dist/recipes/index.d.ts +11 -0
- package/dist/recipes/index.d.ts.map +1 -0
- package/dist/recipes/index.js +14 -0
- package/dist/recipes/index.js.map +1 -0
- package/dist/recipes/types.d.ts +70 -0
- package/dist/recipes/types.d.ts.map +1 -0
- package/dist/recipes/types.js +15 -0
- package/dist/recipes/types.js.map +1 -0
- package/dist/setup.d.ts +29 -0
- package/dist/setup.d.ts.map +1 -0
- package/dist/setup.js +97 -0
- package/dist/setup.js.map +1 -0
- package/package.json +15 -15
- package/skills/mono-agent-composer/references/feature-coverage.md +2 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
2
2
|
import { createServer } from "node:http";
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
|
+
import { assertAgentContinuationOriginContext, } from "@mono-agent/agent-contracts";
|
|
4
5
|
import { acquireContinuationStoreLock, loadOrCreateContinuationSecret, openContinuationStore, } from "./continuation-store.js";
|
|
5
6
|
import { CONTINUATION_CLAIM_TOKEN_ENV, CONTINUATION_CLAIM_TOKEN_HEADER, CONTINUATION_CLAIM_URL_ENV, CONTINUATION_CLAIM_URL_HEADER, CONTINUATION_FINGERPRINT_ENV, CONTINUATION_FINGERPRINT_HEADER, CONTINUATION_MODE_ENV, CONTINUATION_MODE_HEADER, CONTINUATION_STATES, DEFAULT_CONTINUATION_LIMITS, TERMINAL_CONTINUATION_STATES, canonicalContinuationJson, continuationDigest, continuationTokenMatches, normalizeContinuationReplyTarget, } from "./continuations.js";
|
|
6
7
|
const DEFAULT_MAX_RESULT_BYTES = 256 * 1024;
|
|
@@ -12,6 +13,7 @@ const MAX_OPERATOR_PAGE_SIZE = 500;
|
|
|
12
13
|
const MAX_CLAIM_BODY_BYTES = 16 * 1024;
|
|
13
14
|
const MAX_TASK_KEY_CHARS = 256;
|
|
14
15
|
const MAX_TEXT_CHARS = 200_000;
|
|
16
|
+
const ORIGIN_CONTEXT_UNAVAILABLE_TEXT = "The background task finished, but I could not safely restore the original conversation context. Please ask me to check the result again.";
|
|
15
17
|
/**
|
|
16
18
|
* Trusted host signal for the narrow race where readiness disappears after
|
|
17
19
|
* preflight but before the model call begins. It is safe to requeue because the
|
|
@@ -84,6 +86,7 @@ class ContinuationService {
|
|
|
84
86
|
instanceId = randomUUID();
|
|
85
87
|
operatorToken;
|
|
86
88
|
claimBindings = new Map();
|
|
89
|
+
activeClaimBindings = new Set();
|
|
87
90
|
namedRoutes;
|
|
88
91
|
detachedServiceTokenHashes;
|
|
89
92
|
maxResultBytes;
|
|
@@ -96,12 +99,17 @@ class ContinuationService {
|
|
|
96
99
|
logger;
|
|
97
100
|
lifecycleAbort = new AbortController();
|
|
98
101
|
inFlight = new Map();
|
|
102
|
+
activeHttpRequests = new Set();
|
|
103
|
+
activeHandleOperations = 0;
|
|
104
|
+
handleDrainPromise;
|
|
105
|
+
resolveHandleDrain;
|
|
99
106
|
resolvingUnknown = new Set();
|
|
100
107
|
dispatchTail = Promise.resolve();
|
|
101
108
|
server;
|
|
102
109
|
baseUrl = "";
|
|
103
110
|
worker;
|
|
104
111
|
stopped = false;
|
|
112
|
+
stopPromise;
|
|
105
113
|
constructor(store, secret, options, releaseStoreLock) {
|
|
106
114
|
this.store = store;
|
|
107
115
|
this.secret = secret;
|
|
@@ -121,9 +129,11 @@ class ContinuationService {
|
|
|
121
129
|
}
|
|
122
130
|
async start(host, port) {
|
|
123
131
|
const server = createServer((request, response) => {
|
|
124
|
-
|
|
132
|
+
const operation = this.handleRequest(request, response).catch((error) => {
|
|
125
133
|
this.handleRequestError(response, error);
|
|
126
134
|
});
|
|
135
|
+
this.activeHttpRequests.add(operation);
|
|
136
|
+
void operation.finally(() => { this.activeHttpRequests.delete(operation); });
|
|
127
137
|
});
|
|
128
138
|
this.server = server;
|
|
129
139
|
await new Promise((resolve, reject) => {
|
|
@@ -150,21 +160,21 @@ class ContinuationService {
|
|
|
150
160
|
runId: input.runId,
|
|
151
161
|
originConversationId: input.conversationId,
|
|
152
162
|
...(input.replyTo === undefined ? {} : { replyToConversationId: input.replyTo.conversationId }),
|
|
153
|
-
|
|
163
|
+
historyBoundary: input.historyBoundary ?? input.runId,
|
|
154
164
|
mode: "reply",
|
|
155
165
|
}),
|
|
156
166
|
issueRunClaimCapability: (input) => this.issueRunClaimCapability(input),
|
|
157
|
-
status: async (id) => statusOf(await this.store.get(id)),
|
|
158
|
-
list: async () => (await this.store.list()).map(statusOfRequired),
|
|
159
|
-
health: async () => await this.health(),
|
|
160
|
-
processDue: async (limit) => await this.processDue(limit),
|
|
161
|
-
retry: async (id, retryOptions) => await this.retry(id, retryOptions),
|
|
162
|
-
cancel: async (id) => await this.cancel(id),
|
|
163
|
-
resolveUnknown: async (id, outcome) => await this.resolveUnknown(id, outcome),
|
|
164
|
-
capturedText: async (id) => {
|
|
167
|
+
status: async (id) => await this.runHandleOperation(async () => statusOf(await this.store.get(id))),
|
|
168
|
+
list: async () => await this.runHandleOperation(async () => (await this.store.list()).map(statusOfRequired)),
|
|
169
|
+
health: async () => await this.runHandleOperation(async () => await this.health()),
|
|
170
|
+
processDue: async (limit) => await this.runHandleOperation(async () => await this.processDue(limit)),
|
|
171
|
+
retry: async (id, retryOptions) => await this.runHandleOperation(async () => await this.retry(id, retryOptions)),
|
|
172
|
+
cancel: async (id) => await this.runHandleOperation(async () => await this.cancel(id)),
|
|
173
|
+
resolveUnknown: async (id, outcome) => await this.runHandleOperation(async () => await this.resolveUnknown(id, outcome)),
|
|
174
|
+
capturedText: async (id) => await this.runHandleOperation(async () => {
|
|
165
175
|
const record = await this.store.get(id);
|
|
166
176
|
return record?.mode === "capture" && record.state === "delivered" ? record.synthesizedText : undefined;
|
|
167
|
-
},
|
|
177
|
+
}),
|
|
168
178
|
stop: async () => await this.stop(),
|
|
169
179
|
};
|
|
170
180
|
}
|
|
@@ -172,6 +182,10 @@ class ContinuationService {
|
|
|
172
182
|
if (this.stopped)
|
|
173
183
|
throw new Error("Continuation service is stopped.");
|
|
174
184
|
const mode = input.mode ?? "reply";
|
|
185
|
+
// Request-bound capabilities are always interactive and therefore always
|
|
186
|
+
// pin an origin boundary. Context-free detached work has a separate,
|
|
187
|
+
// host-authenticated named-route endpoint.
|
|
188
|
+
const historyBoundary = input.historyBoundary ?? input.runId;
|
|
175
189
|
const replyToConversationId = input.replyToConversationId
|
|
176
190
|
?? (mode === "reply" || mode === "notify_if_actionable"
|
|
177
191
|
? normalizeContinuationReplyTarget(input.originConversationId)
|
|
@@ -185,19 +199,24 @@ class ContinuationService {
|
|
|
185
199
|
input.runId,
|
|
186
200
|
input.originConversationId,
|
|
187
201
|
replyToConversationId ?? "",
|
|
188
|
-
|
|
202
|
+
historyBoundary,
|
|
189
203
|
mode,
|
|
190
204
|
].join("\0"));
|
|
191
205
|
const token = randomBytes(24).toString("base64url");
|
|
192
|
-
|
|
206
|
+
const binding = {
|
|
193
207
|
serverName: input.serverName,
|
|
194
208
|
originRunId: input.runId,
|
|
195
209
|
originConversationId: input.originConversationId,
|
|
196
210
|
...(replyToConversationId === undefined ? {} : { replyToConversationId }),
|
|
197
|
-
|
|
211
|
+
historyBoundary,
|
|
198
212
|
mode,
|
|
199
213
|
fingerprint,
|
|
200
|
-
|
|
214
|
+
closed: false,
|
|
215
|
+
settled: false,
|
|
216
|
+
inFlightOperations: 0,
|
|
217
|
+
};
|
|
218
|
+
this.claimBindings.set(token, binding);
|
|
219
|
+
this.activeClaimBindings.add(binding);
|
|
201
220
|
const url = `${this.baseUrl}/v1/continuations/claim`;
|
|
202
221
|
let released = false;
|
|
203
222
|
return {
|
|
@@ -217,20 +236,222 @@ class ContinuationService {
|
|
|
217
236
|
[CONTINUATION_FINGERPRINT_ENV]: fingerprint,
|
|
218
237
|
[CONTINUATION_MODE_ENV]: mode,
|
|
219
238
|
}),
|
|
220
|
-
|
|
239
|
+
requiresOriginContext: async () => await this.requiresOriginContext(binding),
|
|
240
|
+
finalizeOriginContext: async (snapshot) => await this.finalizeOriginContext(binding, snapshot),
|
|
241
|
+
activateOriginContext: async () => await this.activateOriginContext(binding),
|
|
242
|
+
abandonOriginContext: async () => await this.abandonOriginContext(binding),
|
|
243
|
+
release: async () => {
|
|
221
244
|
if (released)
|
|
222
245
|
return;
|
|
223
246
|
released = true;
|
|
224
|
-
this.
|
|
247
|
+
await this.closeClaimBinding(token, binding);
|
|
225
248
|
},
|
|
226
249
|
};
|
|
227
250
|
}
|
|
251
|
+
async runHandleOperation(operation) {
|
|
252
|
+
if (this.stopped)
|
|
253
|
+
throw new Error("Continuation service is stopped.");
|
|
254
|
+
this.activeHandleOperations += 1;
|
|
255
|
+
try {
|
|
256
|
+
return await operation();
|
|
257
|
+
}
|
|
258
|
+
finally {
|
|
259
|
+
this.activeHandleOperations -= 1;
|
|
260
|
+
if (this.stopped && this.activeHandleOperations === 0) {
|
|
261
|
+
this.resolveHandleDrain?.();
|
|
262
|
+
this.resolveHandleDrain = undefined;
|
|
263
|
+
this.handleDrainPromise = undefined;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
async drainHandleOperations() {
|
|
268
|
+
if (this.activeHandleOperations === 0)
|
|
269
|
+
return;
|
|
270
|
+
this.handleDrainPromise ??= new Promise((resolve) => { this.resolveHandleDrain = resolve; });
|
|
271
|
+
await this.handleDrainPromise;
|
|
272
|
+
}
|
|
273
|
+
async closeClaimBinding(token, binding) {
|
|
274
|
+
binding.closed = true;
|
|
275
|
+
if (this.claimBindings.get(token) === binding)
|
|
276
|
+
this.claimBindings.delete(token);
|
|
277
|
+
if (binding.inFlightOperations === 0)
|
|
278
|
+
return;
|
|
279
|
+
binding.drainPromise ??= new Promise((resolve) => { binding.resolveDrain = resolve; });
|
|
280
|
+
await binding.drainPromise;
|
|
281
|
+
}
|
|
282
|
+
beginClaim(binding) {
|
|
283
|
+
if (binding.closed) {
|
|
284
|
+
throw new ContinuationProtocolError(401, "invalid_claim_capability", "Invalid or expired claim capability.");
|
|
285
|
+
}
|
|
286
|
+
binding.inFlightOperations += 1;
|
|
287
|
+
let finished = false;
|
|
288
|
+
return () => {
|
|
289
|
+
if (finished)
|
|
290
|
+
return;
|
|
291
|
+
finished = true;
|
|
292
|
+
binding.inFlightOperations -= 1;
|
|
293
|
+
if (binding.closed && binding.inFlightOperations === 0) {
|
|
294
|
+
binding.resolveDrain?.();
|
|
295
|
+
delete binding.resolveDrain;
|
|
296
|
+
delete binding.drainPromise;
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
async requiresOriginContext(binding) {
|
|
301
|
+
if (this.stopped || binding.settled)
|
|
302
|
+
return false;
|
|
303
|
+
if (!binding.closed) {
|
|
304
|
+
throw new Error("Continuation claims must be revoked before origin settlement is inspected.");
|
|
305
|
+
}
|
|
306
|
+
if (binding.inFlightOperations > 0) {
|
|
307
|
+
binding.drainPromise ??= new Promise((resolve) => { binding.resolveDrain = resolve; });
|
|
308
|
+
await binding.drainPromise;
|
|
309
|
+
}
|
|
310
|
+
const finishOperation = this.beginOriginSettlementOperation(binding);
|
|
311
|
+
if (finishOperation === undefined)
|
|
312
|
+
return false;
|
|
313
|
+
try {
|
|
314
|
+
if (binding.historyBoundary === undefined) {
|
|
315
|
+
this.settleClaimBinding(binding);
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
const required = (await this.store.list()).some((record) => record.claimFingerprint === binding.fingerprint
|
|
319
|
+
&& !TERMINAL_CONTINUATION_STATES.has(record.state));
|
|
320
|
+
if (!required)
|
|
321
|
+
this.settleClaimBinding(binding);
|
|
322
|
+
return required;
|
|
323
|
+
}
|
|
324
|
+
finally {
|
|
325
|
+
finishOperation();
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
beginOriginSettlementOperation(binding) {
|
|
329
|
+
if (this.stopped || binding.settled)
|
|
330
|
+
return undefined;
|
|
331
|
+
if (!binding.closed) {
|
|
332
|
+
throw new Error("Continuation claims must be revoked before origin settlement begins.");
|
|
333
|
+
}
|
|
334
|
+
if (binding.inFlightOperations !== 0) {
|
|
335
|
+
throw new Error("Continuation capability release has not finished draining admitted work.");
|
|
336
|
+
}
|
|
337
|
+
binding.inFlightOperations += 1;
|
|
338
|
+
let finished = false;
|
|
339
|
+
return () => {
|
|
340
|
+
if (finished)
|
|
341
|
+
return;
|
|
342
|
+
finished = true;
|
|
343
|
+
binding.inFlightOperations -= 1;
|
|
344
|
+
if (binding.closed && binding.inFlightOperations === 0) {
|
|
345
|
+
binding.resolveDrain?.();
|
|
346
|
+
delete binding.resolveDrain;
|
|
347
|
+
delete binding.drainPromise;
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
settleClaimBinding(binding) {
|
|
352
|
+
binding.settled = true;
|
|
353
|
+
this.activeClaimBindings.delete(binding);
|
|
354
|
+
}
|
|
355
|
+
async finalizeOriginContext(binding, snapshot) {
|
|
356
|
+
const finishOperation = this.beginOriginSettlementOperation(binding);
|
|
357
|
+
if (finishOperation === undefined)
|
|
358
|
+
return;
|
|
359
|
+
try {
|
|
360
|
+
if (binding.historyBoundary === undefined)
|
|
361
|
+
return;
|
|
362
|
+
assertAgentContinuationOriginContext(snapshot);
|
|
363
|
+
if (snapshot.conversationId !== binding.originConversationId
|
|
364
|
+
|| snapshot.originRunId !== binding.originRunId
|
|
365
|
+
|| snapshot.historyBoundary !== binding.historyBoundary) {
|
|
366
|
+
throw new Error("Continuation origin context does not match its immutable claim binding.");
|
|
367
|
+
}
|
|
368
|
+
const matching = (await this.store.list()).filter((record) => record.claimFingerprint === binding.fingerprint
|
|
369
|
+
&& !TERMINAL_CONTINUATION_STATES.has(record.state));
|
|
370
|
+
if (matching.length === 0)
|
|
371
|
+
return;
|
|
372
|
+
const pin = await this.store.stageOriginContext(snapshot);
|
|
373
|
+
try {
|
|
374
|
+
await this.store.mutate((records) => {
|
|
375
|
+
const candidates = [...records.values()].filter((record) => record.claimFingerprint === binding.fingerprint
|
|
376
|
+
&& !TERMINAL_CONTINUATION_STATES.has(record.state));
|
|
377
|
+
for (const record of candidates) {
|
|
378
|
+
if (record.originContextState === "pinned"
|
|
379
|
+
|| (record.originContextState === "pending" && record.originContextRef !== undefined)) {
|
|
380
|
+
if (record.originContextDigest !== pin.reference.digest
|
|
381
|
+
|| record.originContextRef?.digest !== pin.reference.digest) {
|
|
382
|
+
throw new Error("Continuation origin context conflicts with an existing pinned snapshot.");
|
|
383
|
+
}
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (record.originContextState !== "pending") {
|
|
387
|
+
throw new Error(`Continuation origin context cannot be finalized from ${record.originContextState}.`);
|
|
388
|
+
}
|
|
389
|
+
record.originContextRef = pin.reference;
|
|
390
|
+
record.originContextDigest = pin.reference.digest;
|
|
391
|
+
record.originContextMessageCount = pin.reference.messageCount;
|
|
392
|
+
record.originContextFingerprint = continuationDigest(`mono-agent-origin-context-binding-v2\0${record.claimFingerprint}\0${pin.reference.digest}`);
|
|
393
|
+
record.originContextBindingMac = originContextBindingMac(this.secret, record, pin.reference);
|
|
394
|
+
record.updatedAt = this.now().toISOString();
|
|
395
|
+
}
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
finally {
|
|
399
|
+
await pin.release();
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
finally {
|
|
403
|
+
finishOperation();
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
async activateOriginContext(binding) {
|
|
407
|
+
const finishOperation = this.beginOriginSettlementOperation(binding);
|
|
408
|
+
if (finishOperation === undefined)
|
|
409
|
+
return;
|
|
410
|
+
try {
|
|
411
|
+
await this.store.activateOriginContextGroup({
|
|
412
|
+
claimFingerprint: binding.fingerprint,
|
|
413
|
+
activatedAt: this.now().toISOString(),
|
|
414
|
+
});
|
|
415
|
+
this.settleClaimBinding(binding);
|
|
416
|
+
}
|
|
417
|
+
finally {
|
|
418
|
+
finishOperation();
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
async abandonOriginContext(binding) {
|
|
422
|
+
const finishOperation = this.beginOriginSettlementOperation(binding);
|
|
423
|
+
if (finishOperation === undefined)
|
|
424
|
+
return;
|
|
425
|
+
const at = this.now().toISOString();
|
|
426
|
+
try {
|
|
427
|
+
await this.store.mutate((records) => {
|
|
428
|
+
for (const record of records.values()) {
|
|
429
|
+
if (record.claimFingerprint !== binding.fingerprint
|
|
430
|
+
|| record.originContextState !== "pending")
|
|
431
|
+
continue;
|
|
432
|
+
record.originContextState = "abandoned";
|
|
433
|
+
delete record.originContextRef;
|
|
434
|
+
record.updatedAt = at;
|
|
435
|
+
record.lastError = errorRecord("origin_context_unavailable", "The origin run did not commit its pinned continuation context.", at);
|
|
436
|
+
delete record.nextAttemptAt;
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
this.settleClaimBinding(binding);
|
|
440
|
+
if (this.autoProcess)
|
|
441
|
+
void this.processDue().catch(() => undefined);
|
|
442
|
+
}
|
|
443
|
+
finally {
|
|
444
|
+
finishOperation();
|
|
445
|
+
}
|
|
446
|
+
}
|
|
228
447
|
async recover(startup = false) {
|
|
229
448
|
const now = this.now().getTime();
|
|
230
449
|
const current = await this.store.list();
|
|
231
450
|
const needsRecovery = current.some((record) => !TERMINAL_CONTINUATION_STATES.has(record.state)
|
|
232
451
|
&& (startup || !this.inFlight.has(record.continuationId))
|
|
233
|
-
&& ((startup && record.
|
|
452
|
+
&& ((startup && record.originContextState === "pending")
|
|
453
|
+
||
|
|
454
|
+
(startup && record.leaseOwner !== undefined)
|
|
234
455
|
|| Date.parse(record.deadline) <= now
|
|
235
456
|
|| (record.leaseUntil !== undefined && Date.parse(record.leaseUntil) <= now)));
|
|
236
457
|
if (!needsRecovery)
|
|
@@ -239,6 +460,13 @@ class ContinuationService {
|
|
|
239
460
|
for (const record of records.values()) {
|
|
240
461
|
if (TERMINAL_CONTINUATION_STATES.has(record.state))
|
|
241
462
|
continue;
|
|
463
|
+
if (startup && record.originContextState === "pending") {
|
|
464
|
+
record.originContextState = "abandoned";
|
|
465
|
+
delete record.originContextRef;
|
|
466
|
+
record.updatedAt = this.now().toISOString();
|
|
467
|
+
record.lastError = errorRecord("origin_context_unavailable", "The service restarted before the origin context was pinned.", record.updatedAt);
|
|
468
|
+
clearLease(record);
|
|
469
|
+
}
|
|
242
470
|
// A configured operation timeout may intentionally exceed the lease.
|
|
243
471
|
// The process-lifetime store lock prevents another owner, while this
|
|
244
472
|
// map proves the current owner is still supervising the attempt.
|
|
@@ -334,13 +562,34 @@ class ContinuationService {
|
|
|
334
562
|
return false;
|
|
335
563
|
this.throwIfStopping();
|
|
336
564
|
let record = leased;
|
|
565
|
+
// Even modes that need no model or channel text must not settle before the
|
|
566
|
+
// originating run publishes (or abandons) its context boundary. Otherwise
|
|
567
|
+
// a fast silent result can become terminal and poison origin settlement.
|
|
568
|
+
if (record.originContextState === "pending") {
|
|
569
|
+
await this.deferOriginContext(record.continuationId);
|
|
570
|
+
return true;
|
|
571
|
+
}
|
|
337
572
|
if (record.synthesizedText === undefined && record.mode !== "silent") {
|
|
338
|
-
const
|
|
339
|
-
if (
|
|
340
|
-
await this.
|
|
573
|
+
const originContext = await this.resolveOriginContext(record);
|
|
574
|
+
if (originContext.kind === "pending") {
|
|
575
|
+
await this.deferOriginContext(record.continuationId);
|
|
576
|
+
return true;
|
|
577
|
+
}
|
|
578
|
+
if (originContext.kind === "invalid") {
|
|
579
|
+
await this.deadLetter(record.continuationId, originContext.code, "Pinned continuation binding failed integrity verification; native delivery was blocked.");
|
|
341
580
|
return true;
|
|
342
581
|
}
|
|
343
|
-
|
|
582
|
+
if (originContext.kind === "unavailable") {
|
|
583
|
+
record = await this.prepareOriginContextFallback(record.continuationId);
|
|
584
|
+
}
|
|
585
|
+
else {
|
|
586
|
+
const available = await this.synthesisAvailable(record, originContext);
|
|
587
|
+
if (!available.ready) {
|
|
588
|
+
await this.deferSynthesis(record.continuationId, available.code, available.reason, available.retryAfterMs);
|
|
589
|
+
return true;
|
|
590
|
+
}
|
|
591
|
+
record = await this.synthesize(record, originContext);
|
|
592
|
+
}
|
|
344
593
|
if (record.state !== "ready_to_deliver")
|
|
345
594
|
return true;
|
|
346
595
|
}
|
|
@@ -370,11 +619,11 @@ class ContinuationService {
|
|
|
370
619
|
await this.deliver(record);
|
|
371
620
|
return true;
|
|
372
621
|
}
|
|
373
|
-
async synthesisAvailable(record) {
|
|
622
|
+
async synthesisAvailable(record, originContext) {
|
|
374
623
|
if (this.options.synthesisPreflight === undefined)
|
|
375
624
|
return { ready: true };
|
|
376
625
|
try {
|
|
377
|
-
return await this.runBoundedOperation("synthesis", Math.min(this.limits.synthesisTimeoutMs, 30_000), async () => await this.options.synthesisPreflight?.(synthesisInput(record)) ?? { ready: true });
|
|
626
|
+
return await this.runBoundedOperation("synthesis", Math.min(this.limits.synthesisTimeoutMs, 30_000), async () => await this.options.synthesisPreflight?.(synthesisInput(record, originContext)) ?? { ready: true });
|
|
378
627
|
}
|
|
379
628
|
catch (error) {
|
|
380
629
|
if (error instanceof ContinuationServiceStoppingError)
|
|
@@ -388,6 +637,79 @@ class ContinuationService {
|
|
|
388
637
|
};
|
|
389
638
|
}
|
|
390
639
|
}
|
|
640
|
+
async resolveOriginContext(record) {
|
|
641
|
+
if (record.originContextState === "detached_latest") {
|
|
642
|
+
return { kind: "ready", policy: "detached_latest" };
|
|
643
|
+
}
|
|
644
|
+
if (record.originContextState === "pending")
|
|
645
|
+
return { kind: "pending" };
|
|
646
|
+
if (record.originContextState !== "pinned" || record.originContextRef === undefined) {
|
|
647
|
+
return { kind: "unavailable" };
|
|
648
|
+
}
|
|
649
|
+
const expectedFingerprint = continuationDigest(`mono-agent-origin-context-binding-v2\0${record.claimFingerprint}\0${record.originContextRef.digest}`);
|
|
650
|
+
const expectedMac = originContextBindingMac(this.secret, record, record.originContextRef);
|
|
651
|
+
if (record.originContextDigest !== record.originContextRef.digest
|
|
652
|
+
|| record.originContextFingerprint !== expectedFingerprint
|
|
653
|
+
|| record.originContextBindingMac !== expectedMac) {
|
|
654
|
+
return { kind: "invalid", code: "origin_context_binding_invalid" };
|
|
655
|
+
}
|
|
656
|
+
let snapshot;
|
|
657
|
+
try {
|
|
658
|
+
snapshot = await this.store.loadOriginContext(record.originContextRef);
|
|
659
|
+
}
|
|
660
|
+
catch {
|
|
661
|
+
// Filesystem identity/permission failures are indistinguishable from a
|
|
662
|
+
// corrupt snapshot at this trust boundary. Fail closed into the same
|
|
663
|
+
// deterministic no-model delivery instead of retrying or dead-lettering.
|
|
664
|
+
await this.markOriginContextUnavailable(record.continuationId, "origin_context_unreadable");
|
|
665
|
+
return { kind: "unavailable" };
|
|
666
|
+
}
|
|
667
|
+
if (snapshot === undefined
|
|
668
|
+
|| snapshot.conversationId !== record.originConversationId
|
|
669
|
+
|| snapshot.originRunId !== record.originRunId
|
|
670
|
+
|| snapshot.historyBoundary !== record.historyBoundary) {
|
|
671
|
+
await this.markOriginContextUnavailable(record.continuationId, "origin_context_missing_or_corrupt");
|
|
672
|
+
return { kind: "unavailable" };
|
|
673
|
+
}
|
|
674
|
+
return { kind: "ready", policy: "pinned", snapshot };
|
|
675
|
+
}
|
|
676
|
+
async markOriginContextUnavailable(id, code) {
|
|
677
|
+
await this.store.mutate((records) => {
|
|
678
|
+
const current = requireRecord(records, id);
|
|
679
|
+
requireLease(current, this.instanceId);
|
|
680
|
+
current.originContextState = "abandoned";
|
|
681
|
+
delete current.originContextRef;
|
|
682
|
+
current.updatedAt = this.now().toISOString();
|
|
683
|
+
current.lastError = errorRecord(code, "Pinned origin context is unavailable.", current.updatedAt);
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
async deferOriginContext(id) {
|
|
687
|
+
return await this.store.mutate((records) => {
|
|
688
|
+
const current = requireRecord(records, id);
|
|
689
|
+
requireLease(current, this.instanceId);
|
|
690
|
+
current.synthesisDeferrals += 1;
|
|
691
|
+
const at = this.now();
|
|
692
|
+
current.state = "result_received";
|
|
693
|
+
current.updatedAt = at.toISOString();
|
|
694
|
+
current.nextAttemptAt = new Date(at.getTime() + backoffMs(current.synthesisDeferrals, 1_000, 5 * 60 * 1_000)).toISOString();
|
|
695
|
+
current.lastError = errorRecord("origin_context_pending", "The origin run has not committed its pinned context yet.", current.updatedAt);
|
|
696
|
+
clearLease(current);
|
|
697
|
+
return structuredClone(current);
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
async prepareOriginContextFallback(id) {
|
|
701
|
+
return await this.store.mutate((records) => {
|
|
702
|
+
const current = requireRecord(records, id);
|
|
703
|
+
requireLease(current, this.instanceId);
|
|
704
|
+
current.synthesizedText = ORIGIN_CONTEXT_UNAVAILABLE_TEXT;
|
|
705
|
+
current.completionKind = "origin_context_unavailable";
|
|
706
|
+
current.state = "ready_to_deliver";
|
|
707
|
+
current.updatedAt = this.now().toISOString();
|
|
708
|
+
delete current.nextAttemptAt;
|
|
709
|
+
delete current.synthesisStartedAt;
|
|
710
|
+
return structuredClone(current);
|
|
711
|
+
});
|
|
712
|
+
}
|
|
391
713
|
async deferSynthesis(id, code, reason, retryAfterMs = 1_000, consumedAttempt = false) {
|
|
392
714
|
return await this.store.mutate((records) => {
|
|
393
715
|
const current = requireRecord(records, id);
|
|
@@ -433,7 +755,7 @@ class ContinuationService {
|
|
|
433
755
|
return structuredClone(record);
|
|
434
756
|
});
|
|
435
757
|
}
|
|
436
|
-
async synthesize(record) {
|
|
758
|
+
async synthesize(record, originContext) {
|
|
437
759
|
const startedAt = this.now().toISOString();
|
|
438
760
|
const prepared = await this.store.mutate((records) => {
|
|
439
761
|
this.throwIfStopping();
|
|
@@ -447,7 +769,7 @@ class ContinuationService {
|
|
|
447
769
|
return structuredClone(current);
|
|
448
770
|
});
|
|
449
771
|
try {
|
|
450
|
-
const result = await this.runBoundedOperation("synthesis", this.limits.synthesisTimeoutMs, async () => await this.options.synthesize(synthesisInput(prepared)));
|
|
772
|
+
const result = await this.runBoundedOperation("synthesis", this.limits.synthesisTimeoutMs, async () => await this.options.synthesize(synthesisInput(prepared, originContext)));
|
|
451
773
|
this.throwIfStopping();
|
|
452
774
|
const text = result.text.trim();
|
|
453
775
|
if (text.length === 0 || text.length > MAX_TEXT_CHARS) {
|
|
@@ -457,6 +779,7 @@ class ContinuationService {
|
|
|
457
779
|
const current = requireRecord(records, prepared.continuationId);
|
|
458
780
|
requireLease(current, this.instanceId);
|
|
459
781
|
current.synthesizedText = text;
|
|
782
|
+
current.completionKind = "synthesized";
|
|
460
783
|
if (result.actionable !== undefined)
|
|
461
784
|
current.actionable = result.actionable;
|
|
462
785
|
current.state = "ready_to_deliver";
|
|
@@ -771,7 +1094,16 @@ class ContinuationService {
|
|
|
771
1094
|
if (binding === undefined)
|
|
772
1095
|
throw new ContinuationProtocolError(401, "invalid_claim_capability", "Invalid or expired claim capability.");
|
|
773
1096
|
const body = await readJson(request, MAX_CLAIM_BODY_BYTES);
|
|
774
|
-
|
|
1097
|
+
// Do not count a slow request body as admitted work. Recheck and enter
|
|
1098
|
+
// the drain only after the complete bounded body exists; release() then
|
|
1099
|
+
// either revokes it or waits for its durable mutation to finish.
|
|
1100
|
+
const finishClaim = this.beginClaim(binding);
|
|
1101
|
+
try {
|
|
1102
|
+
sendJson(response, 200, await this.claim(binding, body));
|
|
1103
|
+
}
|
|
1104
|
+
finally {
|
|
1105
|
+
finishClaim();
|
|
1106
|
+
}
|
|
775
1107
|
return;
|
|
776
1108
|
}
|
|
777
1109
|
if (request.method === "POST" && url.pathname === "/v1/continuations/detached/claim") {
|
|
@@ -858,6 +1190,9 @@ class ContinuationService {
|
|
|
858
1190
|
...(route.conversationId === undefined ? {} : { replyToConversationId: route.conversationId }),
|
|
859
1191
|
mode: route.mode,
|
|
860
1192
|
fingerprint,
|
|
1193
|
+
closed: false,
|
|
1194
|
+
settled: true,
|
|
1195
|
+
inFlightOperations: 0,
|
|
861
1196
|
}, claim, routeName);
|
|
862
1197
|
}
|
|
863
1198
|
async claimBound(binding, claim, routeName) {
|
|
@@ -892,6 +1227,7 @@ class ContinuationService {
|
|
|
892
1227
|
originConversationId: binding.originConversationId,
|
|
893
1228
|
...(binding.replyToConversationId === undefined ? {} : { replyToConversationId: binding.replyToConversationId }),
|
|
894
1229
|
...(binding.historyBoundary === undefined ? {} : { historyBoundary: binding.historyBoundary }),
|
|
1230
|
+
originContextState: binding.historyBoundary === undefined ? "detached_latest" : "pending",
|
|
895
1231
|
mode: binding.mode,
|
|
896
1232
|
...(routeName === undefined ? {} : { routeName }),
|
|
897
1233
|
taskKey: claim.taskKey,
|
|
@@ -903,6 +1239,7 @@ class ContinuationService {
|
|
|
903
1239
|
deadline: claim.deadline,
|
|
904
1240
|
state: "claimed",
|
|
905
1241
|
synthesisAttempts: 0,
|
|
1242
|
+
synthesisDeferrals: 0,
|
|
906
1243
|
deliveryAttempts: 0,
|
|
907
1244
|
};
|
|
908
1245
|
records.set(continuationId, next);
|
|
@@ -1019,13 +1356,18 @@ class ContinuationService {
|
|
|
1019
1356
|
}
|
|
1020
1357
|
}
|
|
1021
1358
|
async stop() {
|
|
1022
|
-
|
|
1023
|
-
|
|
1359
|
+
this.stopPromise ??= this.stopOnce();
|
|
1360
|
+
await this.stopPromise;
|
|
1361
|
+
}
|
|
1362
|
+
async stopOnce() {
|
|
1024
1363
|
this.stopped = true;
|
|
1025
1364
|
this.lifecycleAbort.abort();
|
|
1026
1365
|
if (this.worker !== undefined)
|
|
1027
1366
|
clearInterval(this.worker);
|
|
1028
1367
|
this.worker = undefined;
|
|
1368
|
+
const bindings = [...this.activeClaimBindings];
|
|
1369
|
+
for (const binding of bindings)
|
|
1370
|
+
binding.closed = true;
|
|
1029
1371
|
this.claimBindings.clear();
|
|
1030
1372
|
const server = this.server;
|
|
1031
1373
|
this.server = undefined;
|
|
@@ -1033,8 +1375,47 @@ class ContinuationService {
|
|
|
1033
1375
|
if (server !== undefined) {
|
|
1034
1376
|
await closeContinuationServer(server);
|
|
1035
1377
|
}
|
|
1378
|
+
while (this.activeHttpRequests.size > 0) {
|
|
1379
|
+
await Promise.allSettled([...this.activeHttpRequests]);
|
|
1380
|
+
}
|
|
1381
|
+
await this.drainHandleOperations();
|
|
1382
|
+
await Promise.all(bindings.map(async (binding) => {
|
|
1383
|
+
if (binding.inFlightOperations === 0)
|
|
1384
|
+
return;
|
|
1385
|
+
binding.drainPromise ??= new Promise((resolve) => { binding.resolveDrain = resolve; });
|
|
1386
|
+
await binding.drainPromise;
|
|
1387
|
+
}));
|
|
1036
1388
|
await this.dispatchTail.catch(() => undefined);
|
|
1037
1389
|
await Promise.allSettled([...this.inFlight.values()]);
|
|
1390
|
+
const fingerprints = new Set(bindings.map((binding) => binding.fingerprint));
|
|
1391
|
+
if (fingerprints.size > 0) {
|
|
1392
|
+
const at = this.now().toISOString();
|
|
1393
|
+
try {
|
|
1394
|
+
await this.store.mutate((records) => {
|
|
1395
|
+
for (const record of records.values()) {
|
|
1396
|
+
if (!fingerprints.has(record.claimFingerprint) || record.originContextState !== "pending")
|
|
1397
|
+
continue;
|
|
1398
|
+
record.originContextState = "abandoned";
|
|
1399
|
+
delete record.originContextRef;
|
|
1400
|
+
record.updatedAt = at;
|
|
1401
|
+
record.lastError = errorRecord("origin_context_unavailable", "The origin service stopped before its pinned continuation context committed.", at);
|
|
1402
|
+
delete record.nextAttemptAt;
|
|
1403
|
+
clearLease(record);
|
|
1404
|
+
}
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
catch (error) {
|
|
1408
|
+
// A prior durable transaction may already have poisoned this store.
|
|
1409
|
+
// Shutdown must still revoke stale closures and release OS ownership;
|
|
1410
|
+
// restart recovery deterministically abandons any remaining pending
|
|
1411
|
+
// origin records before processing them.
|
|
1412
|
+
this.logger?.warn?.("Continuation origin settlement will finish during restart recovery.", {
|
|
1413
|
+
reason: safeReason(error),
|
|
1414
|
+
});
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
for (const binding of bindings)
|
|
1418
|
+
this.settleClaimBinding(binding);
|
|
1038
1419
|
}
|
|
1039
1420
|
finally {
|
|
1040
1421
|
await this.releaseStoreLock();
|
|
@@ -1072,6 +1453,13 @@ function statusOfRequired(record) {
|
|
|
1072
1453
|
updatedAt: record.updatedAt,
|
|
1073
1454
|
deadline: record.deadline,
|
|
1074
1455
|
attempts: { synthesis: record.synthesisAttempts, delivery: record.deliveryAttempts },
|
|
1456
|
+
synthesisDeferrals: record.synthesisDeferrals,
|
|
1457
|
+
originContext: {
|
|
1458
|
+
state: record.originContextState,
|
|
1459
|
+
...(record.originContextDigest === undefined ? {} : { digest: record.originContextDigest }),
|
|
1460
|
+
...(record.originContextMessageCount === undefined ? {} : { messageCount: record.originContextMessageCount }),
|
|
1461
|
+
},
|
|
1462
|
+
...(record.completionKind === undefined ? {} : { completionKind: record.completionKind }),
|
|
1075
1463
|
...(record.nextAttemptAt === undefined ? {} : { nextAttemptAt: record.nextAttemptAt }),
|
|
1076
1464
|
...(record.lastError === undefined ? {} : { lastError: record.lastError }),
|
|
1077
1465
|
...(record.receipt === undefined ? {} : { receipt: record.receipt }),
|
|
@@ -1083,16 +1471,49 @@ function requireRecord(records, id) {
|
|
|
1083
1471
|
throw new ContinuationProtocolError(404, "not_found", "Continuation not found.");
|
|
1084
1472
|
return record;
|
|
1085
1473
|
}
|
|
1086
|
-
function synthesisInput(record) {
|
|
1087
|
-
|
|
1474
|
+
function synthesisInput(record, originContext) {
|
|
1475
|
+
const common = {
|
|
1088
1476
|
continuationId: record.continuationId,
|
|
1089
1477
|
originConversationId: record.originConversationId,
|
|
1090
1478
|
originRunId: record.originRunId,
|
|
1091
1479
|
...(record.replyToConversationId === undefined ? {} : { replyToConversationId: record.replyToConversationId }),
|
|
1092
|
-
...(record.historyBoundary === undefined ? {} : { historyBoundary: record.historyBoundary }),
|
|
1093
1480
|
mode: record.mode,
|
|
1094
1481
|
payload: record.resultPayload,
|
|
1095
1482
|
};
|
|
1483
|
+
if (originContext.policy === "detached_latest") {
|
|
1484
|
+
return { ...common, originContextPolicy: "detached_latest" };
|
|
1485
|
+
}
|
|
1486
|
+
if (record.historyBoundary === undefined) {
|
|
1487
|
+
throw new Error(`Pinned continuation ${record.continuationId} is missing its history boundary.`);
|
|
1488
|
+
}
|
|
1489
|
+
return {
|
|
1490
|
+
...common,
|
|
1491
|
+
historyBoundary: record.historyBoundary,
|
|
1492
|
+
originContextPolicy: "pinned",
|
|
1493
|
+
originContext: originContext.snapshot,
|
|
1494
|
+
};
|
|
1495
|
+
}
|
|
1496
|
+
function originContextBindingMac(secret, record, reference) {
|
|
1497
|
+
return createHmac("sha256", secret).update(canonicalContinuationJson({
|
|
1498
|
+
version: 2,
|
|
1499
|
+
continuationId: record.continuationId,
|
|
1500
|
+
claimFingerprint: record.claimFingerprint,
|
|
1501
|
+
serverName: record.serverName,
|
|
1502
|
+
originRunId: record.originRunId,
|
|
1503
|
+
originConversationId: record.originConversationId,
|
|
1504
|
+
replyToConversationId: record.replyToConversationId ?? null,
|
|
1505
|
+
historyBoundary: record.historyBoundary ?? null,
|
|
1506
|
+
mode: record.mode,
|
|
1507
|
+
routeName: record.routeName ?? null,
|
|
1508
|
+
taskKey: record.taskKey,
|
|
1509
|
+
taskHash: record.taskHash,
|
|
1510
|
+
resultTokenHash: record.resultTokenHash,
|
|
1511
|
+
createdAt: record.createdAt,
|
|
1512
|
+
deadline: record.deadline,
|
|
1513
|
+
originContextDigest: reference.digest,
|
|
1514
|
+
originContextBytes: reference.bytes,
|
|
1515
|
+
originContextMessageCount: reference.messageCount,
|
|
1516
|
+
})).digest("hex");
|
|
1096
1517
|
}
|
|
1097
1518
|
function requireLease(record, owner) {
|
|
1098
1519
|
if (record.leaseOwner !== owner)
|