larkin 0.3.3 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -33
- package/assets/readme/larkin-hero.svg +111 -0
- package/dist/agent/agent-state-store.cjs +321 -22
- package/dist/agent/agent-state-store.mjs +320 -22
- package/dist/agent/agent-transport.cjs +6 -4
- package/dist/agent/agent-transport.mjs +6 -4
- package/dist/agent/host-reminder-orchestrator.cjs +9 -3
- package/dist/agent/host-reminder-orchestrator.mjs +9 -3
- package/dist/agent/inbox-projection.cjs +114 -23
- package/dist/agent/inbox-projection.mjs +112 -23
- package/dist/app/cli.mjs +3 -1
- package/dist/app/local-control.mjs +108 -2
- package/dist/app/runtime-process.mjs +4 -0
- package/dist/app/session-cli.mjs +55 -20
- package/dist/feishu/document-comment.cjs +2 -1
- package/dist/feishu/document-comment.mjs +2 -1
- package/dist/feishu/host-business-state.cjs +7 -2
- package/dist/feishu/host-business-state.mjs +7 -2
- package/dist/feishu/host-shell.cjs +138 -32
- package/dist/feishu/host-shell.mjs +138 -32
- package/dist/runtime/pi-compaction-recovery.cjs +623 -0
- package/dist/runtime/pi-compaction-recovery.mjs +602 -0
- package/dist/runtime/pi-provider-config.cjs +2 -2
- package/dist/runtime/pi-provider-config.mjs +2 -2
- package/dist/runtime/pi-rpc-client.cjs +39 -22
- package/dist/runtime/pi-rpc-client.mjs +39 -22
- package/dist/runtime/provider-error-classifier.cjs +39 -0
- package/dist/runtime/provider-error-classifier.mjs +34 -0
- package/dist/runtime/runtime-adapters.mjs +82 -19
- package/dist/runtime/runtime-host.cjs +663 -12
- package/dist/runtime/runtime-host.mjs +661 -11
- package/package.json +1 -1
|
@@ -3,16 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.RuntimeSessionResetError = void 0;
|
|
6
|
+
exports.RuntimeSessionRecoveryError = exports.RuntimeSessionResetError = void 0;
|
|
7
7
|
exports.createRuntimeHost = createRuntimeHost;
|
|
8
8
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
9
9
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
10
10
|
const node_path_1 = __importDefault(require("node:path"));
|
|
11
11
|
const agent_cli_capabilities_js_1 = require("../agent/agent-cli-capabilities.cjs");
|
|
12
|
+
const inbox_projection_js_1 = require("../agent/inbox-projection.cjs");
|
|
12
13
|
const api_1 = require("@opentelemetry/api");
|
|
14
|
+
const provider_error_classifier_js_1 = require("./provider-error-classifier.cjs");
|
|
13
15
|
const runtime_readiness_js_1 = require("./runtime-readiness.cjs");
|
|
14
16
|
const official_lark_cli_js_1 = require("../app/official-lark-cli.cjs");
|
|
15
17
|
const agent_lark_cli_workspace_js_1 = require("../app/agent-lark-cli-workspace.cjs");
|
|
18
|
+
const pi_compaction_recovery_js_1 = require("./pi-compaction-recovery.cjs");
|
|
16
19
|
class RuntimeSessionResetError extends Error {
|
|
17
20
|
code;
|
|
18
21
|
pendingCount;
|
|
@@ -24,9 +27,34 @@ class RuntimeSessionResetError extends Error {
|
|
|
24
27
|
}
|
|
25
28
|
}
|
|
26
29
|
exports.RuntimeSessionResetError = RuntimeSessionResetError;
|
|
30
|
+
class RuntimeSessionRecoveryError extends Error {
|
|
31
|
+
code;
|
|
32
|
+
pendingCount;
|
|
33
|
+
constructor(code, message, pendingCount = 0) {
|
|
34
|
+
super(message);
|
|
35
|
+
this.code = code;
|
|
36
|
+
this.pendingCount = pendingCount;
|
|
37
|
+
this.name = "RuntimeSessionRecoveryError";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
exports.RuntimeSessionRecoveryError = RuntimeSessionRecoveryError;
|
|
27
41
|
const MAX_DELIVERIES = 2048;
|
|
28
42
|
const now = () => new Date().toISOString();
|
|
29
43
|
const isActiveDelivery = (status) => ["pending", "submitting", "accepted"].includes(status);
|
|
44
|
+
const REPLAY_FAILURE_CODES = new Set([
|
|
45
|
+
"canonical_inbox_row_missing", "canonical_inbox_malformed", "duplicate_message_id",
|
|
46
|
+
"inbox_state_conflict", "delivery_target_conflict", "structured_target_invalid",
|
|
47
|
+
"structured_target_missing", "wake_reason_conflict",
|
|
48
|
+
]);
|
|
49
|
+
function replayFailureReason(code) {
|
|
50
|
+
return `Runtime delivery quarantined (${code}): no safe canonical Inbox target is available; the Inbox/ledger remain durable for operator recovery`;
|
|
51
|
+
}
|
|
52
|
+
function replayFailureCodeOf(record) {
|
|
53
|
+
if (record.status !== "error" || typeof record.reason !== "string")
|
|
54
|
+
return null;
|
|
55
|
+
const code = /^Runtime delivery quarantined \(([^)]+)\):/.exec(record.reason)?.[1];
|
|
56
|
+
return code && REPLAY_FAILURE_CODES.has(code) ? code : null;
|
|
57
|
+
}
|
|
30
58
|
function deliveryFile(agent) {
|
|
31
59
|
return agent.config.stateDir ? node_path_1.default.join(agent.config.stateDir, "runtime-deliveries.json") : null;
|
|
32
60
|
}
|
|
@@ -99,6 +127,9 @@ function createRuntimeHost(options) {
|
|
|
99
127
|
const listeners = new Set();
|
|
100
128
|
const log = options.log ?? (() => { });
|
|
101
129
|
const telemetry = options.telemetry;
|
|
130
|
+
const compactTimeoutMs = options.compactTimeoutMs ?? 120_000;
|
|
131
|
+
if (!Number.isFinite(compactTimeoutMs) || compactTimeoutMs <= 0)
|
|
132
|
+
throw new Error("compactTimeoutMs must be positive");
|
|
102
133
|
const retryPolicy = {
|
|
103
134
|
baseDelayMs: options.retryPolicy?.baseDelayMs ?? 250,
|
|
104
135
|
maxDelayMs: options.retryPolicy?.maxDelayMs ?? 10_000,
|
|
@@ -159,6 +190,82 @@ function createRuntimeHost(options) {
|
|
|
159
190
|
emitConsumed(agent, persist(agent));
|
|
160
191
|
return agent.records.get(record.deliveryId) ?? record;
|
|
161
192
|
};
|
|
193
|
+
const scrubQuarantinedInput = (record, reason) => {
|
|
194
|
+
const staleInput = record.input && typeof record.input === "object" ? record.input : null;
|
|
195
|
+
record.input = {
|
|
196
|
+
inputId: typeof staleInput?.inputId === "string" && staleInput.inputId ? staleInput.inputId : record.deliveryId,
|
|
197
|
+
deliveryId: record.deliveryId,
|
|
198
|
+
kind: "wake",
|
|
199
|
+
text: reason,
|
|
200
|
+
attempt: Number.isSafeInteger(staleInput?.attempt) && Number(staleInput?.attempt) >= 0 ? Number(staleInput.attempt) : 0,
|
|
201
|
+
};
|
|
202
|
+
};
|
|
203
|
+
/** Rebuild every submitted notice from canonical structured state, never persisted text. */
|
|
204
|
+
const prepareRecordInput = (agent, record, busy) => {
|
|
205
|
+
if (record.target !== undefined && (typeof record.target !== "string" || !(0, inbox_projection_js_1.isCanonicalInboxTarget)(record.target))) {
|
|
206
|
+
return { status: "error", code: "structured_target_invalid", reason: replayFailureReason("structured_target_invalid") };
|
|
207
|
+
}
|
|
208
|
+
if (record.wakeReason !== undefined && typeof record.wakeReason !== "string") {
|
|
209
|
+
return { status: "error", code: "wake_reason_conflict", reason: replayFailureReason("wake_reason_conflict") };
|
|
210
|
+
}
|
|
211
|
+
let target = typeof record.target === "string" ? record.target : null;
|
|
212
|
+
let wakeReason = typeof record.wakeReason === "string" ? record.wakeReason : undefined;
|
|
213
|
+
if (agent.stateStore?.resolveInboxDeliverySource) {
|
|
214
|
+
let source;
|
|
215
|
+
try {
|
|
216
|
+
source = agent.stateStore.resolveInboxDeliverySource(record.messageId);
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
source = { status: "invalid", code: "canonical_inbox_malformed" };
|
|
220
|
+
}
|
|
221
|
+
if (source.status === "consumed")
|
|
222
|
+
return { status: "consumed" };
|
|
223
|
+
if (source.status === "missing" || source.status === "invalid") {
|
|
224
|
+
return { status: "error", code: source.code, reason: replayFailureReason(source.code) };
|
|
225
|
+
}
|
|
226
|
+
if (target && target !== source.target) {
|
|
227
|
+
return { status: "error", code: "delivery_target_conflict", reason: replayFailureReason("delivery_target_conflict") };
|
|
228
|
+
}
|
|
229
|
+
const canonicalWakeReason = typeof source.envelope.wake_reason === "string" ? source.envelope.wake_reason : undefined;
|
|
230
|
+
if (record.wakeReason !== undefined && wakeReason !== canonicalWakeReason) {
|
|
231
|
+
return { status: "error", code: "wake_reason_conflict", reason: replayFailureReason("wake_reason_conflict") };
|
|
232
|
+
}
|
|
233
|
+
target = source.target;
|
|
234
|
+
wakeReason = canonicalWakeReason;
|
|
235
|
+
}
|
|
236
|
+
if (!target)
|
|
237
|
+
return { status: "error", code: "structured_target_missing", reason: replayFailureReason("structured_target_missing") };
|
|
238
|
+
const staleInput = record.input && typeof record.input === "object" ? record.input : null;
|
|
239
|
+
const attempt = Number.isSafeInteger(staleInput?.attempt) && Number(staleInput?.attempt) >= 0 ? Number(staleInput.attempt) : 0;
|
|
240
|
+
const inputId = typeof staleInput?.inputId === "string" && staleInput.inputId ? staleInput.inputId : record.deliveryId;
|
|
241
|
+
record.target = target;
|
|
242
|
+
if (wakeReason)
|
|
243
|
+
record.wakeReason = wakeReason;
|
|
244
|
+
else
|
|
245
|
+
delete record.wakeReason;
|
|
246
|
+
record.input = {
|
|
247
|
+
inputId,
|
|
248
|
+
deliveryId: record.deliveryId,
|
|
249
|
+
kind: busy ? "inbox_update" : "wake",
|
|
250
|
+
text: options.promptBuilder.buildInboxNotice({ busy, count: 1, deliveryId: record.deliveryId, target,
|
|
251
|
+
...(wakeReason ? { wakeReason } : {}) }),
|
|
252
|
+
attempt,
|
|
253
|
+
};
|
|
254
|
+
return { status: "ready", target };
|
|
255
|
+
};
|
|
256
|
+
const quarantineRecord = (agent, record, code) => {
|
|
257
|
+
const reason = replayFailureReason(code);
|
|
258
|
+
scrubQuarantinedInput(record, reason);
|
|
259
|
+
record.reason = reason;
|
|
260
|
+
record.retryable = false;
|
|
261
|
+
delete record.errorCategory;
|
|
262
|
+
const finalRecord = setRecord(agent, record, "error");
|
|
263
|
+
if (finalRecord.status === "consumed")
|
|
264
|
+
return { status: "accepted", deliveryId: record.deliveryId };
|
|
265
|
+
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
|
|
266
|
+
messageId: record.messageId, status: "error", reason });
|
|
267
|
+
return { status: "error", deliveryId: record.deliveryId, reason, retryable: false };
|
|
268
|
+
};
|
|
162
269
|
const reconcileExternalConsumption = (agent) => {
|
|
163
270
|
if (agent.stateStore) {
|
|
164
271
|
agent.stateStore.withInboxTransaction(() => reconcileFile(agent, agent.stateStore.readJson("runtimeDeliveries", { version: 1, records: [] })));
|
|
@@ -292,12 +399,14 @@ function createRuntimeHost(options) {
|
|
|
292
399
|
if (result.status === "accepted") {
|
|
293
400
|
delete record.reason;
|
|
294
401
|
delete record.retryable;
|
|
402
|
+
delete record.errorCategory;
|
|
295
403
|
const finalRecord = setRecord(agent, record, "accepted");
|
|
296
404
|
if (finalRecord.status !== "consumed")
|
|
297
405
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
298
406
|
deliveryId: record.deliveryId, messageId: record.messageId, status: "accepted" });
|
|
299
407
|
return { status: "accepted", deliveryId: record.deliveryId };
|
|
300
408
|
}
|
|
409
|
+
delete record.errorCategory;
|
|
301
410
|
const retryable = result.status === "deferred" || result.retryable;
|
|
302
411
|
record.reason = result.reason;
|
|
303
412
|
record.retryable = retryable;
|
|
@@ -315,6 +424,20 @@ function createRuntimeHost(options) {
|
|
|
315
424
|
if (!busy)
|
|
316
425
|
agent.busy = true; // Reserve the turn before prompt() can yield.
|
|
317
426
|
try {
|
|
427
|
+
const prepared = prepareRecordInput(agent, record, busy);
|
|
428
|
+
if (prepared.status === "consumed") {
|
|
429
|
+
if (!busy)
|
|
430
|
+
agent.busy = false;
|
|
431
|
+
const consumedRecord = setRecord(agent, record, "consumed");
|
|
432
|
+
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: consumedRecord.deliveryId,
|
|
433
|
+
messageId: consumedRecord.messageId, status: "consumed" });
|
|
434
|
+
return { status: "accepted", deliveryId: record.deliveryId };
|
|
435
|
+
}
|
|
436
|
+
if (prepared.status === "error") {
|
|
437
|
+
if (!busy)
|
|
438
|
+
agent.busy = false;
|
|
439
|
+
return quarantineRecord(agent, record, prepared.code);
|
|
440
|
+
}
|
|
318
441
|
const session = await ensureSession(agent);
|
|
319
442
|
const submittingRecord = setRecord(agent, record, "submitting");
|
|
320
443
|
if (submittingRecord.status === "consumed") {
|
|
@@ -338,6 +461,7 @@ function createRuntimeHost(options) {
|
|
|
338
461
|
const reason = error instanceof Error ? error.message : String(error);
|
|
339
462
|
record.reason = reason;
|
|
340
463
|
record.retryable = true;
|
|
464
|
+
delete record.errorCategory;
|
|
341
465
|
setRecord(agent, record, "pending");
|
|
342
466
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId, messageId: record.messageId, status: "deferred", reason });
|
|
343
467
|
return { status: "deferred", deliveryId: record.deliveryId, reason };
|
|
@@ -464,6 +588,106 @@ function createRuntimeHost(options) {
|
|
|
464
588
|
})();
|
|
465
589
|
agent.configurationRecovery = recovery;
|
|
466
590
|
};
|
|
591
|
+
let recoverContextOverflowInternal;
|
|
592
|
+
const beginPiManualCompaction = (agent, session, record, nativeWillRetry = false) => {
|
|
593
|
+
if (agent.adapter.id !== "pi" || !session.compact || !agent.compactionBreaker)
|
|
594
|
+
return;
|
|
595
|
+
const existing = agent.compactionMachines.get(record.input.inputId);
|
|
596
|
+
if (existing) {
|
|
597
|
+
existing.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
const key = `${record.deliveryId}:${record.input.inputId}`;
|
|
601
|
+
const prior = agent.compactionBreaker.get(key);
|
|
602
|
+
if (prior && prior.manualAttempt >= 1 && ["manual_sent", "manual_ambiguous", "manual_failed", "native_failed"].includes(prior.state)) {
|
|
603
|
+
agent.busy = false;
|
|
604
|
+
agent.turnInProgress = false;
|
|
605
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, "validated Pi compaction breaker state after restart", key)
|
|
606
|
+
.catch((error) => log("Pi restart fallback failed", String(error)));
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
const machine = new pi_compaction_recovery_js_1.PiCompactionRecoveryMachine({
|
|
610
|
+
breaker: agent.compactionBreaker, key, messageId: record.messageId, deliveryId: record.deliveryId,
|
|
611
|
+
inputId: record.input.inputId, sessionGeneration: agent.generation,
|
|
612
|
+
onAction: (action) => {
|
|
613
|
+
if (action === "retry_input") {
|
|
614
|
+
machine.retrySubmitted();
|
|
615
|
+
record.status = "pending";
|
|
616
|
+
record.retryable = true;
|
|
617
|
+
record.reason = "Pi manual compaction succeeded; retrying the same delivery input";
|
|
618
|
+
record.updatedAt = now();
|
|
619
|
+
persist(agent);
|
|
620
|
+
queueMicrotask(() => { if (!agent.stopped)
|
|
621
|
+
void submit(agent, record, false); });
|
|
622
|
+
}
|
|
623
|
+
else if (action === "fresh_session_fallback") {
|
|
624
|
+
agent.busy = false;
|
|
625
|
+
agent.turnInProgress = false;
|
|
626
|
+
if (record.status !== "error") {
|
|
627
|
+
record.reason = machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback";
|
|
628
|
+
record.retryable = false;
|
|
629
|
+
record.errorCategory = "context_window";
|
|
630
|
+
setRecord(agent, record, "error");
|
|
631
|
+
}
|
|
632
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback", key)
|
|
633
|
+
.catch((error) => log("Pi context fallback failed", String(error)));
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
agent.compactionMachines.set(record.input.inputId, machine);
|
|
638
|
+
machine.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
639
|
+
if (nativeWillRetry)
|
|
640
|
+
return;
|
|
641
|
+
machine.agentSettled();
|
|
642
|
+
const deadline = new Date(Date.now() + compactTimeoutMs).toISOString();
|
|
643
|
+
machine.manualCompactSent(deadline);
|
|
644
|
+
let compactSettled = false;
|
|
645
|
+
const compactTimer = setTimeout(() => {
|
|
646
|
+
if (compactSettled)
|
|
647
|
+
return;
|
|
648
|
+
compactSettled = true;
|
|
649
|
+
machine.compactTimeout();
|
|
650
|
+
}, compactTimeoutMs);
|
|
651
|
+
compactTimer.unref?.();
|
|
652
|
+
void session.compact().then(() => {
|
|
653
|
+
if (compactSettled)
|
|
654
|
+
return;
|
|
655
|
+
compactSettled = true;
|
|
656
|
+
clearTimeout(compactTimer);
|
|
657
|
+
machine.compactResponse({ success: true });
|
|
658
|
+
}).catch((error) => {
|
|
659
|
+
if (compactSettled)
|
|
660
|
+
return;
|
|
661
|
+
compactSettled = true;
|
|
662
|
+
clearTimeout(compactTimer);
|
|
663
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
664
|
+
machine.compactResponse({ success: false, ambiguous: /process exited|process failed|session replaced|protocol/i.test(message) });
|
|
665
|
+
});
|
|
666
|
+
};
|
|
667
|
+
const recoverStalePiCompaction = async (agent) => {
|
|
668
|
+
const breaker = agent.compactionBreaker;
|
|
669
|
+
if (!breaker)
|
|
670
|
+
return;
|
|
671
|
+
const stale = breaker.listNonTerminal().find((candidate) => candidate.state !== "native_succeeded");
|
|
672
|
+
if (!stale) {
|
|
673
|
+
const succeeded = breaker.listNonTerminal().find((candidate) => candidate.state === "native_succeeded");
|
|
674
|
+
if (succeeded)
|
|
675
|
+
breaker.transition(succeeded.key, {}, "closed");
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === stale.deliveryId
|
|
679
|
+
|| candidate.input.inputId === stale.inputId);
|
|
680
|
+
if (!record)
|
|
681
|
+
return;
|
|
682
|
+
record.reason = stale.fallbackReason || "Pi compaction recovery resumed after restart";
|
|
683
|
+
record.retryable = false;
|
|
684
|
+
record.errorCategory = "context_window";
|
|
685
|
+
setRecord(agent, record, "error");
|
|
686
|
+
breaker.forceFallback(stale.key, record.reason);
|
|
687
|
+
agent.busy = false;
|
|
688
|
+
agent.turnInProgress = false;
|
|
689
|
+
await recoverContextOverflowInternal(agent.config.agentId, stale.key, record.reason, stale.key);
|
|
690
|
+
};
|
|
467
691
|
const observe = (agent, session, event) => {
|
|
468
692
|
if (agent.session !== session)
|
|
469
693
|
return; // Ignore late output from a replaced child.
|
|
@@ -472,6 +696,34 @@ function createRuntimeHost(options) {
|
|
|
472
696
|
if (event.type !== "turn-end")
|
|
473
697
|
telemetry?.runtimeEvent(agent.config.agentId, event);
|
|
474
698
|
emit({ type: "runtime", agentId: agent.config.agentId, event });
|
|
699
|
+
if (event.type === "runtime-observation") {
|
|
700
|
+
if (event.phase === "compaction_start" && event.reason === "overflow" && event.inputId) {
|
|
701
|
+
const overflowRecord = [...agent.records.values()].find((candidate) => candidate.input.inputId === event.inputId);
|
|
702
|
+
if (overflowRecord && !agent.compactionMachines.has(event.inputId))
|
|
703
|
+
beginPiManualCompaction(agent, session, overflowRecord, true);
|
|
704
|
+
}
|
|
705
|
+
const candidates = [...agent.compactionMachines.values()].filter((machine) => machine.recordSnapshot.sessionGeneration === agent.generation
|
|
706
|
+
&& (!event.inputId || machine.recordSnapshot.inputId === event.inputId));
|
|
707
|
+
const machines = event.inputId ? candidates : candidates.length === 1 ? candidates : [];
|
|
708
|
+
for (const machine of machines) {
|
|
709
|
+
if (event.phase === "compaction_start" && event.reason === "overflow")
|
|
710
|
+
machine.compactionStart({ reason: "overflow" });
|
|
711
|
+
if (event.phase === "compaction_end" && event.reason === "manual") {
|
|
712
|
+
machine.compactionEnd({ reason: "manual", success: event.success === true, willRetry: event.willRetry === true });
|
|
713
|
+
}
|
|
714
|
+
if (event.phase === "compaction_end" && event.reason === "overflow") {
|
|
715
|
+
machine.compactionEnd({ reason: "overflow", success: event.success === true, willRetry: event.willRetry === true });
|
|
716
|
+
}
|
|
717
|
+
if (event.phase === "settled")
|
|
718
|
+
machine.agentSettled();
|
|
719
|
+
}
|
|
720
|
+
if (event.phase === "compaction_end" && event.reason === "overflow" && event.success !== true) {
|
|
721
|
+
for (const record of agent.records.values()) {
|
|
722
|
+
if (isActiveDelivery(record.status))
|
|
723
|
+
agent.piOverflowCompactionFailed.add(record.input.inputId);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
}
|
|
475
727
|
if (event.type === "session-init") {
|
|
476
728
|
agent.config.sessionId = event.sessionId;
|
|
477
729
|
emit({ type: "session", agentId: agent.config.agentId, runtime: agent.adapter.id, sessionId: event.sessionId, launchId: agent.launchId,
|
|
@@ -487,6 +739,13 @@ function createRuntimeHost(options) {
|
|
|
487
739
|
emit({ type: "activity", agentId: agent.config.agentId, activity: "working", activityKind: "working", detailKind: "turn_started" });
|
|
488
740
|
}
|
|
489
741
|
else if (event.type === "turn-end") {
|
|
742
|
+
for (const [inputId, machine] of agent.compactionMachines) {
|
|
743
|
+
if (machine.recordSnapshot.sessionGeneration !== agent.generation)
|
|
744
|
+
continue;
|
|
745
|
+
machine.close();
|
|
746
|
+
if (["closed", "fallback_committed"].includes(machine.state))
|
|
747
|
+
agent.compactionMachines.delete(inputId);
|
|
748
|
+
}
|
|
490
749
|
const recoveredAuthentication = agent.authFailureActive && agent.turnInProgress
|
|
491
750
|
&& agent.turnHadAuthenticatedOutput && !agent.turnHadFailure;
|
|
492
751
|
agent.turnInProgress = false;
|
|
@@ -519,6 +778,29 @@ function createRuntimeHost(options) {
|
|
|
519
778
|
if (!record || record.status === "consumed" || record.status === "error")
|
|
520
779
|
return;
|
|
521
780
|
agent.turnHadFailure = true;
|
|
781
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry === true && session.compact) {
|
|
782
|
+
beginPiManualCompaction(agent, session, record, true);
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry !== true && session.compact) {
|
|
786
|
+
record.reason = event.message;
|
|
787
|
+
record.retryable = false;
|
|
788
|
+
record.errorCategory = "context_window";
|
|
789
|
+
setRecord(agent, record, "error");
|
|
790
|
+
if (agent.piOverflowCompactionFailed.has(record.input.inputId)) {
|
|
791
|
+
agent.busy = false;
|
|
792
|
+
agent.turnInProgress = false;
|
|
793
|
+
void recoverContextOverflowInternal(agent.config.agentId, `${record.deliveryId}:${record.input.inputId}`, "native Pi overflow compaction failed", `${record.deliveryId}:${record.input.inputId}`)
|
|
794
|
+
.catch((error) => log("Pi native compaction fallback failed", String(error)));
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
// Pi emits this terminal error only after agent_settled; release the
|
|
798
|
+
// normal turn reservation before the one authorized recovery action.
|
|
799
|
+
agent.busy = false;
|
|
800
|
+
agent.turnInProgress = false;
|
|
801
|
+
beginPiManualCompaction(agent, session, record);
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
522
804
|
if (event.willRetry) {
|
|
523
805
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
|
|
524
806
|
messageId: record.messageId, status: "deferred", reason: event.message });
|
|
@@ -529,6 +811,13 @@ function createRuntimeHost(options) {
|
|
|
529
811
|
agent.busy = false;
|
|
530
812
|
record.reason = event.message;
|
|
531
813
|
record.retryable = event.retryable;
|
|
814
|
+
const classifiedCategory = event.errorCategory ?? (0, provider_error_classifier_js_1.classifyStrictProviderError)((0, provider_error_classifier_js_1.buildStrictProviderErrorInput)({
|
|
815
|
+
message: event.message, errorCategory: event.errorCategory, upstream: event.upstream,
|
|
816
|
+
}));
|
|
817
|
+
if (classifiedCategory)
|
|
818
|
+
record.errorCategory = classifiedCategory;
|
|
819
|
+
else
|
|
820
|
+
delete record.errorCategory;
|
|
532
821
|
const finalRecord = setRecord(agent, record, event.retryable ? "pending" : "error");
|
|
533
822
|
if (finalRecord.status !== "consumed")
|
|
534
823
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
@@ -615,6 +904,129 @@ function createRuntimeHost(options) {
|
|
|
615
904
|
});
|
|
616
905
|
return agent.starting;
|
|
617
906
|
};
|
|
907
|
+
recoverContextOverflowInternal = async (agentId, deliveryKey, reason, breakerKey) => {
|
|
908
|
+
const agent = managed.get(agentId);
|
|
909
|
+
if (!agent || agent.stopped)
|
|
910
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
911
|
+
if (agent.adapter.id !== "pi")
|
|
912
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "Pi context fallback is unavailable for this runtime");
|
|
913
|
+
const oldSession = agent.session;
|
|
914
|
+
if (!oldSession || agent.busy || agent.submitting || agent.starting) {
|
|
915
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle for Pi context fallback`);
|
|
916
|
+
}
|
|
917
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === deliveryKey || candidate.input.inputId === deliveryKey || `${candidate.deliveryId}:${candidate.input.inputId}` === deliveryKey);
|
|
918
|
+
if (!record)
|
|
919
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "Pi context fallback delivery identity is unavailable");
|
|
920
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:pi-context-fallback`);
|
|
921
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
922
|
+
const readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
923
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir, env: probeEnv }) : { runtime: "pi", state: "ready" };
|
|
924
|
+
if (readiness.state !== "ready")
|
|
925
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback readiness failed");
|
|
926
|
+
const standingPrompt = options.promptBuilder.build({
|
|
927
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
928
|
+
description: agent.config.description || "", runtime: agent.adapter.id, cli: (0, agent_cli_capabilities_js_1.agentCliPromptCapabilities)("larkin"),
|
|
929
|
+
});
|
|
930
|
+
let fresh;
|
|
931
|
+
try {
|
|
932
|
+
fresh = await agent.adapter.createSession({ agentId: agent.config.agentId, model: agent.config.model,
|
|
933
|
+
reasoningEffort: agent.config.effort || null, workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
934
|
+
resumeSessionId: null, standingPrompt, env: probeEnv });
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi fresh fallback session could not be staged");
|
|
938
|
+
}
|
|
939
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
940
|
+
await fresh.close("Pi context fallback commit precondition changed").catch(() => { });
|
|
941
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
942
|
+
}
|
|
943
|
+
let unsubscribeFresh = null;
|
|
944
|
+
try {
|
|
945
|
+
unsubscribeFresh = fresh.subscribe((next) => observe(agent, fresh, next));
|
|
946
|
+
}
|
|
947
|
+
catch {
|
|
948
|
+
await fresh.close("Pi context fallback subscription failed").catch(() => { });
|
|
949
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback subscription failed");
|
|
950
|
+
}
|
|
951
|
+
const oldSessionId = oldSession.sessionId;
|
|
952
|
+
const oldGeneration = agent.generation;
|
|
953
|
+
const oldLaunchId = agent.launchId;
|
|
954
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
955
|
+
const oldReadiness = agent.readiness;
|
|
956
|
+
const oldDisabledReason = agent.disabledReason;
|
|
957
|
+
const oldRecord = { ...record, input: { ...record.input } };
|
|
958
|
+
let durableRollback = null;
|
|
959
|
+
let rearmedCount = 1;
|
|
960
|
+
let remainingPendingCount = 1;
|
|
961
|
+
const commitSwap = (messageIds, rollback) => {
|
|
962
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
963
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
964
|
+
}
|
|
965
|
+
durableRollback = rollback ?? null;
|
|
966
|
+
if (breakerKey && agent.compactionBreaker)
|
|
967
|
+
agent.compactionBreaker.forceFallbackInTransaction(breakerKey, reason);
|
|
968
|
+
agent.generation += 1;
|
|
969
|
+
agent.launchId = node_crypto_1.default.randomUUID();
|
|
970
|
+
agent.config.sessionId = null;
|
|
971
|
+
agent.session = fresh;
|
|
972
|
+
agent.readiness = readiness;
|
|
973
|
+
agent.disabledReason = null;
|
|
974
|
+
for (const candidate of agent.records.values()) {
|
|
975
|
+
if (!messageIds.includes(candidate.messageId))
|
|
976
|
+
continue;
|
|
977
|
+
candidate.status = "pending";
|
|
978
|
+
candidate.retryable = true;
|
|
979
|
+
candidate.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
980
|
+
candidate.updatedAt = now();
|
|
981
|
+
}
|
|
982
|
+
};
|
|
983
|
+
try {
|
|
984
|
+
if (agent.stateStore?.rearmContextOverflow) {
|
|
985
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => commitSwap(messageIds, rollback), {
|
|
986
|
+
messageId: record.messageId, deliveryId: record.deliveryId, inputId: record.input.inputId,
|
|
987
|
+
});
|
|
988
|
+
rearmedCount = rearmed.rearmedCount;
|
|
989
|
+
remainingPendingCount = rearmed.remainingPendingCount;
|
|
990
|
+
}
|
|
991
|
+
else {
|
|
992
|
+
commitSwap([record.messageId]);
|
|
993
|
+
persist(agent);
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
catch (error) {
|
|
997
|
+
try {
|
|
998
|
+
const rollback = durableRollback;
|
|
999
|
+
if (rollback)
|
|
1000
|
+
rollback();
|
|
1001
|
+
}
|
|
1002
|
+
catch { /* preserve primary error */ }
|
|
1003
|
+
agent.generation = oldGeneration;
|
|
1004
|
+
agent.launchId = oldLaunchId;
|
|
1005
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
1006
|
+
agent.session = oldSession;
|
|
1007
|
+
agent.readiness = oldReadiness;
|
|
1008
|
+
agent.disabledReason = oldDisabledReason;
|
|
1009
|
+
Object.assign(record, oldRecord);
|
|
1010
|
+
try {
|
|
1011
|
+
unsubscribeFresh?.();
|
|
1012
|
+
}
|
|
1013
|
+
catch { /* preserve primary failure */ }
|
|
1014
|
+
await fresh.close("Pi context fallback rolled back").catch(() => { });
|
|
1015
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", error instanceof Error ? error.message : "Pi context fallback commit failed");
|
|
1016
|
+
}
|
|
1017
|
+
if (fresh.sessionId) {
|
|
1018
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1019
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1020
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1021
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1022
|
+
}
|
|
1023
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1024
|
+
await oldSession.close("Pi context fallback committed").catch((error) => log("previous Pi session close after fallback failed", String(error)));
|
|
1025
|
+
void retryPending(agent);
|
|
1026
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1027
|
+
runtimeReady: true, pendingCount: remainingPendingCount, rearmedCount,
|
|
1028
|
+
replayStatus: remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1029
|
+
};
|
|
618
1030
|
return {
|
|
619
1031
|
async probe(config) {
|
|
620
1032
|
const adapter = options.adapterFor(config.runtime);
|
|
@@ -789,6 +1201,180 @@ function createRuntimeHost(options) {
|
|
|
789
1201
|
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
790
1202
|
runtimeReady: true, pendingCount: 0, sessionId: fresh.sessionId };
|
|
791
1203
|
},
|
|
1204
|
+
async recoverSession(agentId, reason) {
|
|
1205
|
+
if (reason !== "context-overflow")
|
|
1206
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "unsupported recovery reason");
|
|
1207
|
+
const agent = managed.get(agentId);
|
|
1208
|
+
if (agent && agent.adapter.id !== "pi")
|
|
1209
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "context-overflow recovery is available only for Pi Runtime");
|
|
1210
|
+
if (!agent || agent.stopped)
|
|
1211
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
1212
|
+
if (!agent.stateStore?.rearmContextOverflow) {
|
|
1213
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "canonical Inbox recovery is unavailable");
|
|
1214
|
+
}
|
|
1215
|
+
if (agent.busy || agent.turnInProgress || agent.submitting || agent.starting) {
|
|
1216
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle`);
|
|
1217
|
+
}
|
|
1218
|
+
const oldSession = agent.session;
|
|
1219
|
+
if (!oldSession)
|
|
1220
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} Runtime session is unavailable`);
|
|
1221
|
+
const oldSessionId = oldSession.sessionId;
|
|
1222
|
+
if (agent.stateStore.readNdjson) {
|
|
1223
|
+
const pendingRows = agent.stateStore.withInboxTransaction(() => agent.stateStore.readNdjson("inbox"));
|
|
1224
|
+
if (pendingRows.length === 0) {
|
|
1225
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery has no retained Inbox backlog");
|
|
1226
|
+
}
|
|
1227
|
+
const messageIds = new Set(pendingRows.flatMap((row) => {
|
|
1228
|
+
const messageId = row && typeof row === "object" && !Array.isArray(row)
|
|
1229
|
+
? row.message_id : undefined;
|
|
1230
|
+
return typeof messageId === "string" && messageId ? [messageId] : [];
|
|
1231
|
+
}));
|
|
1232
|
+
const persisted = agent.stateStore.readJson("runtimeDeliveries", { version: 1, records: [] });
|
|
1233
|
+
if (Array.isArray(persisted.records) && persisted.records.some((record) => messageIds.has(record.messageId) && record.status !== "error")) {
|
|
1234
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery is already staged or completed");
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:context-overflow-recovery`);
|
|
1238
|
+
let readiness;
|
|
1239
|
+
try {
|
|
1240
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
1241
|
+
readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
1242
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1243
|
+
env: { LARKIN_PI_COMMAND: process.env.LARKIN_PI_COMMAND, LARKIN_CODEX_COMMAND: process.env.LARKIN_CODEX_COMMAND,
|
|
1244
|
+
LARKIN_CLAUDE_COMMAND: process.env.LARKIN_CLAUDE_COMMAND, ...probeEnv } })
|
|
1245
|
+
: { runtime: agent.adapter.id, state: "ready" };
|
|
1246
|
+
}
|
|
1247
|
+
catch (error) {
|
|
1248
|
+
if (error instanceof runtime_readiness_js_1.RuntimePrerequisiteError)
|
|
1249
|
+
throw error;
|
|
1250
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Runtime readiness probe failed");
|
|
1251
|
+
}
|
|
1252
|
+
if (readiness.state !== "ready")
|
|
1253
|
+
throw new runtime_readiness_js_1.RuntimePrerequisiteError(readiness);
|
|
1254
|
+
const standingPrompt = options.promptBuilder.build({
|
|
1255
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
1256
|
+
description: agent.config.description || "", runtime: agent.adapter.id,
|
|
1257
|
+
cli: (0, agent_cli_capabilities_js_1.agentCliPromptCapabilities)("larkin"),
|
|
1258
|
+
});
|
|
1259
|
+
let fresh;
|
|
1260
|
+
try {
|
|
1261
|
+
fresh = await agent.adapter.createSession({
|
|
1262
|
+
agentId: agent.config.agentId, model: agent.config.model, reasoningEffort: agent.config.effort || null,
|
|
1263
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1264
|
+
resumeSessionId: null, standingPrompt, env: probeEnv,
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
catch {
|
|
1268
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session could not be staged");
|
|
1269
|
+
}
|
|
1270
|
+
let unsubscribeFresh = null;
|
|
1271
|
+
try {
|
|
1272
|
+
unsubscribeFresh = fresh.subscribe((event) => observe(agent, fresh, event));
|
|
1273
|
+
}
|
|
1274
|
+
catch {
|
|
1275
|
+
await fresh.close("context-window recovery subscription failed").catch(() => { });
|
|
1276
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session subscription failed");
|
|
1277
|
+
}
|
|
1278
|
+
const oldGeneration = agent.generation;
|
|
1279
|
+
const oldLaunchId = agent.launchId;
|
|
1280
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
1281
|
+
const oldReadiness = agent.readiness;
|
|
1282
|
+
const oldDisabledReason = agent.disabledReason;
|
|
1283
|
+
const oldRecords = new Map([...agent.records.entries()].map(([id, record]) => [id, { ...record, input: { ...record.input } }]));
|
|
1284
|
+
const cleanupFresh = async (reason) => {
|
|
1285
|
+
const unsubscribe = unsubscribeFresh;
|
|
1286
|
+
unsubscribeFresh = null;
|
|
1287
|
+
try {
|
|
1288
|
+
unsubscribe?.();
|
|
1289
|
+
}
|
|
1290
|
+
catch { /* close still owns staged-session cleanup */ }
|
|
1291
|
+
await fresh.close(reason).catch(() => { });
|
|
1292
|
+
};
|
|
1293
|
+
const restoreRuntime = () => {
|
|
1294
|
+
agent.session = oldSession;
|
|
1295
|
+
agent.generation = oldGeneration;
|
|
1296
|
+
agent.launchId = oldLaunchId;
|
|
1297
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
1298
|
+
agent.readiness = oldReadiness;
|
|
1299
|
+
agent.disabledReason = oldDisabledReason;
|
|
1300
|
+
agent.records = new Map(oldRecords);
|
|
1301
|
+
agent.byMessage = new Map([...oldRecords.values()].map((record) => [record.messageId, record.deliveryId]));
|
|
1302
|
+
};
|
|
1303
|
+
let recoveryProjectionEmitted = false;
|
|
1304
|
+
const compensateProjection = () => {
|
|
1305
|
+
if (!recoveryProjectionEmitted)
|
|
1306
|
+
return;
|
|
1307
|
+
try {
|
|
1308
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: oldSessionId, launchId: oldLaunchId });
|
|
1309
|
+
}
|
|
1310
|
+
catch { /* preserve the original listener failure */ }
|
|
1311
|
+
try {
|
|
1312
|
+
emit({ type: "agent-status", agentId, status: "active", ...(oldReadiness ? { readiness: oldReadiness } : {}) });
|
|
1313
|
+
}
|
|
1314
|
+
catch { /* preserve the original listener failure */ }
|
|
1315
|
+
};
|
|
1316
|
+
let durableRollback = null;
|
|
1317
|
+
try {
|
|
1318
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => {
|
|
1319
|
+
// This callback runs under the canonical Inbox lock. All potentially
|
|
1320
|
+
// racing checks above are synchronous from this process, while the
|
|
1321
|
+
// state-store method rechecks every row and delivery on disk.
|
|
1322
|
+
if (agent.session !== oldSession || agent.busy || agent.turnInProgress || agent.submitting || agent.starting || agent.stopped) {
|
|
1323
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during recovery`);
|
|
1324
|
+
}
|
|
1325
|
+
durableRollback = rollback;
|
|
1326
|
+
agent.generation += 1;
|
|
1327
|
+
agent.launchId = node_crypto_1.default.randomUUID();
|
|
1328
|
+
agent.config.sessionId = null;
|
|
1329
|
+
agent.session = fresh;
|
|
1330
|
+
agent.readiness = readiness;
|
|
1331
|
+
agent.disabledReason = null;
|
|
1332
|
+
for (const record of agent.records.values()) {
|
|
1333
|
+
if (!messageIds.includes(record.messageId))
|
|
1334
|
+
continue;
|
|
1335
|
+
record.status = "pending";
|
|
1336
|
+
record.retryable = true;
|
|
1337
|
+
record.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
1338
|
+
record.updatedAt = now();
|
|
1339
|
+
}
|
|
1340
|
+
});
|
|
1341
|
+
recoveryProjectionEmitted = true;
|
|
1342
|
+
if (fresh.sessionId) {
|
|
1343
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1344
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1345
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1346
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1347
|
+
}
|
|
1348
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1349
|
+
durableRollback = null;
|
|
1350
|
+
await oldSession.close("context-window recovery committed")
|
|
1351
|
+
.catch((error) => log("previous runtime close after context-window recovery failed", String(error)));
|
|
1352
|
+
void retryPending(agent);
|
|
1353
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1354
|
+
runtimeReady: true, pendingCount: rearmed.remainingPendingCount, rearmedCount: rearmed.rearmedCount,
|
|
1355
|
+
replayStatus: rearmed.remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1356
|
+
}
|
|
1357
|
+
catch (error) {
|
|
1358
|
+
try {
|
|
1359
|
+
const rollback = durableRollback;
|
|
1360
|
+
if (rollback)
|
|
1361
|
+
rollback();
|
|
1362
|
+
}
|
|
1363
|
+
catch { /* preserve the original failure */ }
|
|
1364
|
+
restoreRuntime();
|
|
1365
|
+
compensateProjection();
|
|
1366
|
+
await cleanupFresh("context-window recovery not committed");
|
|
1367
|
+
if (error instanceof RuntimeSessionRecoveryError)
|
|
1368
|
+
throw error;
|
|
1369
|
+
if (typeof error.code === "string") {
|
|
1370
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", error instanceof Error ? error.message : "context-window recovery was refused");
|
|
1371
|
+
}
|
|
1372
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "context-window recovery was not committed");
|
|
1373
|
+
}
|
|
1374
|
+
},
|
|
1375
|
+
async recoverContextOverflow(agentId, deliveryKey, reason) {
|
|
1376
|
+
return recoverContextOverflowInternal(agentId, deliveryKey, reason);
|
|
1377
|
+
},
|
|
792
1378
|
async start(configs) {
|
|
793
1379
|
let activeCount = 0;
|
|
794
1380
|
let recoveringCount = 0;
|
|
@@ -807,7 +1393,14 @@ function createRuntimeHost(options) {
|
|
|
807
1393
|
const deliveryFile = stateStore.readJson("runtimeDeliveries", { version: 1, records: [] });
|
|
808
1394
|
if (!stateStore.readNdjson)
|
|
809
1395
|
return deliveryFile;
|
|
810
|
-
|
|
1396
|
+
let inboxRows;
|
|
1397
|
+
try {
|
|
1398
|
+
inboxRows = stateStore.readNdjson("inbox");
|
|
1399
|
+
}
|
|
1400
|
+
catch {
|
|
1401
|
+
return deliveryFile;
|
|
1402
|
+
}
|
|
1403
|
+
const inboxIds = new Set(inboxRows.flatMap((row) => typeof row?.message_id === "string" ? [row.message_id] : []));
|
|
811
1404
|
let changed = false;
|
|
812
1405
|
const records = deliveryFile.records.map((record) => {
|
|
813
1406
|
const synthetic = /^(?:redeliver_|rem_|interaction_)/.test(record.messageId);
|
|
@@ -828,15 +1421,44 @@ function createRuntimeHost(options) {
|
|
|
828
1421
|
byMessage: new Map(persisted.records.map((record) => [record.messageId, record.deliveryId])),
|
|
829
1422
|
generation: 0, poller: null, retryTimer: null, recreateAttempts: 0,
|
|
830
1423
|
stabilityTimer: null, recreateReason: null, stopped: false, disabledReason: null,
|
|
831
|
-
configurationRecovery: null, stateStore,
|
|
1424
|
+
configurationRecovery: null, stateStore,
|
|
1425
|
+
compactionBreaker: config.runtime === "pi"
|
|
1426
|
+
? new pi_compaction_recovery_js_1.PiCompactionBreaker(config.stateDir ?? node_path_1.default.join(config.workspaceDir, ".larkin", "agents", config.agentId), {
|
|
1427
|
+
withLock: stateStore ? (operation) => stateStore.withInboxTransaction(operation) : undefined,
|
|
1428
|
+
}) : undefined,
|
|
1429
|
+
compactionMachines: new Map(), piOverflowCompactionFailed: new Set(), readiness: null,
|
|
832
1430
|
turnInProgress: false, turnHadFailure: false, turnHadAuthenticatedOutput: false, authFailureActive: false };
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
1431
|
+
const startupConsumed = [];
|
|
1432
|
+
const startupQuarantined = [];
|
|
1433
|
+
for (const record of agent.records.values()) {
|
|
1434
|
+
const existingQuarantine = replayFailureCodeOf(record);
|
|
1435
|
+
if (existingQuarantine) {
|
|
1436
|
+
startupQuarantined.push({ record, code: existingQuarantine });
|
|
1437
|
+
continue;
|
|
1438
|
+
}
|
|
1439
|
+
if (!isActiveDelivery(record.status))
|
|
1440
|
+
continue;
|
|
1441
|
+
record.status = "pending";
|
|
1442
|
+
const prepared = prepareRecordInput(agent, record, false);
|
|
1443
|
+
if (prepared.status === "consumed") {
|
|
1444
|
+
record.status = "consumed";
|
|
1445
|
+
record.updatedAt = now();
|
|
1446
|
+
startupConsumed.push(record);
|
|
1447
|
+
}
|
|
1448
|
+
else if (prepared.status === "error") {
|
|
1449
|
+
record.status = "error";
|
|
1450
|
+
record.updatedAt = now();
|
|
1451
|
+
scrubQuarantinedInput(record, prepared.reason);
|
|
1452
|
+
record.reason = prepared.reason;
|
|
1453
|
+
record.retryable = false;
|
|
1454
|
+
startupQuarantined.push({ record, code: prepared.code });
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
837
1457
|
managed.set(config.agentId, agent);
|
|
1458
|
+
emitConsumed(agent, [...startupConsumed, ...persist(agent)]);
|
|
838
1459
|
try {
|
|
839
1460
|
await ensureSession(agent);
|
|
1461
|
+
await recoverStalePiCompaction(agent);
|
|
840
1462
|
activeCount += 1;
|
|
841
1463
|
await retryPending(agent);
|
|
842
1464
|
}
|
|
@@ -852,6 +1474,13 @@ function createRuntimeHost(options) {
|
|
|
852
1474
|
emit({ type: "agent-status", agentId: config.agentId, status: "error", error: reason,
|
|
853
1475
|
...(error instanceof runtime_readiness_js_1.RuntimePrerequisiteError ? { readiness: error.readiness } : {}) });
|
|
854
1476
|
}
|
|
1477
|
+
const visibleQuarantines = startupQuarantined.filter(({ record }) => agent.records.get(record.deliveryId)?.status === "error");
|
|
1478
|
+
for (const { record, code } of visibleQuarantines)
|
|
1479
|
+
emit({ type: "delivery", agentId: config.agentId,
|
|
1480
|
+
deliveryId: record.deliveryId, messageId: record.messageId, status: "error", reason: replayFailureReason(code) });
|
|
1481
|
+
if (visibleQuarantines.length)
|
|
1482
|
+
emit({ type: "agent-status", agentId: config.agentId, status: "error",
|
|
1483
|
+
error: `${visibleQuarantines.length} Runtime delivery record(s) quarantined; canonical Inbox recovery is required` });
|
|
855
1484
|
agent.poller = setInterval(() => reconcileExternalConsumption(agent), 250);
|
|
856
1485
|
agent.poller.unref?.();
|
|
857
1486
|
});
|
|
@@ -867,12 +1496,33 @@ function createRuntimeHost(options) {
|
|
|
867
1496
|
telemetry?.delivery(agentId, messageId, "error");
|
|
868
1497
|
throw new Error(`unknown runtime Agent: ${agentId}`);
|
|
869
1498
|
}
|
|
1499
|
+
let target;
|
|
1500
|
+
try {
|
|
1501
|
+
target = (0, inbox_projection_js_1.targetKeyOfInboxEnvelope)(envelope);
|
|
1502
|
+
}
|
|
1503
|
+
catch (error) {
|
|
1504
|
+
const reason = `Inbox target derivation failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
1505
|
+
const deliveryId = `invalid-target-${node_crypto_1.default.createHash("sha256").update(`${agentId}\0${messageId}`).digest("hex").slice(0, 24)}`;
|
|
1506
|
+
telemetry?.delivery(agentId, messageId, "error");
|
|
1507
|
+
emit({ type: "delivery", agentId, deliveryId, messageId, status: "error", reason });
|
|
1508
|
+
return { status: "error", deliveryId, reason, retryable: false };
|
|
1509
|
+
}
|
|
870
1510
|
reconcileExternalConsumption(agent);
|
|
871
1511
|
const existingId = agent.byMessage.get(messageId);
|
|
872
1512
|
if (existingId) {
|
|
873
1513
|
const existing = agent.records.get(existingId);
|
|
874
1514
|
if (existing?.status === "error") {
|
|
875
|
-
existing.
|
|
1515
|
+
if (existing.target && existing.target !== target)
|
|
1516
|
+
return quarantineRecord(agent, existing, "delivery_target_conflict");
|
|
1517
|
+
existing.target = target;
|
|
1518
|
+
if (typeof envelope.wake_reason === "string" && envelope.wake_reason)
|
|
1519
|
+
existing.wakeReason = envelope.wake_reason;
|
|
1520
|
+
else
|
|
1521
|
+
delete existing.wakeReason;
|
|
1522
|
+
const priorAttempt = existing.input && Number.isSafeInteger(existing.input.attempt) ? existing.input.attempt : 0;
|
|
1523
|
+
existing.input = existing.input && typeof existing.input === "object"
|
|
1524
|
+
? { ...existing.input, attempt: priorAttempt + 1 }
|
|
1525
|
+
: { inputId: existing.deliveryId, deliveryId: existing.deliveryId, kind: "wake", text: "", attempt: priorAttempt + 1 };
|
|
876
1526
|
delete existing.reason;
|
|
877
1527
|
delete existing.retryable;
|
|
878
1528
|
setRecord(agent, existing, "pending");
|
|
@@ -893,11 +1543,12 @@ function createRuntimeHost(options) {
|
|
|
893
1543
|
}
|
|
894
1544
|
const deliveryId = node_crypto_1.default.randomUUID();
|
|
895
1545
|
const busy = agent.busy || agent.submitting;
|
|
1546
|
+
const wakeReason = typeof envelope.wake_reason === "string" ? envelope.wake_reason : undefined;
|
|
896
1547
|
const input = { inputId: deliveryId, deliveryId, kind: busy ? "inbox_update" : "wake",
|
|
897
|
-
text: options.promptBuilder.buildInboxNotice({ busy, count: 1, deliveryId,
|
|
898
|
-
...(
|
|
899
|
-
|
|
900
|
-
|
|
1548
|
+
text: options.promptBuilder.buildInboxNotice({ busy, count: 1, deliveryId, target,
|
|
1549
|
+
...(wakeReason ? { wakeReason } : {}) }), attempt: 0 };
|
|
1550
|
+
const record = { deliveryId, messageId, status: "pending", input, target,
|
|
1551
|
+
...(wakeReason ? { wakeReason } : {}), updatedAt: now() };
|
|
901
1552
|
agent.records.set(deliveryId, record);
|
|
902
1553
|
agent.byMessage.set(messageId, deliveryId);
|
|
903
1554
|
persist(agent);
|