larkin 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -33
- package/assets/readme/larkin-hero.svg +111 -0
- package/dist/agent/agent-state-store.cjs +194 -1
- package/dist/agent/agent-state-store.mjs +192 -0
- package/dist/app/cli.mjs +3 -1
- package/dist/app/local-control.mjs +108 -2
- package/dist/app/runtime-process.mjs +4 -0
- package/dist/app/session-cli.mjs +55 -20
- package/dist/feishu/host-shell.cjs +72 -1
- package/dist/feishu/host-shell.mjs +72 -1
- package/dist/runtime/pi-compaction-recovery.cjs +623 -0
- package/dist/runtime/pi-compaction-recovery.mjs +602 -0
- package/dist/runtime/pi-provider-config.cjs +2 -2
- package/dist/runtime/pi-provider-config.mjs +2 -2
- package/dist/runtime/pi-rpc-client.cjs +39 -22
- package/dist/runtime/pi-rpc-client.mjs +39 -22
- package/dist/runtime/provider-error-classifier.cjs +39 -0
- package/dist/runtime/provider-error-classifier.mjs +34 -0
- package/dist/runtime/runtime-adapters.mjs +82 -19
- package/dist/runtime/runtime-host.cjs +490 -2
- package/dist/runtime/runtime-host.mjs +488 -1
- package/package.json +1 -1
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.RuntimeSessionResetError = void 0;
|
|
6
|
+
exports.RuntimeSessionRecoveryError = exports.RuntimeSessionResetError = void 0;
|
|
7
7
|
exports.createRuntimeHost = createRuntimeHost;
|
|
8
8
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
9
9
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
@@ -11,9 +11,11 @@ const node_path_1 = __importDefault(require("node:path"));
|
|
|
11
11
|
const agent_cli_capabilities_js_1 = require("../agent/agent-cli-capabilities.cjs");
|
|
12
12
|
const inbox_projection_js_1 = require("../agent/inbox-projection.cjs");
|
|
13
13
|
const api_1 = require("@opentelemetry/api");
|
|
14
|
+
const provider_error_classifier_js_1 = require("./provider-error-classifier.cjs");
|
|
14
15
|
const runtime_readiness_js_1 = require("./runtime-readiness.cjs");
|
|
15
16
|
const official_lark_cli_js_1 = require("../app/official-lark-cli.cjs");
|
|
16
17
|
const agent_lark_cli_workspace_js_1 = require("../app/agent-lark-cli-workspace.cjs");
|
|
18
|
+
const pi_compaction_recovery_js_1 = require("./pi-compaction-recovery.cjs");
|
|
17
19
|
class RuntimeSessionResetError extends Error {
|
|
18
20
|
code;
|
|
19
21
|
pendingCount;
|
|
@@ -25,6 +27,17 @@ class RuntimeSessionResetError extends Error {
|
|
|
25
27
|
}
|
|
26
28
|
}
|
|
27
29
|
exports.RuntimeSessionResetError = RuntimeSessionResetError;
|
|
30
|
+
class RuntimeSessionRecoveryError extends Error {
|
|
31
|
+
code;
|
|
32
|
+
pendingCount;
|
|
33
|
+
constructor(code, message, pendingCount = 0) {
|
|
34
|
+
super(message);
|
|
35
|
+
this.code = code;
|
|
36
|
+
this.pendingCount = pendingCount;
|
|
37
|
+
this.name = "RuntimeSessionRecoveryError";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
exports.RuntimeSessionRecoveryError = RuntimeSessionRecoveryError;
|
|
28
41
|
const MAX_DELIVERIES = 2048;
|
|
29
42
|
const now = () => new Date().toISOString();
|
|
30
43
|
const isActiveDelivery = (status) => ["pending", "submitting", "accepted"].includes(status);
|
|
@@ -114,6 +127,9 @@ function createRuntimeHost(options) {
|
|
|
114
127
|
const listeners = new Set();
|
|
115
128
|
const log = options.log ?? (() => { });
|
|
116
129
|
const telemetry = options.telemetry;
|
|
130
|
+
const compactTimeoutMs = options.compactTimeoutMs ?? 120_000;
|
|
131
|
+
if (!Number.isFinite(compactTimeoutMs) || compactTimeoutMs <= 0)
|
|
132
|
+
throw new Error("compactTimeoutMs must be positive");
|
|
117
133
|
const retryPolicy = {
|
|
118
134
|
baseDelayMs: options.retryPolicy?.baseDelayMs ?? 250,
|
|
119
135
|
maxDelayMs: options.retryPolicy?.maxDelayMs ?? 10_000,
|
|
@@ -242,6 +258,7 @@ function createRuntimeHost(options) {
|
|
|
242
258
|
scrubQuarantinedInput(record, reason);
|
|
243
259
|
record.reason = reason;
|
|
244
260
|
record.retryable = false;
|
|
261
|
+
delete record.errorCategory;
|
|
245
262
|
const finalRecord = setRecord(agent, record, "error");
|
|
246
263
|
if (finalRecord.status === "consumed")
|
|
247
264
|
return { status: "accepted", deliveryId: record.deliveryId };
|
|
@@ -382,12 +399,14 @@ function createRuntimeHost(options) {
|
|
|
382
399
|
if (result.status === "accepted") {
|
|
383
400
|
delete record.reason;
|
|
384
401
|
delete record.retryable;
|
|
402
|
+
delete record.errorCategory;
|
|
385
403
|
const finalRecord = setRecord(agent, record, "accepted");
|
|
386
404
|
if (finalRecord.status !== "consumed")
|
|
387
405
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
388
406
|
deliveryId: record.deliveryId, messageId: record.messageId, status: "accepted" });
|
|
389
407
|
return { status: "accepted", deliveryId: record.deliveryId };
|
|
390
408
|
}
|
|
409
|
+
delete record.errorCategory;
|
|
391
410
|
const retryable = result.status === "deferred" || result.retryable;
|
|
392
411
|
record.reason = result.reason;
|
|
393
412
|
record.retryable = retryable;
|
|
@@ -442,6 +461,7 @@ function createRuntimeHost(options) {
|
|
|
442
461
|
const reason = error instanceof Error ? error.message : String(error);
|
|
443
462
|
record.reason = reason;
|
|
444
463
|
record.retryable = true;
|
|
464
|
+
delete record.errorCategory;
|
|
445
465
|
setRecord(agent, record, "pending");
|
|
446
466
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId, messageId: record.messageId, status: "deferred", reason });
|
|
447
467
|
return { status: "deferred", deliveryId: record.deliveryId, reason };
|
|
@@ -568,6 +588,106 @@ function createRuntimeHost(options) {
|
|
|
568
588
|
})();
|
|
569
589
|
agent.configurationRecovery = recovery;
|
|
570
590
|
};
|
|
591
|
+
let recoverContextOverflowInternal;
|
|
592
|
+
const beginPiManualCompaction = (agent, session, record, nativeWillRetry = false) => {
|
|
593
|
+
if (agent.adapter.id !== "pi" || !session.compact || !agent.compactionBreaker)
|
|
594
|
+
return;
|
|
595
|
+
const existing = agent.compactionMachines.get(record.input.inputId);
|
|
596
|
+
if (existing) {
|
|
597
|
+
existing.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
const key = `${record.deliveryId}:${record.input.inputId}`;
|
|
601
|
+
const prior = agent.compactionBreaker.get(key);
|
|
602
|
+
if (prior && prior.manualAttempt >= 1 && ["manual_sent", "manual_ambiguous", "manual_failed", "native_failed"].includes(prior.state)) {
|
|
603
|
+
agent.busy = false;
|
|
604
|
+
agent.turnInProgress = false;
|
|
605
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, "validated Pi compaction breaker state after restart", key)
|
|
606
|
+
.catch((error) => log("Pi restart fallback failed", String(error)));
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
const machine = new pi_compaction_recovery_js_1.PiCompactionRecoveryMachine({
|
|
610
|
+
breaker: agent.compactionBreaker, key, messageId: record.messageId, deliveryId: record.deliveryId,
|
|
611
|
+
inputId: record.input.inputId, sessionGeneration: agent.generation,
|
|
612
|
+
onAction: (action) => {
|
|
613
|
+
if (action === "retry_input") {
|
|
614
|
+
machine.retrySubmitted();
|
|
615
|
+
record.status = "pending";
|
|
616
|
+
record.retryable = true;
|
|
617
|
+
record.reason = "Pi manual compaction succeeded; retrying the same delivery input";
|
|
618
|
+
record.updatedAt = now();
|
|
619
|
+
persist(agent);
|
|
620
|
+
queueMicrotask(() => { if (!agent.stopped)
|
|
621
|
+
void submit(agent, record, false); });
|
|
622
|
+
}
|
|
623
|
+
else if (action === "fresh_session_fallback") {
|
|
624
|
+
agent.busy = false;
|
|
625
|
+
agent.turnInProgress = false;
|
|
626
|
+
if (record.status !== "error") {
|
|
627
|
+
record.reason = machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback";
|
|
628
|
+
record.retryable = false;
|
|
629
|
+
record.errorCategory = "context_window";
|
|
630
|
+
setRecord(agent, record, "error");
|
|
631
|
+
}
|
|
632
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback", key)
|
|
633
|
+
.catch((error) => log("Pi context fallback failed", String(error)));
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
agent.compactionMachines.set(record.input.inputId, machine);
|
|
638
|
+
machine.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
639
|
+
if (nativeWillRetry)
|
|
640
|
+
return;
|
|
641
|
+
machine.agentSettled();
|
|
642
|
+
const deadline = new Date(Date.now() + compactTimeoutMs).toISOString();
|
|
643
|
+
machine.manualCompactSent(deadline);
|
|
644
|
+
let compactSettled = false;
|
|
645
|
+
const compactTimer = setTimeout(() => {
|
|
646
|
+
if (compactSettled)
|
|
647
|
+
return;
|
|
648
|
+
compactSettled = true;
|
|
649
|
+
machine.compactTimeout();
|
|
650
|
+
}, compactTimeoutMs);
|
|
651
|
+
compactTimer.unref?.();
|
|
652
|
+
void session.compact().then(() => {
|
|
653
|
+
if (compactSettled)
|
|
654
|
+
return;
|
|
655
|
+
compactSettled = true;
|
|
656
|
+
clearTimeout(compactTimer);
|
|
657
|
+
machine.compactResponse({ success: true });
|
|
658
|
+
}).catch((error) => {
|
|
659
|
+
if (compactSettled)
|
|
660
|
+
return;
|
|
661
|
+
compactSettled = true;
|
|
662
|
+
clearTimeout(compactTimer);
|
|
663
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
664
|
+
machine.compactResponse({ success: false, ambiguous: /process exited|process failed|session replaced|protocol/i.test(message) });
|
|
665
|
+
});
|
|
666
|
+
};
|
|
667
|
+
const recoverStalePiCompaction = async (agent) => {
|
|
668
|
+
const breaker = agent.compactionBreaker;
|
|
669
|
+
if (!breaker)
|
|
670
|
+
return;
|
|
671
|
+
const stale = breaker.listNonTerminal().find((candidate) => candidate.state !== "native_succeeded");
|
|
672
|
+
if (!stale) {
|
|
673
|
+
const succeeded = breaker.listNonTerminal().find((candidate) => candidate.state === "native_succeeded");
|
|
674
|
+
if (succeeded)
|
|
675
|
+
breaker.transition(succeeded.key, {}, "closed");
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
678
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === stale.deliveryId
|
|
679
|
+
|| candidate.input.inputId === stale.inputId);
|
|
680
|
+
if (!record)
|
|
681
|
+
return;
|
|
682
|
+
record.reason = stale.fallbackReason || "Pi compaction recovery resumed after restart";
|
|
683
|
+
record.retryable = false;
|
|
684
|
+
record.errorCategory = "context_window";
|
|
685
|
+
setRecord(agent, record, "error");
|
|
686
|
+
breaker.forceFallback(stale.key, record.reason);
|
|
687
|
+
agent.busy = false;
|
|
688
|
+
agent.turnInProgress = false;
|
|
689
|
+
await recoverContextOverflowInternal(agent.config.agentId, stale.key, record.reason, stale.key);
|
|
690
|
+
};
|
|
571
691
|
const observe = (agent, session, event) => {
|
|
572
692
|
if (agent.session !== session)
|
|
573
693
|
return; // Ignore late output from a replaced child.
|
|
@@ -576,6 +696,34 @@ function createRuntimeHost(options) {
|
|
|
576
696
|
if (event.type !== "turn-end")
|
|
577
697
|
telemetry?.runtimeEvent(agent.config.agentId, event);
|
|
578
698
|
emit({ type: "runtime", agentId: agent.config.agentId, event });
|
|
699
|
+
if (event.type === "runtime-observation") {
|
|
700
|
+
if (event.phase === "compaction_start" && event.reason === "overflow" && event.inputId) {
|
|
701
|
+
const overflowRecord = [...agent.records.values()].find((candidate) => candidate.input.inputId === event.inputId);
|
|
702
|
+
if (overflowRecord && !agent.compactionMachines.has(event.inputId))
|
|
703
|
+
beginPiManualCompaction(agent, session, overflowRecord, true);
|
|
704
|
+
}
|
|
705
|
+
const candidates = [...agent.compactionMachines.values()].filter((machine) => machine.recordSnapshot.sessionGeneration === agent.generation
|
|
706
|
+
&& (!event.inputId || machine.recordSnapshot.inputId === event.inputId));
|
|
707
|
+
const machines = event.inputId ? candidates : candidates.length === 1 ? candidates : [];
|
|
708
|
+
for (const machine of machines) {
|
|
709
|
+
if (event.phase === "compaction_start" && event.reason === "overflow")
|
|
710
|
+
machine.compactionStart({ reason: "overflow" });
|
|
711
|
+
if (event.phase === "compaction_end" && event.reason === "manual") {
|
|
712
|
+
machine.compactionEnd({ reason: "manual", success: event.success === true, willRetry: event.willRetry === true });
|
|
713
|
+
}
|
|
714
|
+
if (event.phase === "compaction_end" && event.reason === "overflow") {
|
|
715
|
+
machine.compactionEnd({ reason: "overflow", success: event.success === true, willRetry: event.willRetry === true });
|
|
716
|
+
}
|
|
717
|
+
if (event.phase === "settled")
|
|
718
|
+
machine.agentSettled();
|
|
719
|
+
}
|
|
720
|
+
if (event.phase === "compaction_end" && event.reason === "overflow" && event.success !== true) {
|
|
721
|
+
for (const record of agent.records.values()) {
|
|
722
|
+
if (isActiveDelivery(record.status))
|
|
723
|
+
agent.piOverflowCompactionFailed.add(record.input.inputId);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
}
|
|
579
727
|
if (event.type === "session-init") {
|
|
580
728
|
agent.config.sessionId = event.sessionId;
|
|
581
729
|
emit({ type: "session", agentId: agent.config.agentId, runtime: agent.adapter.id, sessionId: event.sessionId, launchId: agent.launchId,
|
|
@@ -591,6 +739,13 @@ function createRuntimeHost(options) {
|
|
|
591
739
|
emit({ type: "activity", agentId: agent.config.agentId, activity: "working", activityKind: "working", detailKind: "turn_started" });
|
|
592
740
|
}
|
|
593
741
|
else if (event.type === "turn-end") {
|
|
742
|
+
for (const [inputId, machine] of agent.compactionMachines) {
|
|
743
|
+
if (machine.recordSnapshot.sessionGeneration !== agent.generation)
|
|
744
|
+
continue;
|
|
745
|
+
machine.close();
|
|
746
|
+
if (["closed", "fallback_committed"].includes(machine.state))
|
|
747
|
+
agent.compactionMachines.delete(inputId);
|
|
748
|
+
}
|
|
594
749
|
const recoveredAuthentication = agent.authFailureActive && agent.turnInProgress
|
|
595
750
|
&& agent.turnHadAuthenticatedOutput && !agent.turnHadFailure;
|
|
596
751
|
agent.turnInProgress = false;
|
|
@@ -623,6 +778,29 @@ function createRuntimeHost(options) {
|
|
|
623
778
|
if (!record || record.status === "consumed" || record.status === "error")
|
|
624
779
|
return;
|
|
625
780
|
agent.turnHadFailure = true;
|
|
781
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry === true && session.compact) {
|
|
782
|
+
beginPiManualCompaction(agent, session, record, true);
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry !== true && session.compact) {
|
|
786
|
+
record.reason = event.message;
|
|
787
|
+
record.retryable = false;
|
|
788
|
+
record.errorCategory = "context_window";
|
|
789
|
+
setRecord(agent, record, "error");
|
|
790
|
+
if (agent.piOverflowCompactionFailed.has(record.input.inputId)) {
|
|
791
|
+
agent.busy = false;
|
|
792
|
+
agent.turnInProgress = false;
|
|
793
|
+
void recoverContextOverflowInternal(agent.config.agentId, `${record.deliveryId}:${record.input.inputId}`, "native Pi overflow compaction failed", `${record.deliveryId}:${record.input.inputId}`)
|
|
794
|
+
.catch((error) => log("Pi native compaction fallback failed", String(error)));
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
// Pi emits this terminal error only after agent_settled; release the
|
|
798
|
+
// normal turn reservation before the one authorized recovery action.
|
|
799
|
+
agent.busy = false;
|
|
800
|
+
agent.turnInProgress = false;
|
|
801
|
+
beginPiManualCompaction(agent, session, record);
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
626
804
|
if (event.willRetry) {
|
|
627
805
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
|
|
628
806
|
messageId: record.messageId, status: "deferred", reason: event.message });
|
|
@@ -633,6 +811,13 @@ function createRuntimeHost(options) {
|
|
|
633
811
|
agent.busy = false;
|
|
634
812
|
record.reason = event.message;
|
|
635
813
|
record.retryable = event.retryable;
|
|
814
|
+
const classifiedCategory = event.errorCategory ?? (0, provider_error_classifier_js_1.classifyStrictProviderError)((0, provider_error_classifier_js_1.buildStrictProviderErrorInput)({
|
|
815
|
+
message: event.message, errorCategory: event.errorCategory, upstream: event.upstream,
|
|
816
|
+
}));
|
|
817
|
+
if (classifiedCategory)
|
|
818
|
+
record.errorCategory = classifiedCategory;
|
|
819
|
+
else
|
|
820
|
+
delete record.errorCategory;
|
|
636
821
|
const finalRecord = setRecord(agent, record, event.retryable ? "pending" : "error");
|
|
637
822
|
if (finalRecord.status !== "consumed")
|
|
638
823
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
@@ -719,6 +904,129 @@ function createRuntimeHost(options) {
|
|
|
719
904
|
});
|
|
720
905
|
return agent.starting;
|
|
721
906
|
};
|
|
907
|
+
recoverContextOverflowInternal = async (agentId, deliveryKey, reason, breakerKey) => {
|
|
908
|
+
const agent = managed.get(agentId);
|
|
909
|
+
if (!agent || agent.stopped)
|
|
910
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
911
|
+
if (agent.adapter.id !== "pi")
|
|
912
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "Pi context fallback is unavailable for this runtime");
|
|
913
|
+
const oldSession = agent.session;
|
|
914
|
+
if (!oldSession || agent.busy || agent.submitting || agent.starting) {
|
|
915
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle for Pi context fallback`);
|
|
916
|
+
}
|
|
917
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === deliveryKey || candidate.input.inputId === deliveryKey || `${candidate.deliveryId}:${candidate.input.inputId}` === deliveryKey);
|
|
918
|
+
if (!record)
|
|
919
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "Pi context fallback delivery identity is unavailable");
|
|
920
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:pi-context-fallback`);
|
|
921
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
922
|
+
const readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
923
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir, env: probeEnv }) : { runtime: "pi", state: "ready" };
|
|
924
|
+
if (readiness.state !== "ready")
|
|
925
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback readiness failed");
|
|
926
|
+
const standingPrompt = options.promptBuilder.build({
|
|
927
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
928
|
+
description: agent.config.description || "", runtime: agent.adapter.id, cli: (0, agent_cli_capabilities_js_1.agentCliPromptCapabilities)("larkin"),
|
|
929
|
+
});
|
|
930
|
+
let fresh;
|
|
931
|
+
try {
|
|
932
|
+
fresh = await agent.adapter.createSession({ agentId: agent.config.agentId, model: agent.config.model,
|
|
933
|
+
reasoningEffort: agent.config.effort || null, workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
934
|
+
resumeSessionId: null, standingPrompt, env: probeEnv });
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi fresh fallback session could not be staged");
|
|
938
|
+
}
|
|
939
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
940
|
+
await fresh.close("Pi context fallback commit precondition changed").catch(() => { });
|
|
941
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
942
|
+
}
|
|
943
|
+
let unsubscribeFresh = null;
|
|
944
|
+
try {
|
|
945
|
+
unsubscribeFresh = fresh.subscribe((next) => observe(agent, fresh, next));
|
|
946
|
+
}
|
|
947
|
+
catch {
|
|
948
|
+
await fresh.close("Pi context fallback subscription failed").catch(() => { });
|
|
949
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback subscription failed");
|
|
950
|
+
}
|
|
951
|
+
const oldSessionId = oldSession.sessionId;
|
|
952
|
+
const oldGeneration = agent.generation;
|
|
953
|
+
const oldLaunchId = agent.launchId;
|
|
954
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
955
|
+
const oldReadiness = agent.readiness;
|
|
956
|
+
const oldDisabledReason = agent.disabledReason;
|
|
957
|
+
const oldRecord = { ...record, input: { ...record.input } };
|
|
958
|
+
let durableRollback = null;
|
|
959
|
+
let rearmedCount = 1;
|
|
960
|
+
let remainingPendingCount = 1;
|
|
961
|
+
const commitSwap = (messageIds, rollback) => {
|
|
962
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
963
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
964
|
+
}
|
|
965
|
+
durableRollback = rollback ?? null;
|
|
966
|
+
if (breakerKey && agent.compactionBreaker)
|
|
967
|
+
agent.compactionBreaker.forceFallbackInTransaction(breakerKey, reason);
|
|
968
|
+
agent.generation += 1;
|
|
969
|
+
agent.launchId = node_crypto_1.default.randomUUID();
|
|
970
|
+
agent.config.sessionId = null;
|
|
971
|
+
agent.session = fresh;
|
|
972
|
+
agent.readiness = readiness;
|
|
973
|
+
agent.disabledReason = null;
|
|
974
|
+
for (const candidate of agent.records.values()) {
|
|
975
|
+
if (!messageIds.includes(candidate.messageId))
|
|
976
|
+
continue;
|
|
977
|
+
candidate.status = "pending";
|
|
978
|
+
candidate.retryable = true;
|
|
979
|
+
candidate.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
980
|
+
candidate.updatedAt = now();
|
|
981
|
+
}
|
|
982
|
+
};
|
|
983
|
+
try {
|
|
984
|
+
if (agent.stateStore?.rearmContextOverflow) {
|
|
985
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => commitSwap(messageIds, rollback), {
|
|
986
|
+
messageId: record.messageId, deliveryId: record.deliveryId, inputId: record.input.inputId,
|
|
987
|
+
});
|
|
988
|
+
rearmedCount = rearmed.rearmedCount;
|
|
989
|
+
remainingPendingCount = rearmed.remainingPendingCount;
|
|
990
|
+
}
|
|
991
|
+
else {
|
|
992
|
+
commitSwap([record.messageId]);
|
|
993
|
+
persist(agent);
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
catch (error) {
|
|
997
|
+
try {
|
|
998
|
+
const rollback = durableRollback;
|
|
999
|
+
if (rollback)
|
|
1000
|
+
rollback();
|
|
1001
|
+
}
|
|
1002
|
+
catch { /* preserve primary error */ }
|
|
1003
|
+
agent.generation = oldGeneration;
|
|
1004
|
+
agent.launchId = oldLaunchId;
|
|
1005
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
1006
|
+
agent.session = oldSession;
|
|
1007
|
+
agent.readiness = oldReadiness;
|
|
1008
|
+
agent.disabledReason = oldDisabledReason;
|
|
1009
|
+
Object.assign(record, oldRecord);
|
|
1010
|
+
try {
|
|
1011
|
+
unsubscribeFresh?.();
|
|
1012
|
+
}
|
|
1013
|
+
catch { /* preserve primary failure */ }
|
|
1014
|
+
await fresh.close("Pi context fallback rolled back").catch(() => { });
|
|
1015
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", error instanceof Error ? error.message : "Pi context fallback commit failed");
|
|
1016
|
+
}
|
|
1017
|
+
if (fresh.sessionId) {
|
|
1018
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1019
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1020
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1021
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1022
|
+
}
|
|
1023
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1024
|
+
await oldSession.close("Pi context fallback committed").catch((error) => log("previous Pi session close after fallback failed", String(error)));
|
|
1025
|
+
void retryPending(agent);
|
|
1026
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1027
|
+
runtimeReady: true, pendingCount: remainingPendingCount, rearmedCount,
|
|
1028
|
+
replayStatus: remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1029
|
+
};
|
|
722
1030
|
return {
|
|
723
1031
|
async probe(config) {
|
|
724
1032
|
const adapter = options.adapterFor(config.runtime);
|
|
@@ -893,6 +1201,180 @@ function createRuntimeHost(options) {
|
|
|
893
1201
|
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
894
1202
|
runtimeReady: true, pendingCount: 0, sessionId: fresh.sessionId };
|
|
895
1203
|
},
|
|
1204
|
+
async recoverSession(agentId, reason) {
|
|
1205
|
+
if (reason !== "context-overflow")
|
|
1206
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "unsupported recovery reason");
|
|
1207
|
+
const agent = managed.get(agentId);
|
|
1208
|
+
if (agent && agent.adapter.id !== "pi")
|
|
1209
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "context-overflow recovery is available only for Pi Runtime");
|
|
1210
|
+
if (!agent || agent.stopped)
|
|
1211
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
1212
|
+
if (!agent.stateStore?.rearmContextOverflow) {
|
|
1213
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "canonical Inbox recovery is unavailable");
|
|
1214
|
+
}
|
|
1215
|
+
if (agent.busy || agent.turnInProgress || agent.submitting || agent.starting) {
|
|
1216
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle`);
|
|
1217
|
+
}
|
|
1218
|
+
const oldSession = agent.session;
|
|
1219
|
+
if (!oldSession)
|
|
1220
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} Runtime session is unavailable`);
|
|
1221
|
+
const oldSessionId = oldSession.sessionId;
|
|
1222
|
+
if (agent.stateStore.readNdjson) {
|
|
1223
|
+
const pendingRows = agent.stateStore.withInboxTransaction(() => agent.stateStore.readNdjson("inbox"));
|
|
1224
|
+
if (pendingRows.length === 0) {
|
|
1225
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery has no retained Inbox backlog");
|
|
1226
|
+
}
|
|
1227
|
+
const messageIds = new Set(pendingRows.flatMap((row) => {
|
|
1228
|
+
const messageId = row && typeof row === "object" && !Array.isArray(row)
|
|
1229
|
+
? row.message_id : undefined;
|
|
1230
|
+
return typeof messageId === "string" && messageId ? [messageId] : [];
|
|
1231
|
+
}));
|
|
1232
|
+
const persisted = agent.stateStore.readJson("runtimeDeliveries", { version: 1, records: [] });
|
|
1233
|
+
if (Array.isArray(persisted.records) && persisted.records.some((record) => messageIds.has(record.messageId) && record.status !== "error")) {
|
|
1234
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery is already staged or completed");
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:context-overflow-recovery`);
|
|
1238
|
+
let readiness;
|
|
1239
|
+
try {
|
|
1240
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
1241
|
+
readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
1242
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1243
|
+
env: { LARKIN_PI_COMMAND: process.env.LARKIN_PI_COMMAND, LARKIN_CODEX_COMMAND: process.env.LARKIN_CODEX_COMMAND,
|
|
1244
|
+
LARKIN_CLAUDE_COMMAND: process.env.LARKIN_CLAUDE_COMMAND, ...probeEnv } })
|
|
1245
|
+
: { runtime: agent.adapter.id, state: "ready" };
|
|
1246
|
+
}
|
|
1247
|
+
catch (error) {
|
|
1248
|
+
if (error instanceof runtime_readiness_js_1.RuntimePrerequisiteError)
|
|
1249
|
+
throw error;
|
|
1250
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Runtime readiness probe failed");
|
|
1251
|
+
}
|
|
1252
|
+
if (readiness.state !== "ready")
|
|
1253
|
+
throw new runtime_readiness_js_1.RuntimePrerequisiteError(readiness);
|
|
1254
|
+
const standingPrompt = options.promptBuilder.build({
|
|
1255
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
1256
|
+
description: agent.config.description || "", runtime: agent.adapter.id,
|
|
1257
|
+
cli: (0, agent_cli_capabilities_js_1.agentCliPromptCapabilities)("larkin"),
|
|
1258
|
+
});
|
|
1259
|
+
let fresh;
|
|
1260
|
+
try {
|
|
1261
|
+
fresh = await agent.adapter.createSession({
|
|
1262
|
+
agentId: agent.config.agentId, model: agent.config.model, reasoningEffort: agent.config.effort || null,
|
|
1263
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1264
|
+
resumeSessionId: null, standingPrompt, env: probeEnv,
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
catch {
|
|
1268
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session could not be staged");
|
|
1269
|
+
}
|
|
1270
|
+
let unsubscribeFresh = null;
|
|
1271
|
+
try {
|
|
1272
|
+
unsubscribeFresh = fresh.subscribe((event) => observe(agent, fresh, event));
|
|
1273
|
+
}
|
|
1274
|
+
catch {
|
|
1275
|
+
await fresh.close("context-window recovery subscription failed").catch(() => { });
|
|
1276
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session subscription failed");
|
|
1277
|
+
}
|
|
1278
|
+
const oldGeneration = agent.generation;
|
|
1279
|
+
const oldLaunchId = agent.launchId;
|
|
1280
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
1281
|
+
const oldReadiness = agent.readiness;
|
|
1282
|
+
const oldDisabledReason = agent.disabledReason;
|
|
1283
|
+
const oldRecords = new Map([...agent.records.entries()].map(([id, record]) => [id, { ...record, input: { ...record.input } }]));
|
|
1284
|
+
const cleanupFresh = async (reason) => {
|
|
1285
|
+
const unsubscribe = unsubscribeFresh;
|
|
1286
|
+
unsubscribeFresh = null;
|
|
1287
|
+
try {
|
|
1288
|
+
unsubscribe?.();
|
|
1289
|
+
}
|
|
1290
|
+
catch { /* close still owns staged-session cleanup */ }
|
|
1291
|
+
await fresh.close(reason).catch(() => { });
|
|
1292
|
+
};
|
|
1293
|
+
const restoreRuntime = () => {
|
|
1294
|
+
agent.session = oldSession;
|
|
1295
|
+
agent.generation = oldGeneration;
|
|
1296
|
+
agent.launchId = oldLaunchId;
|
|
1297
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
1298
|
+
agent.readiness = oldReadiness;
|
|
1299
|
+
agent.disabledReason = oldDisabledReason;
|
|
1300
|
+
agent.records = new Map(oldRecords);
|
|
1301
|
+
agent.byMessage = new Map([...oldRecords.values()].map((record) => [record.messageId, record.deliveryId]));
|
|
1302
|
+
};
|
|
1303
|
+
let recoveryProjectionEmitted = false;
|
|
1304
|
+
const compensateProjection = () => {
|
|
1305
|
+
if (!recoveryProjectionEmitted)
|
|
1306
|
+
return;
|
|
1307
|
+
try {
|
|
1308
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: oldSessionId, launchId: oldLaunchId });
|
|
1309
|
+
}
|
|
1310
|
+
catch { /* preserve the original listener failure */ }
|
|
1311
|
+
try {
|
|
1312
|
+
emit({ type: "agent-status", agentId, status: "active", ...(oldReadiness ? { readiness: oldReadiness } : {}) });
|
|
1313
|
+
}
|
|
1314
|
+
catch { /* preserve the original listener failure */ }
|
|
1315
|
+
};
|
|
1316
|
+
let durableRollback = null;
|
|
1317
|
+
try {
|
|
1318
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => {
|
|
1319
|
+
// This callback runs under the canonical Inbox lock. All potentially
|
|
1320
|
+
// racing checks above are synchronous from this process, while the
|
|
1321
|
+
// state-store method rechecks every row and delivery on disk.
|
|
1322
|
+
if (agent.session !== oldSession || agent.busy || agent.turnInProgress || agent.submitting || agent.starting || agent.stopped) {
|
|
1323
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during recovery`);
|
|
1324
|
+
}
|
|
1325
|
+
durableRollback = rollback;
|
|
1326
|
+
agent.generation += 1;
|
|
1327
|
+
agent.launchId = node_crypto_1.default.randomUUID();
|
|
1328
|
+
agent.config.sessionId = null;
|
|
1329
|
+
agent.session = fresh;
|
|
1330
|
+
agent.readiness = readiness;
|
|
1331
|
+
agent.disabledReason = null;
|
|
1332
|
+
for (const record of agent.records.values()) {
|
|
1333
|
+
if (!messageIds.includes(record.messageId))
|
|
1334
|
+
continue;
|
|
1335
|
+
record.status = "pending";
|
|
1336
|
+
record.retryable = true;
|
|
1337
|
+
record.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
1338
|
+
record.updatedAt = now();
|
|
1339
|
+
}
|
|
1340
|
+
});
|
|
1341
|
+
recoveryProjectionEmitted = true;
|
|
1342
|
+
if (fresh.sessionId) {
|
|
1343
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1344
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1345
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1346
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1347
|
+
}
|
|
1348
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1349
|
+
durableRollback = null;
|
|
1350
|
+
await oldSession.close("context-window recovery committed")
|
|
1351
|
+
.catch((error) => log("previous runtime close after context-window recovery failed", String(error)));
|
|
1352
|
+
void retryPending(agent);
|
|
1353
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1354
|
+
runtimeReady: true, pendingCount: rearmed.remainingPendingCount, rearmedCount: rearmed.rearmedCount,
|
|
1355
|
+
replayStatus: rearmed.remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1356
|
+
}
|
|
1357
|
+
catch (error) {
|
|
1358
|
+
try {
|
|
1359
|
+
const rollback = durableRollback;
|
|
1360
|
+
if (rollback)
|
|
1361
|
+
rollback();
|
|
1362
|
+
}
|
|
1363
|
+
catch { /* preserve the original failure */ }
|
|
1364
|
+
restoreRuntime();
|
|
1365
|
+
compensateProjection();
|
|
1366
|
+
await cleanupFresh("context-window recovery not committed");
|
|
1367
|
+
if (error instanceof RuntimeSessionRecoveryError)
|
|
1368
|
+
throw error;
|
|
1369
|
+
if (typeof error.code === "string") {
|
|
1370
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", error instanceof Error ? error.message : "context-window recovery was refused");
|
|
1371
|
+
}
|
|
1372
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "context-window recovery was not committed");
|
|
1373
|
+
}
|
|
1374
|
+
},
|
|
1375
|
+
async recoverContextOverflow(agentId, deliveryKey, reason) {
|
|
1376
|
+
return recoverContextOverflowInternal(agentId, deliveryKey, reason);
|
|
1377
|
+
},
|
|
896
1378
|
async start(configs) {
|
|
897
1379
|
let activeCount = 0;
|
|
898
1380
|
let recoveringCount = 0;
|
|
@@ -939,7 +1421,12 @@ function createRuntimeHost(options) {
|
|
|
939
1421
|
byMessage: new Map(persisted.records.map((record) => [record.messageId, record.deliveryId])),
|
|
940
1422
|
generation: 0, poller: null, retryTimer: null, recreateAttempts: 0,
|
|
941
1423
|
stabilityTimer: null, recreateReason: null, stopped: false, disabledReason: null,
|
|
942
|
-
configurationRecovery: null, stateStore,
|
|
1424
|
+
configurationRecovery: null, stateStore,
|
|
1425
|
+
compactionBreaker: config.runtime === "pi"
|
|
1426
|
+
? new pi_compaction_recovery_js_1.PiCompactionBreaker(config.stateDir ?? node_path_1.default.join(config.workspaceDir, ".larkin", "agents", config.agentId), {
|
|
1427
|
+
withLock: stateStore ? (operation) => stateStore.withInboxTransaction(operation) : undefined,
|
|
1428
|
+
}) : undefined,
|
|
1429
|
+
compactionMachines: new Map(), piOverflowCompactionFailed: new Set(), readiness: null,
|
|
943
1430
|
turnInProgress: false, turnHadFailure: false, turnHadAuthenticatedOutput: false, authFailureActive: false };
|
|
944
1431
|
const startupConsumed = [];
|
|
945
1432
|
const startupQuarantined = [];
|
|
@@ -971,6 +1458,7 @@ function createRuntimeHost(options) {
|
|
|
971
1458
|
emitConsumed(agent, [...startupConsumed, ...persist(agent)]);
|
|
972
1459
|
try {
|
|
973
1460
|
await ensureSession(agent);
|
|
1461
|
+
await recoverStalePiCompaction(agent);
|
|
974
1462
|
activeCount += 1;
|
|
975
1463
|
await retryPending(agent);
|
|
976
1464
|
}
|