larkin 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -33
- package/assets/readme/larkin-hero.svg +111 -0
- package/dist/agent/agent-state-store.cjs +194 -1
- package/dist/agent/agent-state-store.mjs +192 -0
- package/dist/app/cli.mjs +3 -1
- package/dist/app/local-control.mjs +108 -2
- package/dist/app/runtime-process.mjs +4 -0
- package/dist/app/session-cli.mjs +55 -20
- package/dist/feishu/host-shell.cjs +72 -1
- package/dist/feishu/host-shell.mjs +72 -1
- package/dist/runtime/pi-compaction-recovery.cjs +623 -0
- package/dist/runtime/pi-compaction-recovery.mjs +602 -0
- package/dist/runtime/pi-provider-config.cjs +2 -2
- package/dist/runtime/pi-provider-config.mjs +2 -2
- package/dist/runtime/pi-rpc-client.cjs +39 -22
- package/dist/runtime/pi-rpc-client.mjs +39 -22
- package/dist/runtime/provider-error-classifier.cjs +39 -0
- package/dist/runtime/provider-error-classifier.mjs +34 -0
- package/dist/runtime/runtime-adapters.mjs +82 -19
- package/dist/runtime/runtime-host.cjs +490 -2
- package/dist/runtime/runtime-host.mjs +488 -1
- package/package.json +1 -1
|
@@ -4,9 +4,11 @@ import path from "node:path";
|
|
|
4
4
|
import { agentCliPromptCapabilities } from "../agent/agent-cli-capabilities.mjs";
|
|
5
5
|
import { isCanonicalInboxTarget, targetKeyOfInboxEnvelope } from "../agent/inbox-projection.mjs";
|
|
6
6
|
import { SpanKind } from "@opentelemetry/api";
|
|
7
|
+
import { buildStrictProviderErrorInput, classifyStrictProviderError } from "./provider-error-classifier.mjs";
|
|
7
8
|
import { classifyRuntimePrerequisite, providerAuthenticationFailureReadiness, RuntimePrerequisiteError, } from "./runtime-readiness.mjs";
|
|
8
9
|
import { resolveOfficialLarkCli } from "../app/official-lark-cli.mjs";
|
|
9
10
|
import { assertAgentWorkspaceBound, managedLarkCliEnv } from "../app/agent-lark-cli-workspace.mjs";
|
|
11
|
+
import { PiCompactionBreaker, PiCompactionRecoveryMachine } from "./pi-compaction-recovery.mjs";
|
|
10
12
|
export class RuntimeSessionResetError extends Error {
|
|
11
13
|
code;
|
|
12
14
|
pendingCount;
|
|
@@ -17,6 +19,16 @@ export class RuntimeSessionResetError extends Error {
|
|
|
17
19
|
this.name = "RuntimeSessionResetError";
|
|
18
20
|
}
|
|
19
21
|
}
|
|
22
|
+
export class RuntimeSessionRecoveryError extends Error {
|
|
23
|
+
code;
|
|
24
|
+
pendingCount;
|
|
25
|
+
constructor(code, message, pendingCount = 0) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.code = code;
|
|
28
|
+
this.pendingCount = pendingCount;
|
|
29
|
+
this.name = "RuntimeSessionRecoveryError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
20
32
|
const MAX_DELIVERIES = 2048;
|
|
21
33
|
const now = () => new Date().toISOString();
|
|
22
34
|
const isActiveDelivery = (status) => ["pending", "submitting", "accepted"].includes(status);
|
|
@@ -106,6 +118,9 @@ export function createRuntimeHost(options) {
|
|
|
106
118
|
const listeners = new Set();
|
|
107
119
|
const log = options.log ?? (() => { });
|
|
108
120
|
const telemetry = options.telemetry;
|
|
121
|
+
const compactTimeoutMs = options.compactTimeoutMs ?? 120_000;
|
|
122
|
+
if (!Number.isFinite(compactTimeoutMs) || compactTimeoutMs <= 0)
|
|
123
|
+
throw new Error("compactTimeoutMs must be positive");
|
|
109
124
|
const retryPolicy = {
|
|
110
125
|
baseDelayMs: options.retryPolicy?.baseDelayMs ?? 250,
|
|
111
126
|
maxDelayMs: options.retryPolicy?.maxDelayMs ?? 10_000,
|
|
@@ -234,6 +249,7 @@ export function createRuntimeHost(options) {
|
|
|
234
249
|
scrubQuarantinedInput(record, reason);
|
|
235
250
|
record.reason = reason;
|
|
236
251
|
record.retryable = false;
|
|
252
|
+
delete record.errorCategory;
|
|
237
253
|
const finalRecord = setRecord(agent, record, "error");
|
|
238
254
|
if (finalRecord.status === "consumed")
|
|
239
255
|
return { status: "accepted", deliveryId: record.deliveryId };
|
|
@@ -374,12 +390,14 @@ export function createRuntimeHost(options) {
|
|
|
374
390
|
if (result.status === "accepted") {
|
|
375
391
|
delete record.reason;
|
|
376
392
|
delete record.retryable;
|
|
393
|
+
delete record.errorCategory;
|
|
377
394
|
const finalRecord = setRecord(agent, record, "accepted");
|
|
378
395
|
if (finalRecord.status !== "consumed")
|
|
379
396
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
380
397
|
deliveryId: record.deliveryId, messageId: record.messageId, status: "accepted" });
|
|
381
398
|
return { status: "accepted", deliveryId: record.deliveryId };
|
|
382
399
|
}
|
|
400
|
+
delete record.errorCategory;
|
|
383
401
|
const retryable = result.status === "deferred" || result.retryable;
|
|
384
402
|
record.reason = result.reason;
|
|
385
403
|
record.retryable = retryable;
|
|
@@ -434,6 +452,7 @@ export function createRuntimeHost(options) {
|
|
|
434
452
|
const reason = error instanceof Error ? error.message : String(error);
|
|
435
453
|
record.reason = reason;
|
|
436
454
|
record.retryable = true;
|
|
455
|
+
delete record.errorCategory;
|
|
437
456
|
setRecord(agent, record, "pending");
|
|
438
457
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId, messageId: record.messageId, status: "deferred", reason });
|
|
439
458
|
return { status: "deferred", deliveryId: record.deliveryId, reason };
|
|
@@ -560,6 +579,106 @@ export function createRuntimeHost(options) {
|
|
|
560
579
|
})();
|
|
561
580
|
agent.configurationRecovery = recovery;
|
|
562
581
|
};
|
|
582
|
+
let recoverContextOverflowInternal;
|
|
583
|
+
const beginPiManualCompaction = (agent, session, record, nativeWillRetry = false) => {
|
|
584
|
+
if (agent.adapter.id !== "pi" || !session.compact || !agent.compactionBreaker)
|
|
585
|
+
return;
|
|
586
|
+
const existing = agent.compactionMachines.get(record.input.inputId);
|
|
587
|
+
if (existing) {
|
|
588
|
+
existing.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
const key = `${record.deliveryId}:${record.input.inputId}`;
|
|
592
|
+
const prior = agent.compactionBreaker.get(key);
|
|
593
|
+
if (prior && prior.manualAttempt >= 1 && ["manual_sent", "manual_ambiguous", "manual_failed", "native_failed"].includes(prior.state)) {
|
|
594
|
+
agent.busy = false;
|
|
595
|
+
agent.turnInProgress = false;
|
|
596
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, "validated Pi compaction breaker state after restart", key)
|
|
597
|
+
.catch((error) => log("Pi restart fallback failed", String(error)));
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
const machine = new PiCompactionRecoveryMachine({
|
|
601
|
+
breaker: agent.compactionBreaker, key, messageId: record.messageId, deliveryId: record.deliveryId,
|
|
602
|
+
inputId: record.input.inputId, sessionGeneration: agent.generation,
|
|
603
|
+
onAction: (action) => {
|
|
604
|
+
if (action === "retry_input") {
|
|
605
|
+
machine.retrySubmitted();
|
|
606
|
+
record.status = "pending";
|
|
607
|
+
record.retryable = true;
|
|
608
|
+
record.reason = "Pi manual compaction succeeded; retrying the same delivery input";
|
|
609
|
+
record.updatedAt = now();
|
|
610
|
+
persist(agent);
|
|
611
|
+
queueMicrotask(() => { if (!agent.stopped)
|
|
612
|
+
void submit(agent, record, false); });
|
|
613
|
+
}
|
|
614
|
+
else if (action === "fresh_session_fallback") {
|
|
615
|
+
agent.busy = false;
|
|
616
|
+
agent.turnInProgress = false;
|
|
617
|
+
if (record.status !== "error") {
|
|
618
|
+
record.reason = machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback";
|
|
619
|
+
record.retryable = false;
|
|
620
|
+
record.errorCategory = "context_window";
|
|
621
|
+
setRecord(agent, record, "error");
|
|
622
|
+
}
|
|
623
|
+
void recoverContextOverflowInternal(agent.config.agentId, key, machine.recordSnapshot.fallbackReason || "Pi compaction recovery fallback", key)
|
|
624
|
+
.catch((error) => log("Pi context fallback failed", String(error)));
|
|
625
|
+
}
|
|
626
|
+
},
|
|
627
|
+
});
|
|
628
|
+
agent.compactionMachines.set(record.input.inputId, machine);
|
|
629
|
+
machine.agentEnd({ exactOverflow: true, willRetry: nativeWillRetry });
|
|
630
|
+
if (nativeWillRetry)
|
|
631
|
+
return;
|
|
632
|
+
machine.agentSettled();
|
|
633
|
+
const deadline = new Date(Date.now() + compactTimeoutMs).toISOString();
|
|
634
|
+
machine.manualCompactSent(deadline);
|
|
635
|
+
let compactSettled = false;
|
|
636
|
+
const compactTimer = setTimeout(() => {
|
|
637
|
+
if (compactSettled)
|
|
638
|
+
return;
|
|
639
|
+
compactSettled = true;
|
|
640
|
+
machine.compactTimeout();
|
|
641
|
+
}, compactTimeoutMs);
|
|
642
|
+
compactTimer.unref?.();
|
|
643
|
+
void session.compact().then(() => {
|
|
644
|
+
if (compactSettled)
|
|
645
|
+
return;
|
|
646
|
+
compactSettled = true;
|
|
647
|
+
clearTimeout(compactTimer);
|
|
648
|
+
machine.compactResponse({ success: true });
|
|
649
|
+
}).catch((error) => {
|
|
650
|
+
if (compactSettled)
|
|
651
|
+
return;
|
|
652
|
+
compactSettled = true;
|
|
653
|
+
clearTimeout(compactTimer);
|
|
654
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
655
|
+
machine.compactResponse({ success: false, ambiguous: /process exited|process failed|session replaced|protocol/i.test(message) });
|
|
656
|
+
});
|
|
657
|
+
};
|
|
658
|
+
const recoverStalePiCompaction = async (agent) => {
|
|
659
|
+
const breaker = agent.compactionBreaker;
|
|
660
|
+
if (!breaker)
|
|
661
|
+
return;
|
|
662
|
+
const stale = breaker.listNonTerminal().find((candidate) => candidate.state !== "native_succeeded");
|
|
663
|
+
if (!stale) {
|
|
664
|
+
const succeeded = breaker.listNonTerminal().find((candidate) => candidate.state === "native_succeeded");
|
|
665
|
+
if (succeeded)
|
|
666
|
+
breaker.transition(succeeded.key, {}, "closed");
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === stale.deliveryId
|
|
670
|
+
|| candidate.input.inputId === stale.inputId);
|
|
671
|
+
if (!record)
|
|
672
|
+
return;
|
|
673
|
+
record.reason = stale.fallbackReason || "Pi compaction recovery resumed after restart";
|
|
674
|
+
record.retryable = false;
|
|
675
|
+
record.errorCategory = "context_window";
|
|
676
|
+
setRecord(agent, record, "error");
|
|
677
|
+
breaker.forceFallback(stale.key, record.reason);
|
|
678
|
+
agent.busy = false;
|
|
679
|
+
agent.turnInProgress = false;
|
|
680
|
+
await recoverContextOverflowInternal(agent.config.agentId, stale.key, record.reason, stale.key);
|
|
681
|
+
};
|
|
563
682
|
const observe = (agent, session, event) => {
|
|
564
683
|
if (agent.session !== session)
|
|
565
684
|
return; // Ignore late output from a replaced child.
|
|
@@ -568,6 +687,34 @@ export function createRuntimeHost(options) {
|
|
|
568
687
|
if (event.type !== "turn-end")
|
|
569
688
|
telemetry?.runtimeEvent(agent.config.agentId, event);
|
|
570
689
|
emit({ type: "runtime", agentId: agent.config.agentId, event });
|
|
690
|
+
if (event.type === "runtime-observation") {
|
|
691
|
+
if (event.phase === "compaction_start" && event.reason === "overflow" && event.inputId) {
|
|
692
|
+
const overflowRecord = [...agent.records.values()].find((candidate) => candidate.input.inputId === event.inputId);
|
|
693
|
+
if (overflowRecord && !agent.compactionMachines.has(event.inputId))
|
|
694
|
+
beginPiManualCompaction(agent, session, overflowRecord, true);
|
|
695
|
+
}
|
|
696
|
+
const candidates = [...agent.compactionMachines.values()].filter((machine) => machine.recordSnapshot.sessionGeneration === agent.generation
|
|
697
|
+
&& (!event.inputId || machine.recordSnapshot.inputId === event.inputId));
|
|
698
|
+
const machines = event.inputId ? candidates : candidates.length === 1 ? candidates : [];
|
|
699
|
+
for (const machine of machines) {
|
|
700
|
+
if (event.phase === "compaction_start" && event.reason === "overflow")
|
|
701
|
+
machine.compactionStart({ reason: "overflow" });
|
|
702
|
+
if (event.phase === "compaction_end" && event.reason === "manual") {
|
|
703
|
+
machine.compactionEnd({ reason: "manual", success: event.success === true, willRetry: event.willRetry === true });
|
|
704
|
+
}
|
|
705
|
+
if (event.phase === "compaction_end" && event.reason === "overflow") {
|
|
706
|
+
machine.compactionEnd({ reason: "overflow", success: event.success === true, willRetry: event.willRetry === true });
|
|
707
|
+
}
|
|
708
|
+
if (event.phase === "settled")
|
|
709
|
+
machine.agentSettled();
|
|
710
|
+
}
|
|
711
|
+
if (event.phase === "compaction_end" && event.reason === "overflow" && event.success !== true) {
|
|
712
|
+
for (const record of agent.records.values()) {
|
|
713
|
+
if (isActiveDelivery(record.status))
|
|
714
|
+
agent.piOverflowCompactionFailed.add(record.input.inputId);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
571
718
|
if (event.type === "session-init") {
|
|
572
719
|
agent.config.sessionId = event.sessionId;
|
|
573
720
|
emit({ type: "session", agentId: agent.config.agentId, runtime: agent.adapter.id, sessionId: event.sessionId, launchId: agent.launchId,
|
|
@@ -583,6 +730,13 @@ export function createRuntimeHost(options) {
|
|
|
583
730
|
emit({ type: "activity", agentId: agent.config.agentId, activity: "working", activityKind: "working", detailKind: "turn_started" });
|
|
584
731
|
}
|
|
585
732
|
else if (event.type === "turn-end") {
|
|
733
|
+
for (const [inputId, machine] of agent.compactionMachines) {
|
|
734
|
+
if (machine.recordSnapshot.sessionGeneration !== agent.generation)
|
|
735
|
+
continue;
|
|
736
|
+
machine.close();
|
|
737
|
+
if (["closed", "fallback_committed"].includes(machine.state))
|
|
738
|
+
agent.compactionMachines.delete(inputId);
|
|
739
|
+
}
|
|
586
740
|
const recoveredAuthentication = agent.authFailureActive && agent.turnInProgress
|
|
587
741
|
&& agent.turnHadAuthenticatedOutput && !agent.turnHadFailure;
|
|
588
742
|
agent.turnInProgress = false;
|
|
@@ -615,6 +769,29 @@ export function createRuntimeHost(options) {
|
|
|
615
769
|
if (!record || record.status === "consumed" || record.status === "error")
|
|
616
770
|
return;
|
|
617
771
|
agent.turnHadFailure = true;
|
|
772
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry === true && session.compact) {
|
|
773
|
+
beginPiManualCompaction(agent, session, record, true);
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
if (agent.adapter.id === "pi" && event.errorCategory === "context_window" && event.willRetry !== true && session.compact) {
|
|
777
|
+
record.reason = event.message;
|
|
778
|
+
record.retryable = false;
|
|
779
|
+
record.errorCategory = "context_window";
|
|
780
|
+
setRecord(agent, record, "error");
|
|
781
|
+
if (agent.piOverflowCompactionFailed.has(record.input.inputId)) {
|
|
782
|
+
agent.busy = false;
|
|
783
|
+
agent.turnInProgress = false;
|
|
784
|
+
void recoverContextOverflowInternal(agent.config.agentId, `${record.deliveryId}:${record.input.inputId}`, "native Pi overflow compaction failed", `${record.deliveryId}:${record.input.inputId}`)
|
|
785
|
+
.catch((error) => log("Pi native compaction fallback failed", String(error)));
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
// Pi emits this terminal error only after agent_settled; release the
|
|
789
|
+
// normal turn reservation before the one authorized recovery action.
|
|
790
|
+
agent.busy = false;
|
|
791
|
+
agent.turnInProgress = false;
|
|
792
|
+
beginPiManualCompaction(agent, session, record);
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
618
795
|
if (event.willRetry) {
|
|
619
796
|
emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
|
|
620
797
|
messageId: record.messageId, status: "deferred", reason: event.message });
|
|
@@ -625,6 +802,13 @@ export function createRuntimeHost(options) {
|
|
|
625
802
|
agent.busy = false;
|
|
626
803
|
record.reason = event.message;
|
|
627
804
|
record.retryable = event.retryable;
|
|
805
|
+
const classifiedCategory = event.errorCategory ?? classifyStrictProviderError(buildStrictProviderErrorInput({
|
|
806
|
+
message: event.message, errorCategory: event.errorCategory, upstream: event.upstream,
|
|
807
|
+
}));
|
|
808
|
+
if (classifiedCategory)
|
|
809
|
+
record.errorCategory = classifiedCategory;
|
|
810
|
+
else
|
|
811
|
+
delete record.errorCategory;
|
|
628
812
|
const finalRecord = setRecord(agent, record, event.retryable ? "pending" : "error");
|
|
629
813
|
if (finalRecord.status !== "consumed")
|
|
630
814
|
emit({ type: "delivery", agentId: agent.config.agentId,
|
|
@@ -711,6 +895,129 @@ export function createRuntimeHost(options) {
|
|
|
711
895
|
});
|
|
712
896
|
return agent.starting;
|
|
713
897
|
};
|
|
898
|
+
recoverContextOverflowInternal = async (agentId, deliveryKey, reason, breakerKey) => {
|
|
899
|
+
const agent = managed.get(agentId);
|
|
900
|
+
if (!agent || agent.stopped)
|
|
901
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
902
|
+
if (agent.adapter.id !== "pi")
|
|
903
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "Pi context fallback is unavailable for this runtime");
|
|
904
|
+
const oldSession = agent.session;
|
|
905
|
+
if (!oldSession || agent.busy || agent.submitting || agent.starting) {
|
|
906
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle for Pi context fallback`);
|
|
907
|
+
}
|
|
908
|
+
const record = [...agent.records.values()].find((candidate) => candidate.deliveryId === deliveryKey || candidate.input.inputId === deliveryKey || `${candidate.deliveryId}:${candidate.input.inputId}` === deliveryKey);
|
|
909
|
+
if (!record)
|
|
910
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "Pi context fallback delivery identity is unavailable");
|
|
911
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:pi-context-fallback`);
|
|
912
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
913
|
+
const readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
914
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir, env: probeEnv }) : { runtime: "pi", state: "ready" };
|
|
915
|
+
if (readiness.state !== "ready")
|
|
916
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback readiness failed");
|
|
917
|
+
const standingPrompt = options.promptBuilder.build({
|
|
918
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
919
|
+
description: agent.config.description || "", runtime: agent.adapter.id, cli: agentCliPromptCapabilities("larkin"),
|
|
920
|
+
});
|
|
921
|
+
let fresh;
|
|
922
|
+
try {
|
|
923
|
+
fresh = await agent.adapter.createSession({ agentId: agent.config.agentId, model: agent.config.model,
|
|
924
|
+
reasoningEffort: agent.config.effort || null, workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
925
|
+
resumeSessionId: null, standingPrompt, env: probeEnv });
|
|
926
|
+
}
|
|
927
|
+
catch {
|
|
928
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi fresh fallback session could not be staged");
|
|
929
|
+
}
|
|
930
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
931
|
+
await fresh.close("Pi context fallback commit precondition changed").catch(() => { });
|
|
932
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
933
|
+
}
|
|
934
|
+
let unsubscribeFresh = null;
|
|
935
|
+
try {
|
|
936
|
+
unsubscribeFresh = fresh.subscribe((next) => observe(agent, fresh, next));
|
|
937
|
+
}
|
|
938
|
+
catch {
|
|
939
|
+
await fresh.close("Pi context fallback subscription failed").catch(() => { });
|
|
940
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Pi context fallback subscription failed");
|
|
941
|
+
}
|
|
942
|
+
const oldSessionId = oldSession.sessionId;
|
|
943
|
+
const oldGeneration = agent.generation;
|
|
944
|
+
const oldLaunchId = agent.launchId;
|
|
945
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
946
|
+
const oldReadiness = agent.readiness;
|
|
947
|
+
const oldDisabledReason = agent.disabledReason;
|
|
948
|
+
const oldRecord = { ...record, input: { ...record.input } };
|
|
949
|
+
let durableRollback = null;
|
|
950
|
+
let rearmedCount = 1;
|
|
951
|
+
let remainingPendingCount = 1;
|
|
952
|
+
const commitSwap = (messageIds, rollback) => {
|
|
953
|
+
if (agent.session !== oldSession || agent.busy || agent.submitting || agent.starting || agent.stopped) {
|
|
954
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during Pi context fallback`);
|
|
955
|
+
}
|
|
956
|
+
durableRollback = rollback ?? null;
|
|
957
|
+
if (breakerKey && agent.compactionBreaker)
|
|
958
|
+
agent.compactionBreaker.forceFallbackInTransaction(breakerKey, reason);
|
|
959
|
+
agent.generation += 1;
|
|
960
|
+
agent.launchId = crypto.randomUUID();
|
|
961
|
+
agent.config.sessionId = null;
|
|
962
|
+
agent.session = fresh;
|
|
963
|
+
agent.readiness = readiness;
|
|
964
|
+
agent.disabledReason = null;
|
|
965
|
+
for (const candidate of agent.records.values()) {
|
|
966
|
+
if (!messageIds.includes(candidate.messageId))
|
|
967
|
+
continue;
|
|
968
|
+
candidate.status = "pending";
|
|
969
|
+
candidate.retryable = true;
|
|
970
|
+
candidate.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
971
|
+
candidate.updatedAt = now();
|
|
972
|
+
}
|
|
973
|
+
};
|
|
974
|
+
try {
|
|
975
|
+
if (agent.stateStore?.rearmContextOverflow) {
|
|
976
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => commitSwap(messageIds, rollback), {
|
|
977
|
+
messageId: record.messageId, deliveryId: record.deliveryId, inputId: record.input.inputId,
|
|
978
|
+
});
|
|
979
|
+
rearmedCount = rearmed.rearmedCount;
|
|
980
|
+
remainingPendingCount = rearmed.remainingPendingCount;
|
|
981
|
+
}
|
|
982
|
+
else {
|
|
983
|
+
commitSwap([record.messageId]);
|
|
984
|
+
persist(agent);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
catch (error) {
|
|
988
|
+
try {
|
|
989
|
+
const rollback = durableRollback;
|
|
990
|
+
if (rollback)
|
|
991
|
+
rollback();
|
|
992
|
+
}
|
|
993
|
+
catch { /* preserve primary error */ }
|
|
994
|
+
agent.generation = oldGeneration;
|
|
995
|
+
agent.launchId = oldLaunchId;
|
|
996
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
997
|
+
agent.session = oldSession;
|
|
998
|
+
agent.readiness = oldReadiness;
|
|
999
|
+
agent.disabledReason = oldDisabledReason;
|
|
1000
|
+
Object.assign(record, oldRecord);
|
|
1001
|
+
try {
|
|
1002
|
+
unsubscribeFresh?.();
|
|
1003
|
+
}
|
|
1004
|
+
catch { /* preserve primary failure */ }
|
|
1005
|
+
await fresh.close("Pi context fallback rolled back").catch(() => { });
|
|
1006
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", error instanceof Error ? error.message : "Pi context fallback commit failed");
|
|
1007
|
+
}
|
|
1008
|
+
if (fresh.sessionId) {
|
|
1009
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1010
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1011
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1012
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1013
|
+
}
|
|
1014
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1015
|
+
await oldSession.close("Pi context fallback committed").catch((error) => log("previous Pi session close after fallback failed", String(error)));
|
|
1016
|
+
void retryPending(agent);
|
|
1017
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1018
|
+
runtimeReady: true, pendingCount: remainingPendingCount, rearmedCount,
|
|
1019
|
+
replayStatus: remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1020
|
+
};
|
|
714
1021
|
return {
|
|
715
1022
|
async probe(config) {
|
|
716
1023
|
const adapter = options.adapterFor(config.runtime);
|
|
@@ -885,6 +1192,180 @@ export function createRuntimeHost(options) {
|
|
|
885
1192
|
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
886
1193
|
runtimeReady: true, pendingCount: 0, sessionId: fresh.sessionId };
|
|
887
1194
|
},
|
|
1195
|
+
async recoverSession(agentId, reason) {
|
|
1196
|
+
if (reason !== "context-overflow")
|
|
1197
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "unsupported recovery reason");
|
|
1198
|
+
const agent = managed.get(agentId);
|
|
1199
|
+
if (agent && agent.adapter.id !== "pi")
|
|
1200
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "context-overflow recovery is available only for Pi Runtime");
|
|
1201
|
+
if (!agent || agent.stopped)
|
|
1202
|
+
throw new RuntimeSessionRecoveryError("unknown_agent", `unknown runtime Agent: ${agentId}`);
|
|
1203
|
+
if (!agent.stateStore?.rearmContextOverflow) {
|
|
1204
|
+
throw new RuntimeSessionRecoveryError("recovery_unavailable", "canonical Inbox recovery is unavailable");
|
|
1205
|
+
}
|
|
1206
|
+
if (agent.busy || agent.turnInProgress || agent.submitting || agent.starting) {
|
|
1207
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} is not idle`);
|
|
1208
|
+
}
|
|
1209
|
+
const oldSession = agent.session;
|
|
1210
|
+
if (!oldSession)
|
|
1211
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} Runtime session is unavailable`);
|
|
1212
|
+
const oldSessionId = oldSession.sessionId;
|
|
1213
|
+
if (agent.stateStore.readNdjson) {
|
|
1214
|
+
const pendingRows = agent.stateStore.withInboxTransaction(() => agent.stateStore.readNdjson("inbox"));
|
|
1215
|
+
if (pendingRows.length === 0) {
|
|
1216
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery has no retained Inbox backlog");
|
|
1217
|
+
}
|
|
1218
|
+
const messageIds = new Set(pendingRows.flatMap((row) => {
|
|
1219
|
+
const messageId = row && typeof row === "object" && !Array.isArray(row)
|
|
1220
|
+
? row.message_id : undefined;
|
|
1221
|
+
return typeof messageId === "string" && messageId ? [messageId] : [];
|
|
1222
|
+
}));
|
|
1223
|
+
const persisted = agent.stateStore.readJson("runtimeDeliveries", { version: 1, records: [] });
|
|
1224
|
+
if (Array.isArray(persisted.records) && persisted.records.some((record) => messageIds.has(record.messageId) && record.status !== "error")) {
|
|
1225
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", "context-window recovery is already staged or completed");
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
const probeEnv = runtimeEnv(agent.config, `${agent.launchId}:context-overflow-recovery`);
|
|
1229
|
+
let readiness;
|
|
1230
|
+
try {
|
|
1231
|
+
await assertOfficialCliReady(agent.config, probeEnv);
|
|
1232
|
+
readiness = agent.adapter.probe ? await agent.adapter.probe({ agentId: agent.config.agentId,
|
|
1233
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1234
|
+
env: { LARKIN_PI_COMMAND: process.env.LARKIN_PI_COMMAND, LARKIN_CODEX_COMMAND: process.env.LARKIN_CODEX_COMMAND,
|
|
1235
|
+
LARKIN_CLAUDE_COMMAND: process.env.LARKIN_CLAUDE_COMMAND, ...probeEnv } })
|
|
1236
|
+
: { runtime: agent.adapter.id, state: "ready" };
|
|
1237
|
+
}
|
|
1238
|
+
catch (error) {
|
|
1239
|
+
if (error instanceof RuntimePrerequisiteError)
|
|
1240
|
+
throw error;
|
|
1241
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "Runtime readiness probe failed");
|
|
1242
|
+
}
|
|
1243
|
+
if (readiness.state !== "ready")
|
|
1244
|
+
throw new RuntimePrerequisiteError(readiness);
|
|
1245
|
+
const standingPrompt = options.promptBuilder.build({
|
|
1246
|
+
agentId: agent.config.agentId, name: agent.config.displayName || agent.config.name,
|
|
1247
|
+
description: agent.config.description || "", runtime: agent.adapter.id,
|
|
1248
|
+
cli: agentCliPromptCapabilities("larkin"),
|
|
1249
|
+
});
|
|
1250
|
+
let fresh;
|
|
1251
|
+
try {
|
|
1252
|
+
fresh = await agent.adapter.createSession({
|
|
1253
|
+
agentId: agent.config.agentId, model: agent.config.model, reasoningEffort: agent.config.effort || null,
|
|
1254
|
+
workspaceDir: agent.config.workspaceDir, stateDir: agent.config.stateDir,
|
|
1255
|
+
resumeSessionId: null, standingPrompt, env: probeEnv,
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
catch {
|
|
1259
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session could not be staged");
|
|
1260
|
+
}
|
|
1261
|
+
let unsubscribeFresh = null;
|
|
1262
|
+
try {
|
|
1263
|
+
unsubscribeFresh = fresh.subscribe((event) => observe(agent, fresh, event));
|
|
1264
|
+
}
|
|
1265
|
+
catch {
|
|
1266
|
+
await fresh.close("context-window recovery subscription failed").catch(() => { });
|
|
1267
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "fresh Runtime session subscription failed");
|
|
1268
|
+
}
|
|
1269
|
+
const oldGeneration = agent.generation;
|
|
1270
|
+
const oldLaunchId = agent.launchId;
|
|
1271
|
+
const oldConfigSessionId = agent.config.sessionId;
|
|
1272
|
+
const oldReadiness = agent.readiness;
|
|
1273
|
+
const oldDisabledReason = agent.disabledReason;
|
|
1274
|
+
const oldRecords = new Map([...agent.records.entries()].map(([id, record]) => [id, { ...record, input: { ...record.input } }]));
|
|
1275
|
+
const cleanupFresh = async (reason) => {
|
|
1276
|
+
const unsubscribe = unsubscribeFresh;
|
|
1277
|
+
unsubscribeFresh = null;
|
|
1278
|
+
try {
|
|
1279
|
+
unsubscribe?.();
|
|
1280
|
+
}
|
|
1281
|
+
catch { /* close still owns staged-session cleanup */ }
|
|
1282
|
+
await fresh.close(reason).catch(() => { });
|
|
1283
|
+
};
|
|
1284
|
+
const restoreRuntime = () => {
|
|
1285
|
+
agent.session = oldSession;
|
|
1286
|
+
agent.generation = oldGeneration;
|
|
1287
|
+
agent.launchId = oldLaunchId;
|
|
1288
|
+
agent.config.sessionId = oldConfigSessionId;
|
|
1289
|
+
agent.readiness = oldReadiness;
|
|
1290
|
+
agent.disabledReason = oldDisabledReason;
|
|
1291
|
+
agent.records = new Map(oldRecords);
|
|
1292
|
+
agent.byMessage = new Map([...oldRecords.values()].map((record) => [record.messageId, record.deliveryId]));
|
|
1293
|
+
};
|
|
1294
|
+
let recoveryProjectionEmitted = false;
|
|
1295
|
+
const compensateProjection = () => {
|
|
1296
|
+
if (!recoveryProjectionEmitted)
|
|
1297
|
+
return;
|
|
1298
|
+
try {
|
|
1299
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: oldSessionId, launchId: oldLaunchId });
|
|
1300
|
+
}
|
|
1301
|
+
catch { /* preserve the original listener failure */ }
|
|
1302
|
+
try {
|
|
1303
|
+
emit({ type: "agent-status", agentId, status: "active", ...(oldReadiness ? { readiness: oldReadiness } : {}) });
|
|
1304
|
+
}
|
|
1305
|
+
catch { /* preserve the original listener failure */ }
|
|
1306
|
+
};
|
|
1307
|
+
let durableRollback = null;
|
|
1308
|
+
try {
|
|
1309
|
+
const rearmed = agent.stateStore.rearmContextOverflow((messageIds, rollback) => {
|
|
1310
|
+
// This callback runs under the canonical Inbox lock. All potentially
|
|
1311
|
+
// racing checks above are synchronous from this process, while the
|
|
1312
|
+
// state-store method rechecks every row and delivery on disk.
|
|
1313
|
+
if (agent.session !== oldSession || agent.busy || agent.turnInProgress || agent.submitting || agent.starting || agent.stopped) {
|
|
1314
|
+
throw new RuntimeSessionRecoveryError("agent_busy", `Agent ${agentId} changed during recovery`);
|
|
1315
|
+
}
|
|
1316
|
+
durableRollback = rollback;
|
|
1317
|
+
agent.generation += 1;
|
|
1318
|
+
agent.launchId = crypto.randomUUID();
|
|
1319
|
+
agent.config.sessionId = null;
|
|
1320
|
+
agent.session = fresh;
|
|
1321
|
+
agent.readiness = readiness;
|
|
1322
|
+
agent.disabledReason = null;
|
|
1323
|
+
for (const record of agent.records.values()) {
|
|
1324
|
+
if (!messageIds.includes(record.messageId))
|
|
1325
|
+
continue;
|
|
1326
|
+
record.status = "pending";
|
|
1327
|
+
record.retryable = true;
|
|
1328
|
+
record.reason = "context-window recovery rearmed the retained Inbox delivery";
|
|
1329
|
+
record.updatedAt = now();
|
|
1330
|
+
}
|
|
1331
|
+
});
|
|
1332
|
+
recoveryProjectionEmitted = true;
|
|
1333
|
+
if (fresh.sessionId) {
|
|
1334
|
+
agent.config.sessionId = fresh.sessionId;
|
|
1335
|
+
emit({ type: "session", agentId, runtime: agent.adapter.id, sessionId: fresh.sessionId, launchId: agent.launchId,
|
|
1336
|
+
...(fresh.effectiveModel ? { model: fresh.effectiveModel } : {}),
|
|
1337
|
+
...(fresh.effectiveReasoningEffort ? { reasoningEffort: fresh.effectiveReasoningEffort } : {}) });
|
|
1338
|
+
}
|
|
1339
|
+
emit({ type: "agent-status", agentId, status: "active", readiness });
|
|
1340
|
+
durableRollback = null;
|
|
1341
|
+
await oldSession.close("context-window recovery committed")
|
|
1342
|
+
.catch((error) => log("previous runtime close after context-window recovery failed", String(error)));
|
|
1343
|
+
void retryPending(agent);
|
|
1344
|
+
return { generationChanged: true, sessionChanged: oldSessionId !== fresh.sessionId, turns: 0,
|
|
1345
|
+
runtimeReady: true, pendingCount: rearmed.remainingPendingCount, rearmedCount: rearmed.rearmedCount,
|
|
1346
|
+
replayStatus: rearmed.remainingPendingCount > 0 ? "pending" : "consumed", sessionId: fresh.sessionId };
|
|
1347
|
+
}
|
|
1348
|
+
catch (error) {
|
|
1349
|
+
try {
|
|
1350
|
+
const rollback = durableRollback;
|
|
1351
|
+
if (rollback)
|
|
1352
|
+
rollback();
|
|
1353
|
+
}
|
|
1354
|
+
catch { /* preserve the original failure */ }
|
|
1355
|
+
restoreRuntime();
|
|
1356
|
+
compensateProjection();
|
|
1357
|
+
await cleanupFresh("context-window recovery not committed");
|
|
1358
|
+
if (error instanceof RuntimeSessionRecoveryError)
|
|
1359
|
+
throw error;
|
|
1360
|
+
if (typeof error.code === "string") {
|
|
1361
|
+
throw new RuntimeSessionRecoveryError("recovery_refused", error instanceof Error ? error.message : "context-window recovery was refused");
|
|
1362
|
+
}
|
|
1363
|
+
throw new RuntimeSessionRecoveryError("recovery_staged_not_committed", "context-window recovery was not committed");
|
|
1364
|
+
}
|
|
1365
|
+
},
|
|
1366
|
+
async recoverContextOverflow(agentId, deliveryKey, reason) {
|
|
1367
|
+
return recoverContextOverflowInternal(agentId, deliveryKey, reason);
|
|
1368
|
+
},
|
|
888
1369
|
async start(configs) {
|
|
889
1370
|
let activeCount = 0;
|
|
890
1371
|
let recoveringCount = 0;
|
|
@@ -931,7 +1412,12 @@ export function createRuntimeHost(options) {
|
|
|
931
1412
|
byMessage: new Map(persisted.records.map((record) => [record.messageId, record.deliveryId])),
|
|
932
1413
|
generation: 0, poller: null, retryTimer: null, recreateAttempts: 0,
|
|
933
1414
|
stabilityTimer: null, recreateReason: null, stopped: false, disabledReason: null,
|
|
934
|
-
configurationRecovery: null, stateStore,
|
|
1415
|
+
configurationRecovery: null, stateStore,
|
|
1416
|
+
compactionBreaker: config.runtime === "pi"
|
|
1417
|
+
? new PiCompactionBreaker(config.stateDir ?? path.join(config.workspaceDir, ".larkin", "agents", config.agentId), {
|
|
1418
|
+
withLock: stateStore ? (operation) => stateStore.withInboxTransaction(operation) : undefined,
|
|
1419
|
+
}) : undefined,
|
|
1420
|
+
compactionMachines: new Map(), piOverflowCompactionFailed: new Set(), readiness: null,
|
|
935
1421
|
turnInProgress: false, turnHadFailure: false, turnHadAuthenticatedOutput: false, authFailureActive: false };
|
|
936
1422
|
const startupConsumed = [];
|
|
937
1423
|
const startupQuarantined = [];
|
|
@@ -963,6 +1449,7 @@ export function createRuntimeHost(options) {
|
|
|
963
1449
|
emitConsumed(agent, [...startupConsumed, ...persist(agent)]);
|
|
964
1450
|
try {
|
|
965
1451
|
await ensureSession(agent);
|
|
1452
|
+
await recoverStalePiCompaction(agent);
|
|
966
1453
|
activeCount += 1;
|
|
967
1454
|
await retryPending(agent);
|
|
968
1455
|
}
|