larkin 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/agent/context-prompt.cjs +2 -1
- package/dist/agent/context-prompt.mjs +2 -1
- package/dist/app/cli.mjs +2 -2
- package/dist/app/runtime-agent-config.mjs +7 -3
- package/dist/app/setup.mjs +11 -0
- package/dist/feishu/host-shell.cjs +4 -3
- package/dist/feishu/host-shell.mjs +4 -3
- package/dist/feishu/platform-hosts.cjs +49 -0
- package/dist/feishu/platform-hosts.mjs +40 -0
- package/dist/runtime/pi-subagents-notification.mjs +161 -0
- package/dist/runtime/runtime-adapters.mjs +32 -1
- package/dist/runtime/runtime-host.cjs +250 -1
- package/dist/runtime/runtime-host.mjs +250 -1
- package/dist/setup/bot-register.mjs +50 -4
- package/dist/setup/grant-scopes.mjs +16 -3
- package/package.json +1 -1
|
@@ -39,6 +39,7 @@ class RuntimeSessionRecoveryError extends Error {
|
|
|
39
39
|
}
|
|
40
40
|
exports.RuntimeSessionRecoveryError = RuntimeSessionRecoveryError;
|
|
41
41
|
const MAX_DELIVERIES = 2048;
|
|
42
|
+
const BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT = 5;
|
|
42
43
|
const now = () => new Date().toISOString();
|
|
43
44
|
const isActiveDelivery = (status) => ["pending", "submitting", "accepted"].includes(status);
|
|
44
45
|
const REPLAY_FAILURE_CODES = new Set([
|
|
@@ -521,6 +522,12 @@ function createRuntimeHost(options) {
|
|
|
521
522
|
queueMicrotask(() => { void retryPending(agent); });
|
|
522
523
|
}
|
|
523
524
|
queueMicrotask(() => { void scanAndPromoteAcceptedInboxUpdates(agent); });
|
|
525
|
+
// A completion queued during this submit has no turn-end if prompt
|
|
526
|
+
// rejected/threw before turn-start. Drain now that submitting is clear.
|
|
527
|
+
if (!agent.busy && !agent.turnInProgress && agent.backgroundCompletionQueue.length > 0
|
|
528
|
+
&& !agent.backgroundCompletionRetryTimer) {
|
|
529
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
530
|
+
}
|
|
524
531
|
}
|
|
525
532
|
};
|
|
526
533
|
const retryPending = async (agent) => {
|
|
@@ -599,6 +606,210 @@ function createRuntimeHost(options) {
|
|
|
599
606
|
agent.retryAfterSubmit = true;
|
|
600
607
|
await retryPending(agent);
|
|
601
608
|
};
|
|
609
|
+
const clearBackgroundCompletionRetryTimer = (agent) => {
|
|
610
|
+
if (!agent.backgroundCompletionRetryTimer)
|
|
611
|
+
return;
|
|
612
|
+
clearTimeout(agent.backgroundCompletionRetryTimer);
|
|
613
|
+
agent.backgroundCompletionRetryTimer = null;
|
|
614
|
+
};
|
|
615
|
+
const scheduleBackgroundCompletionDrain = (agent, delayMs = 0) => {
|
|
616
|
+
if (agent.stopped)
|
|
617
|
+
return;
|
|
618
|
+
if (delayMs > 0) {
|
|
619
|
+
if (agent.backgroundCompletionRetryTimer)
|
|
620
|
+
return;
|
|
621
|
+
agent.backgroundCompletionRetryTimer = setTimeout(() => {
|
|
622
|
+
agent.backgroundCompletionRetryTimer = null;
|
|
623
|
+
if (agent.stopped)
|
|
624
|
+
return;
|
|
625
|
+
void drainBackgroundCompletionQueue(agent);
|
|
626
|
+
}, delayMs);
|
|
627
|
+
agent.backgroundCompletionRetryTimer.unref?.();
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
631
|
+
queueMicrotask(() => { void drainBackgroundCompletionQueue(agent); });
|
|
632
|
+
};
|
|
633
|
+
const dropBackgroundCompletion = (agent, completionKey) => {
|
|
634
|
+
if (!completionKey)
|
|
635
|
+
return;
|
|
636
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
637
|
+
agent.backgroundCompletionQueue.shift();
|
|
638
|
+
else {
|
|
639
|
+
const index = agent.backgroundCompletionQueue.indexOf(completionKey);
|
|
640
|
+
if (index >= 0)
|
|
641
|
+
agent.backgroundCompletionQueue.splice(index, 1);
|
|
642
|
+
}
|
|
643
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
644
|
+
if (agent.backgroundCompletionInFlight === completionKey) {
|
|
645
|
+
agent.backgroundCompletionInFlight = null;
|
|
646
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
647
|
+
}
|
|
648
|
+
agent.backgroundCompletionRejectStreak = 0;
|
|
649
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
650
|
+
};
|
|
651
|
+
const scheduleBackgroundCompletionRetry = (agent) => {
|
|
652
|
+
const streak = agent.backgroundCompletionRejectStreak;
|
|
653
|
+
if (streak < BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT) {
|
|
654
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
const delayedAttempt = streak - BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT;
|
|
658
|
+
if (delayedAttempt >= retryPolicy.maxAttempts) {
|
|
659
|
+
const exhausted = agent.backgroundCompletionQueue[0];
|
|
660
|
+
if (exhausted) {
|
|
661
|
+
log("background completion wake retries exhausted", exhausted);
|
|
662
|
+
dropBackgroundCompletion(agent, exhausted);
|
|
663
|
+
}
|
|
664
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
665
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
666
|
+
return;
|
|
667
|
+
}
|
|
668
|
+
const delay = Math.min(retryPolicy.maxDelayMs, retryPolicy.baseDelayMs * 2 ** delayedAttempt);
|
|
669
|
+
scheduleBackgroundCompletionDrain(agent, delay);
|
|
670
|
+
};
|
|
671
|
+
const isPersistentBackgroundWakeFailure = (event) => {
|
|
672
|
+
if (!event)
|
|
673
|
+
return false;
|
|
674
|
+
const category = event.errorCategory;
|
|
675
|
+
if (category === "auth" || category === "billing" || category === "quota")
|
|
676
|
+
return true;
|
|
677
|
+
return category === "provider" && event.retryable === false;
|
|
678
|
+
};
|
|
679
|
+
const failBackgroundCompletionWake = (agent, event) => {
|
|
680
|
+
const completionKey = agent.backgroundCompletionInFlight ?? agent.backgroundCompletionQueue[0];
|
|
681
|
+
if (!completionKey)
|
|
682
|
+
return;
|
|
683
|
+
rearmBackgroundCompletion(agent, completionKey);
|
|
684
|
+
if (isPersistentBackgroundWakeFailure(event)) {
|
|
685
|
+
log("background completion wake dropped", event?.errorCategory ?? "persistent");
|
|
686
|
+
dropBackgroundCompletion(agent, completionKey);
|
|
687
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
688
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
agent.backgroundCompletionRejectStreak += 1;
|
|
692
|
+
scheduleBackgroundCompletionRetry(agent);
|
|
693
|
+
};
|
|
694
|
+
const rearmBackgroundCompletion = (agent, completionKey = agent.backgroundCompletionInFlight) => {
|
|
695
|
+
if (!completionKey)
|
|
696
|
+
return;
|
|
697
|
+
agent.backgroundCompletionKeys.delete(completionKey);
|
|
698
|
+
if (agent.backgroundCompletionInFlight === completionKey) {
|
|
699
|
+
agent.backgroundCompletionInFlight = null;
|
|
700
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
701
|
+
}
|
|
702
|
+
if (!agent.backgroundCompletionQueue.includes(completionKey)) {
|
|
703
|
+
agent.backgroundCompletionQueue.unshift(completionKey);
|
|
704
|
+
}
|
|
705
|
+
};
|
|
706
|
+
const commitBackgroundCompletion = (agent) => {
|
|
707
|
+
const completionKey = agent.backgroundCompletionInFlight;
|
|
708
|
+
if (!completionKey)
|
|
709
|
+
return;
|
|
710
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
711
|
+
agent.backgroundCompletionQueue.shift();
|
|
712
|
+
agent.backgroundCompletionInFlight = null;
|
|
713
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
714
|
+
agent.backgroundCompletionRejectStreak = 0;
|
|
715
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
716
|
+
};
|
|
717
|
+
const wakeOnBackgroundCompletion = async (agent) => {
|
|
718
|
+
// Caller already reserved submitting/busy so a concurrent idle drain cannot
|
|
719
|
+
// dequeue another head while ensureSession/prompt yields.
|
|
720
|
+
try {
|
|
721
|
+
const session = await ensureSession(agent);
|
|
722
|
+
if (agent.session !== session || agent.stopped)
|
|
723
|
+
return { status: "dropped" };
|
|
724
|
+
const input = options.promptBuilder.buildRuntimeInput("wake", node_crypto_1.default.randomUUID(), { wakeReason: "background subagent completed" });
|
|
725
|
+
const result = await session.prompt(input);
|
|
726
|
+
if (agent.session !== session || agent.stopped)
|
|
727
|
+
return { status: "dropped" };
|
|
728
|
+
if (result.status === "accepted") {
|
|
729
|
+
markSessionStable(agent, session);
|
|
730
|
+
return { status: "accepted", inputId: input.inputId };
|
|
731
|
+
}
|
|
732
|
+
agent.busy = false;
|
|
733
|
+
return { status: "retry" };
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
agent.busy = false;
|
|
737
|
+
log("background completion wake failed", error instanceof Error ? error.message : String(error));
|
|
738
|
+
return { status: "retry" };
|
|
739
|
+
}
|
|
740
|
+
};
|
|
741
|
+
const drainBackgroundCompletionQueue = async (agent) => {
|
|
742
|
+
if (agent.stopped || agent.busy || agent.turnInProgress || agent.submitting)
|
|
743
|
+
return;
|
|
744
|
+
if (agent.backgroundCompletionInFlight || !agent.backgroundCompletionQueue.length)
|
|
745
|
+
return;
|
|
746
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
747
|
+
const proactive = agent.piProactiveCompaction
|
|
748
|
+
&& agent.piProactiveCompactionGeneration === agent.generation
|
|
749
|
+
&& agent.piProactiveCompactionSession === agent.session
|
|
750
|
+
? agent.piProactiveCompaction : null;
|
|
751
|
+
if (proactive && await proactive === "failed")
|
|
752
|
+
return;
|
|
753
|
+
if (agent.adapter.id === "pi" && agent.piProactiveCompactionFailedGeneration === agent.generation)
|
|
754
|
+
return;
|
|
755
|
+
if (agent.adapter.id === "pi" && !agent.busy && !agent.turnInProgress && agent.session) {
|
|
756
|
+
const idleGate = proactivelyCompactPiAtIdle(agent, agent.session);
|
|
757
|
+
if (idleGate && await idleGate === "failed")
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
if (agent.stopped || agent.busy || agent.turnInProgress || agent.submitting)
|
|
761
|
+
return;
|
|
762
|
+
if (agent.backgroundCompletionInFlight || !agent.backgroundCompletionQueue.length)
|
|
763
|
+
return;
|
|
764
|
+
agent.submitting = true;
|
|
765
|
+
agent.busy = true;
|
|
766
|
+
const completionKey = agent.backgroundCompletionQueue[0];
|
|
767
|
+
try {
|
|
768
|
+
const outcome = await wakeOnBackgroundCompletion(agent);
|
|
769
|
+
if (outcome.status === "accepted") {
|
|
770
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
771
|
+
agent.backgroundCompletionInFlight = completionKey;
|
|
772
|
+
agent.backgroundCompletionWakeInputId = outcome.inputId;
|
|
773
|
+
// Keep the reject streak until the wake turn actually succeeds. An accepted
|
|
774
|
+
// prompt that later terminal-fails must not reset the bounded retry budget.
|
|
775
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
776
|
+
}
|
|
777
|
+
else if (outcome.status === "dropped") {
|
|
778
|
+
if (agent.stopped) {
|
|
779
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
780
|
+
agent.backgroundCompletionQueue.shift();
|
|
781
|
+
agent.backgroundCompletionKeys.delete(completionKey);
|
|
782
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
783
|
+
}
|
|
784
|
+
else {
|
|
785
|
+
rearmBackgroundCompletion(agent, completionKey);
|
|
786
|
+
agent.busy = false;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
else {
|
|
790
|
+
agent.busy = false;
|
|
791
|
+
agent.backgroundCompletionRejectStreak += 1;
|
|
792
|
+
// Leave the item queued+deduped. Immediate microtask retries cover a
|
|
793
|
+
// brief reject; after that a bounded timer still drains because Pi will
|
|
794
|
+
// not re-emit the same completion.
|
|
795
|
+
scheduleBackgroundCompletionRetry(agent);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
finally {
|
|
799
|
+
agent.submitting = false;
|
|
800
|
+
}
|
|
801
|
+
};
|
|
802
|
+
const noteBackgroundCompletion = (agent, completionKey) => {
|
|
803
|
+
if (agent.backgroundCompletionKeys.has(completionKey))
|
|
804
|
+
return;
|
|
805
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
806
|
+
if (!agent.backgroundCompletionQueue.includes(completionKey)) {
|
|
807
|
+
agent.backgroundCompletionQueue.push(completionKey);
|
|
808
|
+
}
|
|
809
|
+
if (!agent.busy && !agent.turnInProgress && !agent.submitting && !agent.backgroundCompletionInFlight) {
|
|
810
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
811
|
+
}
|
|
812
|
+
};
|
|
602
813
|
const scheduleRecreate = (agent, reason) => {
|
|
603
814
|
if (agent.stopped || agent.retryTimer || agent.starting || agent.session)
|
|
604
815
|
return;
|
|
@@ -621,6 +832,8 @@ function createRuntimeHost(options) {
|
|
|
621
832
|
if (proactive && await proactive === "failed")
|
|
622
833
|
return;
|
|
623
834
|
await retryPending(agent);
|
|
835
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
836
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
624
837
|
}).catch((error) => {
|
|
625
838
|
const readiness = error instanceof runtime_readiness_js_1.RuntimePrerequisiteError ? error.readiness
|
|
626
839
|
: (0, runtime_readiness_js_1.classifyRuntimePrerequisite)(agent.adapter.id, error);
|
|
@@ -643,6 +856,7 @@ function createRuntimeHost(options) {
|
|
|
643
856
|
agent.busy = false;
|
|
644
857
|
agent.submitting = false;
|
|
645
858
|
agent.generation += 1;
|
|
859
|
+
rearmBackgroundCompletion(agent);
|
|
646
860
|
agent.recreateReason = reason;
|
|
647
861
|
if (agent.stabilityTimer)
|
|
648
862
|
clearTimeout(agent.stabilityTimer);
|
|
@@ -663,6 +877,7 @@ function createRuntimeHost(options) {
|
|
|
663
877
|
agent.busy = false;
|
|
664
878
|
agent.submitting = false;
|
|
665
879
|
agent.generation += 1;
|
|
880
|
+
rearmBackgroundCompletion(agent);
|
|
666
881
|
if (agent.stabilityTimer)
|
|
667
882
|
clearTimeout(agent.stabilityTimer);
|
|
668
883
|
agent.stabilityTimer = null;
|
|
@@ -919,6 +1134,15 @@ function createRuntimeHost(options) {
|
|
|
919
1134
|
agent.busy = false;
|
|
920
1135
|
emit({ type: "activity", agentId: agent.config.agentId, activity: "idle", activityKind: "idle", detailKind: "turn_ended" });
|
|
921
1136
|
reconcileAcceptedAtTurnEnd(agent);
|
|
1137
|
+
if (agent.backgroundCompletionInFlight) {
|
|
1138
|
+
if (agent.turnHadFailure)
|
|
1139
|
+
failBackgroundCompletionWake(agent);
|
|
1140
|
+
else
|
|
1141
|
+
commitBackgroundCompletion(agent);
|
|
1142
|
+
}
|
|
1143
|
+
if (agent.backgroundCompletionQueue.length > 0 && !agent.backgroundCompletionRetryTimer) {
|
|
1144
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
1145
|
+
}
|
|
922
1146
|
telemetry?.runtimeEvent(agent.config.agentId, event);
|
|
923
1147
|
if (recoveredAuthentication) {
|
|
924
1148
|
agent.authFailureActive = false;
|
|
@@ -938,12 +1162,24 @@ function createRuntimeHost(options) {
|
|
|
938
1162
|
});
|
|
939
1163
|
queueMicrotask(() => { void scanAndPromoteAcceptedInboxUpdates(agent); });
|
|
940
1164
|
}
|
|
1165
|
+
else if (event.type === "runtime-observation") {
|
|
1166
|
+
if (event.phase === "completed" && typeof event.completionKey === "string") {
|
|
1167
|
+
noteBackgroundCompletion(agent, event.completionKey);
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
941
1170
|
else if (event.type === "activity") {
|
|
942
1171
|
if (agent.turnInProgress && event.activity !== "internal")
|
|
943
1172
|
agent.turnHadAuthenticatedOutput = true;
|
|
944
1173
|
emit({ type: "activity", agentId: agent.config.agentId, activity: event.activity, activityKind: event.activity });
|
|
945
1174
|
}
|
|
946
1175
|
else if (event.type === "input-error") {
|
|
1176
|
+
if (agent.backgroundCompletionInFlight
|
|
1177
|
+
&& (!event.inputId || !agent.backgroundCompletionWakeInputId
|
|
1178
|
+
|| event.inputId === agent.backgroundCompletionWakeInputId)
|
|
1179
|
+
&& event.willRetry !== true) {
|
|
1180
|
+
agent.turnHadFailure = true;
|
|
1181
|
+
failBackgroundCompletionWake(agent, event);
|
|
1182
|
+
}
|
|
947
1183
|
const record = event.inputId
|
|
948
1184
|
? agent.records.get(event.inputId) ?? [...agent.records.values()].find((candidate) => candidate.input.inputId === event.inputId)
|
|
949
1185
|
: undefined;
|
|
@@ -1270,10 +1506,13 @@ function createRuntimeHost(options) {
|
|
|
1270
1506
|
clearTimeout(previous.retryTimer);
|
|
1271
1507
|
if (previous.stabilityTimer)
|
|
1272
1508
|
clearTimeout(previous.stabilityTimer);
|
|
1509
|
+
if (previous.backgroundCompletionRetryTimer)
|
|
1510
|
+
clearTimeout(previous.backgroundCompletionRetryTimer);
|
|
1273
1511
|
const candidate = {
|
|
1274
1512
|
...previous, config, adapter, session, launchId: node_crypto_1.default.randomUUID(), busy: false, submitting: false,
|
|
1275
1513
|
starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null,
|
|
1276
1514
|
recreateAttempts: 0, stabilityTimer: null, recreateReason: null, stopped: false,
|
|
1515
|
+
backgroundCompletionRetryTimer: null,
|
|
1277
1516
|
disabledReason: null, configurationRecovery: null,
|
|
1278
1517
|
readiness: previous.authFailureActive ? previous.readiness : readiness,
|
|
1279
1518
|
};
|
|
@@ -1290,6 +1529,11 @@ function createRuntimeHost(options) {
|
|
|
1290
1529
|
if (!candidate.authFailureActive) {
|
|
1291
1530
|
emit({ type: "agent-status", agentId: config.agentId, status: "active", readiness: candidate.readiness ?? readiness });
|
|
1292
1531
|
}
|
|
1532
|
+
// Commit clears any inherited backoff timer; reschedule so a queued
|
|
1533
|
+
// completion still drains on the new session.
|
|
1534
|
+
if (candidate.backgroundCompletionQueue.length > 0) {
|
|
1535
|
+
scheduleBackgroundCompletionDrain(candidate);
|
|
1536
|
+
}
|
|
1293
1537
|
await previous.session?.close("runtime candidate committed")
|
|
1294
1538
|
.catch((error) => log("previous runtime close after candidate commit failed", String(error)));
|
|
1295
1539
|
},
|
|
@@ -1622,7 +1866,10 @@ function createRuntimeHost(options) {
|
|
|
1622
1866
|
piProactiveCompaction: null, piProactiveCompactionGeneration: null, piProactiveCompactionSession: null,
|
|
1623
1867
|
piProactiveCompactionFailedGeneration: null, readiness: null,
|
|
1624
1868
|
turnInProgress: false, turnHadFailure: false, turnHadAuthenticatedOutput: false, authFailureActive: false,
|
|
1625
|
-
promotedInboxUpdateIds: new Set()
|
|
1869
|
+
promotedInboxUpdateIds: new Set(),
|
|
1870
|
+
backgroundCompletionQueue: [], backgroundCompletionKeys: new Set(),
|
|
1871
|
+
backgroundCompletionInFlight: null, backgroundCompletionWakeInputId: null,
|
|
1872
|
+
backgroundCompletionRejectStreak: 0, backgroundCompletionRetryTimer: null };
|
|
1626
1873
|
const startupConsumed = [];
|
|
1627
1874
|
const startupQuarantined = [];
|
|
1628
1875
|
for (const record of agent.records.values()) {
|
|
@@ -1773,6 +2020,8 @@ function createRuntimeHost(options) {
|
|
|
1773
2020
|
clearTimeout(agent.retryTimer);
|
|
1774
2021
|
if (agent.stabilityTimer)
|
|
1775
2022
|
clearTimeout(agent.stabilityTimer);
|
|
2023
|
+
if (agent.backgroundCompletionRetryTimer)
|
|
2024
|
+
clearTimeout(agent.backgroundCompletionRetryTimer);
|
|
1776
2025
|
if (agent.busy)
|
|
1777
2026
|
await agent.session?.cancel(reason);
|
|
1778
2027
|
await agent.session?.close(reason);
|
|
@@ -30,6 +30,7 @@ export class RuntimeSessionRecoveryError extends Error {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
const MAX_DELIVERIES = 2048;
|
|
33
|
+
const BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT = 5;
|
|
33
34
|
const now = () => new Date().toISOString();
|
|
34
35
|
const isActiveDelivery = (status) => ["pending", "submitting", "accepted"].includes(status);
|
|
35
36
|
const REPLAY_FAILURE_CODES = new Set([
|
|
@@ -512,6 +513,12 @@ export function createRuntimeHost(options) {
|
|
|
512
513
|
queueMicrotask(() => { void retryPending(agent); });
|
|
513
514
|
}
|
|
514
515
|
queueMicrotask(() => { void scanAndPromoteAcceptedInboxUpdates(agent); });
|
|
516
|
+
// A completion queued during this submit has no turn-end if prompt
|
|
517
|
+
// rejected/threw before turn-start. Drain now that submitting is clear.
|
|
518
|
+
if (!agent.busy && !agent.turnInProgress && agent.backgroundCompletionQueue.length > 0
|
|
519
|
+
&& !agent.backgroundCompletionRetryTimer) {
|
|
520
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
521
|
+
}
|
|
515
522
|
}
|
|
516
523
|
};
|
|
517
524
|
const retryPending = async (agent) => {
|
|
@@ -590,6 +597,210 @@ export function createRuntimeHost(options) {
|
|
|
590
597
|
agent.retryAfterSubmit = true;
|
|
591
598
|
await retryPending(agent);
|
|
592
599
|
};
|
|
600
|
+
const clearBackgroundCompletionRetryTimer = (agent) => {
|
|
601
|
+
if (!agent.backgroundCompletionRetryTimer)
|
|
602
|
+
return;
|
|
603
|
+
clearTimeout(agent.backgroundCompletionRetryTimer);
|
|
604
|
+
agent.backgroundCompletionRetryTimer = null;
|
|
605
|
+
};
|
|
606
|
+
const scheduleBackgroundCompletionDrain = (agent, delayMs = 0) => {
|
|
607
|
+
if (agent.stopped)
|
|
608
|
+
return;
|
|
609
|
+
if (delayMs > 0) {
|
|
610
|
+
if (agent.backgroundCompletionRetryTimer)
|
|
611
|
+
return;
|
|
612
|
+
agent.backgroundCompletionRetryTimer = setTimeout(() => {
|
|
613
|
+
agent.backgroundCompletionRetryTimer = null;
|
|
614
|
+
if (agent.stopped)
|
|
615
|
+
return;
|
|
616
|
+
void drainBackgroundCompletionQueue(agent);
|
|
617
|
+
}, delayMs);
|
|
618
|
+
agent.backgroundCompletionRetryTimer.unref?.();
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
622
|
+
queueMicrotask(() => { void drainBackgroundCompletionQueue(agent); });
|
|
623
|
+
};
|
|
624
|
+
const dropBackgroundCompletion = (agent, completionKey) => {
|
|
625
|
+
if (!completionKey)
|
|
626
|
+
return;
|
|
627
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
628
|
+
agent.backgroundCompletionQueue.shift();
|
|
629
|
+
else {
|
|
630
|
+
const index = agent.backgroundCompletionQueue.indexOf(completionKey);
|
|
631
|
+
if (index >= 0)
|
|
632
|
+
agent.backgroundCompletionQueue.splice(index, 1);
|
|
633
|
+
}
|
|
634
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
635
|
+
if (agent.backgroundCompletionInFlight === completionKey) {
|
|
636
|
+
agent.backgroundCompletionInFlight = null;
|
|
637
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
638
|
+
}
|
|
639
|
+
agent.backgroundCompletionRejectStreak = 0;
|
|
640
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
641
|
+
};
|
|
642
|
+
const scheduleBackgroundCompletionRetry = (agent) => {
|
|
643
|
+
const streak = agent.backgroundCompletionRejectStreak;
|
|
644
|
+
if (streak < BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT) {
|
|
645
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
const delayedAttempt = streak - BACKGROUND_COMPLETION_IMMEDIATE_RETRY_LIMIT;
|
|
649
|
+
if (delayedAttempt >= retryPolicy.maxAttempts) {
|
|
650
|
+
const exhausted = agent.backgroundCompletionQueue[0];
|
|
651
|
+
if (exhausted) {
|
|
652
|
+
log("background completion wake retries exhausted", exhausted);
|
|
653
|
+
dropBackgroundCompletion(agent, exhausted);
|
|
654
|
+
}
|
|
655
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
656
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
const delay = Math.min(retryPolicy.maxDelayMs, retryPolicy.baseDelayMs * 2 ** delayedAttempt);
|
|
660
|
+
scheduleBackgroundCompletionDrain(agent, delay);
|
|
661
|
+
};
|
|
662
|
+
const isPersistentBackgroundWakeFailure = (event) => {
|
|
663
|
+
if (!event)
|
|
664
|
+
return false;
|
|
665
|
+
const category = event.errorCategory;
|
|
666
|
+
if (category === "auth" || category === "billing" || category === "quota")
|
|
667
|
+
return true;
|
|
668
|
+
return category === "provider" && event.retryable === false;
|
|
669
|
+
};
|
|
670
|
+
const failBackgroundCompletionWake = (agent, event) => {
|
|
671
|
+
const completionKey = agent.backgroundCompletionInFlight ?? agent.backgroundCompletionQueue[0];
|
|
672
|
+
if (!completionKey)
|
|
673
|
+
return;
|
|
674
|
+
rearmBackgroundCompletion(agent, completionKey);
|
|
675
|
+
if (isPersistentBackgroundWakeFailure(event)) {
|
|
676
|
+
log("background completion wake dropped", event?.errorCategory ?? "persistent");
|
|
677
|
+
dropBackgroundCompletion(agent, completionKey);
|
|
678
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
679
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
680
|
+
return;
|
|
681
|
+
}
|
|
682
|
+
agent.backgroundCompletionRejectStreak += 1;
|
|
683
|
+
scheduleBackgroundCompletionRetry(agent);
|
|
684
|
+
};
|
|
685
|
+
const rearmBackgroundCompletion = (agent, completionKey = agent.backgroundCompletionInFlight) => {
|
|
686
|
+
if (!completionKey)
|
|
687
|
+
return;
|
|
688
|
+
agent.backgroundCompletionKeys.delete(completionKey);
|
|
689
|
+
if (agent.backgroundCompletionInFlight === completionKey) {
|
|
690
|
+
agent.backgroundCompletionInFlight = null;
|
|
691
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
692
|
+
}
|
|
693
|
+
if (!agent.backgroundCompletionQueue.includes(completionKey)) {
|
|
694
|
+
agent.backgroundCompletionQueue.unshift(completionKey);
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
const commitBackgroundCompletion = (agent) => {
|
|
698
|
+
const completionKey = agent.backgroundCompletionInFlight;
|
|
699
|
+
if (!completionKey)
|
|
700
|
+
return;
|
|
701
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
702
|
+
agent.backgroundCompletionQueue.shift();
|
|
703
|
+
agent.backgroundCompletionInFlight = null;
|
|
704
|
+
agent.backgroundCompletionWakeInputId = null;
|
|
705
|
+
agent.backgroundCompletionRejectStreak = 0;
|
|
706
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
707
|
+
};
|
|
708
|
+
const wakeOnBackgroundCompletion = async (agent) => {
|
|
709
|
+
// Caller already reserved submitting/busy so a concurrent idle drain cannot
|
|
710
|
+
// dequeue another head while ensureSession/prompt yields.
|
|
711
|
+
try {
|
|
712
|
+
const session = await ensureSession(agent);
|
|
713
|
+
if (agent.session !== session || agent.stopped)
|
|
714
|
+
return { status: "dropped" };
|
|
715
|
+
const input = options.promptBuilder.buildRuntimeInput("wake", crypto.randomUUID(), { wakeReason: "background subagent completed" });
|
|
716
|
+
const result = await session.prompt(input);
|
|
717
|
+
if (agent.session !== session || agent.stopped)
|
|
718
|
+
return { status: "dropped" };
|
|
719
|
+
if (result.status === "accepted") {
|
|
720
|
+
markSessionStable(agent, session);
|
|
721
|
+
return { status: "accepted", inputId: input.inputId };
|
|
722
|
+
}
|
|
723
|
+
agent.busy = false;
|
|
724
|
+
return { status: "retry" };
|
|
725
|
+
}
|
|
726
|
+
catch (error) {
|
|
727
|
+
agent.busy = false;
|
|
728
|
+
log("background completion wake failed", error instanceof Error ? error.message : String(error));
|
|
729
|
+
return { status: "retry" };
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
const drainBackgroundCompletionQueue = async (agent) => {
|
|
733
|
+
if (agent.stopped || agent.busy || agent.turnInProgress || agent.submitting)
|
|
734
|
+
return;
|
|
735
|
+
if (agent.backgroundCompletionInFlight || !agent.backgroundCompletionQueue.length)
|
|
736
|
+
return;
|
|
737
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
738
|
+
const proactive = agent.piProactiveCompaction
|
|
739
|
+
&& agent.piProactiveCompactionGeneration === agent.generation
|
|
740
|
+
&& agent.piProactiveCompactionSession === agent.session
|
|
741
|
+
? agent.piProactiveCompaction : null;
|
|
742
|
+
if (proactive && await proactive === "failed")
|
|
743
|
+
return;
|
|
744
|
+
if (agent.adapter.id === "pi" && agent.piProactiveCompactionFailedGeneration === agent.generation)
|
|
745
|
+
return;
|
|
746
|
+
if (agent.adapter.id === "pi" && !agent.busy && !agent.turnInProgress && agent.session) {
|
|
747
|
+
const idleGate = proactivelyCompactPiAtIdle(agent, agent.session);
|
|
748
|
+
if (idleGate && await idleGate === "failed")
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
if (agent.stopped || agent.busy || agent.turnInProgress || agent.submitting)
|
|
752
|
+
return;
|
|
753
|
+
if (agent.backgroundCompletionInFlight || !agent.backgroundCompletionQueue.length)
|
|
754
|
+
return;
|
|
755
|
+
agent.submitting = true;
|
|
756
|
+
agent.busy = true;
|
|
757
|
+
const completionKey = agent.backgroundCompletionQueue[0];
|
|
758
|
+
try {
|
|
759
|
+
const outcome = await wakeOnBackgroundCompletion(agent);
|
|
760
|
+
if (outcome.status === "accepted") {
|
|
761
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
762
|
+
agent.backgroundCompletionInFlight = completionKey;
|
|
763
|
+
agent.backgroundCompletionWakeInputId = outcome.inputId;
|
|
764
|
+
// Keep the reject streak until the wake turn actually succeeds. An accepted
|
|
765
|
+
// prompt that later terminal-fails must not reset the bounded retry budget.
|
|
766
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
767
|
+
}
|
|
768
|
+
else if (outcome.status === "dropped") {
|
|
769
|
+
if (agent.stopped) {
|
|
770
|
+
if (agent.backgroundCompletionQueue[0] === completionKey)
|
|
771
|
+
agent.backgroundCompletionQueue.shift();
|
|
772
|
+
agent.backgroundCompletionKeys.delete(completionKey);
|
|
773
|
+
clearBackgroundCompletionRetryTimer(agent);
|
|
774
|
+
}
|
|
775
|
+
else {
|
|
776
|
+
rearmBackgroundCompletion(agent, completionKey);
|
|
777
|
+
agent.busy = false;
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
else {
|
|
781
|
+
agent.busy = false;
|
|
782
|
+
agent.backgroundCompletionRejectStreak += 1;
|
|
783
|
+
// Leave the item queued+deduped. Immediate microtask retries cover a
|
|
784
|
+
// brief reject; after that a bounded timer still drains because Pi will
|
|
785
|
+
// not re-emit the same completion.
|
|
786
|
+
scheduleBackgroundCompletionRetry(agent);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
finally {
|
|
790
|
+
agent.submitting = false;
|
|
791
|
+
}
|
|
792
|
+
};
|
|
793
|
+
const noteBackgroundCompletion = (agent, completionKey) => {
|
|
794
|
+
if (agent.backgroundCompletionKeys.has(completionKey))
|
|
795
|
+
return;
|
|
796
|
+
agent.backgroundCompletionKeys.add(completionKey);
|
|
797
|
+
if (!agent.backgroundCompletionQueue.includes(completionKey)) {
|
|
798
|
+
agent.backgroundCompletionQueue.push(completionKey);
|
|
799
|
+
}
|
|
800
|
+
if (!agent.busy && !agent.turnInProgress && !agent.submitting && !agent.backgroundCompletionInFlight) {
|
|
801
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
802
|
+
}
|
|
803
|
+
};
|
|
593
804
|
const scheduleRecreate = (agent, reason) => {
|
|
594
805
|
if (agent.stopped || agent.retryTimer || agent.starting || agent.session)
|
|
595
806
|
return;
|
|
@@ -612,6 +823,8 @@ export function createRuntimeHost(options) {
|
|
|
612
823
|
if (proactive && await proactive === "failed")
|
|
613
824
|
return;
|
|
614
825
|
await retryPending(agent);
|
|
826
|
+
if (agent.backgroundCompletionQueue.length > 0)
|
|
827
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
615
828
|
}).catch((error) => {
|
|
616
829
|
const readiness = error instanceof RuntimePrerequisiteError ? error.readiness
|
|
617
830
|
: classifyRuntimePrerequisite(agent.adapter.id, error);
|
|
@@ -634,6 +847,7 @@ export function createRuntimeHost(options) {
|
|
|
634
847
|
agent.busy = false;
|
|
635
848
|
agent.submitting = false;
|
|
636
849
|
agent.generation += 1;
|
|
850
|
+
rearmBackgroundCompletion(agent);
|
|
637
851
|
agent.recreateReason = reason;
|
|
638
852
|
if (agent.stabilityTimer)
|
|
639
853
|
clearTimeout(agent.stabilityTimer);
|
|
@@ -654,6 +868,7 @@ export function createRuntimeHost(options) {
|
|
|
654
868
|
agent.busy = false;
|
|
655
869
|
agent.submitting = false;
|
|
656
870
|
agent.generation += 1;
|
|
871
|
+
rearmBackgroundCompletion(agent);
|
|
657
872
|
if (agent.stabilityTimer)
|
|
658
873
|
clearTimeout(agent.stabilityTimer);
|
|
659
874
|
agent.stabilityTimer = null;
|
|
@@ -910,6 +1125,15 @@ export function createRuntimeHost(options) {
|
|
|
910
1125
|
agent.busy = false;
|
|
911
1126
|
emit({ type: "activity", agentId: agent.config.agentId, activity: "idle", activityKind: "idle", detailKind: "turn_ended" });
|
|
912
1127
|
reconcileAcceptedAtTurnEnd(agent);
|
|
1128
|
+
if (agent.backgroundCompletionInFlight) {
|
|
1129
|
+
if (agent.turnHadFailure)
|
|
1130
|
+
failBackgroundCompletionWake(agent);
|
|
1131
|
+
else
|
|
1132
|
+
commitBackgroundCompletion(agent);
|
|
1133
|
+
}
|
|
1134
|
+
if (agent.backgroundCompletionQueue.length > 0 && !agent.backgroundCompletionRetryTimer) {
|
|
1135
|
+
scheduleBackgroundCompletionDrain(agent);
|
|
1136
|
+
}
|
|
913
1137
|
telemetry?.runtimeEvent(agent.config.agentId, event);
|
|
914
1138
|
if (recoveredAuthentication) {
|
|
915
1139
|
agent.authFailureActive = false;
|
|
@@ -929,12 +1153,24 @@ export function createRuntimeHost(options) {
|
|
|
929
1153
|
});
|
|
930
1154
|
queueMicrotask(() => { void scanAndPromoteAcceptedInboxUpdates(agent); });
|
|
931
1155
|
}
|
|
1156
|
+
else if (event.type === "runtime-observation") {
|
|
1157
|
+
if (event.phase === "completed" && typeof event.completionKey === "string") {
|
|
1158
|
+
noteBackgroundCompletion(agent, event.completionKey);
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
932
1161
|
else if (event.type === "activity") {
|
|
933
1162
|
if (agent.turnInProgress && event.activity !== "internal")
|
|
934
1163
|
agent.turnHadAuthenticatedOutput = true;
|
|
935
1164
|
emit({ type: "activity", agentId: agent.config.agentId, activity: event.activity, activityKind: event.activity });
|
|
936
1165
|
}
|
|
937
1166
|
else if (event.type === "input-error") {
|
|
1167
|
+
if (agent.backgroundCompletionInFlight
|
|
1168
|
+
&& (!event.inputId || !agent.backgroundCompletionWakeInputId
|
|
1169
|
+
|| event.inputId === agent.backgroundCompletionWakeInputId)
|
|
1170
|
+
&& event.willRetry !== true) {
|
|
1171
|
+
agent.turnHadFailure = true;
|
|
1172
|
+
failBackgroundCompletionWake(agent, event);
|
|
1173
|
+
}
|
|
938
1174
|
const record = event.inputId
|
|
939
1175
|
? agent.records.get(event.inputId) ?? [...agent.records.values()].find((candidate) => candidate.input.inputId === event.inputId)
|
|
940
1176
|
: undefined;
|
|
@@ -1261,10 +1497,13 @@ export function createRuntimeHost(options) {
|
|
|
1261
1497
|
clearTimeout(previous.retryTimer);
|
|
1262
1498
|
if (previous.stabilityTimer)
|
|
1263
1499
|
clearTimeout(previous.stabilityTimer);
|
|
1500
|
+
if (previous.backgroundCompletionRetryTimer)
|
|
1501
|
+
clearTimeout(previous.backgroundCompletionRetryTimer);
|
|
1264
1502
|
const candidate = {
|
|
1265
1503
|
...previous, config, adapter, session, launchId: crypto.randomUUID(), busy: false, submitting: false,
|
|
1266
1504
|
starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null,
|
|
1267
1505
|
recreateAttempts: 0, stabilityTimer: null, recreateReason: null, stopped: false,
|
|
1506
|
+
backgroundCompletionRetryTimer: null,
|
|
1268
1507
|
disabledReason: null, configurationRecovery: null,
|
|
1269
1508
|
readiness: previous.authFailureActive ? previous.readiness : readiness,
|
|
1270
1509
|
};
|
|
@@ -1281,6 +1520,11 @@ export function createRuntimeHost(options) {
|
|
|
1281
1520
|
if (!candidate.authFailureActive) {
|
|
1282
1521
|
emit({ type: "agent-status", agentId: config.agentId, status: "active", readiness: candidate.readiness ?? readiness });
|
|
1283
1522
|
}
|
|
1523
|
+
// Commit clears any inherited backoff timer; reschedule so a queued
|
|
1524
|
+
// completion still drains on the new session.
|
|
1525
|
+
if (candidate.backgroundCompletionQueue.length > 0) {
|
|
1526
|
+
scheduleBackgroundCompletionDrain(candidate);
|
|
1527
|
+
}
|
|
1284
1528
|
await previous.session?.close("runtime candidate committed")
|
|
1285
1529
|
.catch((error) => log("previous runtime close after candidate commit failed", String(error)));
|
|
1286
1530
|
},
|
|
@@ -1613,7 +1857,10 @@ export function createRuntimeHost(options) {
|
|
|
1613
1857
|
piProactiveCompaction: null, piProactiveCompactionGeneration: null, piProactiveCompactionSession: null,
|
|
1614
1858
|
piProactiveCompactionFailedGeneration: null, readiness: null,
|
|
1615
1859
|
turnInProgress: false, turnHadFailure: false, turnHadAuthenticatedOutput: false, authFailureActive: false,
|
|
1616
|
-
promotedInboxUpdateIds: new Set()
|
|
1860
|
+
promotedInboxUpdateIds: new Set(),
|
|
1861
|
+
backgroundCompletionQueue: [], backgroundCompletionKeys: new Set(),
|
|
1862
|
+
backgroundCompletionInFlight: null, backgroundCompletionWakeInputId: null,
|
|
1863
|
+
backgroundCompletionRejectStreak: 0, backgroundCompletionRetryTimer: null };
|
|
1617
1864
|
const startupConsumed = [];
|
|
1618
1865
|
const startupQuarantined = [];
|
|
1619
1866
|
for (const record of agent.records.values()) {
|
|
@@ -1764,6 +2011,8 @@ export function createRuntimeHost(options) {
|
|
|
1764
2011
|
clearTimeout(agent.retryTimer);
|
|
1765
2012
|
if (agent.stabilityTimer)
|
|
1766
2013
|
clearTimeout(agent.stabilityTimer);
|
|
2014
|
+
if (agent.backgroundCompletionRetryTimer)
|
|
2015
|
+
clearTimeout(agent.backgroundCompletionRetryTimer);
|
|
1767
2016
|
if (agent.busy)
|
|
1768
2017
|
await agent.session?.cancel(reason);
|
|
1769
2018
|
await agent.session?.close(reason);
|