@songsid/agend 2.3.0 → 2.3.2-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/authority/authority-state.d.ts +4 -0
- package/dist/authority/authority-state.js +16 -10
- package/dist/authority/authority-state.js.map +1 -1
- package/dist/authority/session-authorizations.d.ts +7 -1
- package/dist/authority/session-authorizations.js +6 -4
- package/dist/authority/session-authorizations.js.map +1 -1
- package/dist/authority/web-authority.d.ts +19 -6
- package/dist/authority/web-authority.js +22 -8
- package/dist/authority/web-authority.js.map +1 -1
- package/dist/backend/muse.d.ts +20 -2
- package/dist/backend/muse.js +154 -16
- package/dist/backend/muse.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +2 -0
- package/dist/channel/adapters/discord.js +33 -3
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +1 -1
- package/dist/channel/adapters/telegram.js +3 -35
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/mcp-tools.js +5 -5
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/types.d.ts +11 -0
- package/dist/cli.js +6 -3
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +42 -5
- package/dist/config-validator.js.map +1 -1
- package/dist/daemon.d.ts +47 -3
- package/dist/daemon.js +380 -60
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +23 -1
- package/dist/delivery-outbox.js +114 -17
- package/dist/delivery-outbox.js.map +1 -1
- package/dist/doctor.d.ts +3 -0
- package/dist/doctor.js +9 -0
- package/dist/doctor.js.map +1 -1
- package/dist/event-log.d.ts +5 -0
- package/dist/event-log.js +13 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +1 -1
- package/dist/fleet-lock.d.ts +6 -0
- package/dist/fleet-lock.js +19 -0
- package/dist/fleet-lock.js.map +1 -1
- package/dist/fleet-manager.d.ts +174 -2
- package/dist/fleet-manager.js +678 -68
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-process-handlers.d.ts +1 -0
- package/dist/fleet-process-handlers.js +9 -1
- package/dist/fleet-process-handlers.js.map +1 -1
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +11 -0
- package/dist/general-knowledge/skills/web-appearance/SKILL.canvas-worker-off.md +4 -4
- package/dist/general-knowledge/skills/web-appearance/SKILL.md +7 -7
- package/dist/general-knowledge/skills/web-dashboard/SKILL.canvas-worker-off.md +109 -57
- package/dist/general-knowledge/skills/web-dashboard/SKILL.md +110 -58
- package/dist/guest-seen-people.d.ts +38 -0
- package/dist/guest-seen-people.js +78 -0
- package/dist/guest-seen-people.js.map +1 -0
- package/dist/instance-config-impact.js +4 -1
- package/dist/instance-config-impact.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +2 -0
- package/dist/instance-lifecycle.js +27 -1
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +62 -12
- package/dist/locale.js.map +1 -1
- package/dist/log-service.js +1 -0
- package/dist/log-service.js.map +1 -1
- package/dist/mac-auto-restore.d.ts +117 -0
- package/dist/mac-auto-restore.js +294 -0
- package/dist/mac-auto-restore.js.map +1 -0
- package/dist/needs-you.js +1 -0
- package/dist/needs-you.js.map +1 -1
- package/dist/outbound-handlers.d.ts +2 -2
- package/dist/outbound-handlers.js +5 -5
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +2 -0
- package/dist/outbound-schemas.js +2 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/public-web-gateway.js +9 -1
- package/dist/public-web-gateway.js.map +1 -1
- package/dist/search/search-index.d.ts +10 -2
- package/dist/search/search-index.js +17 -4
- package/dist/search/search-index.js.map +1 -1
- package/dist/service-installer.d.ts +23 -1
- package/dist/service-installer.js +40 -11
- package/dist/service-installer.js.map +1 -1
- package/dist/session-oom-stop.d.ts +45 -0
- package/dist/session-oom-stop.js +259 -0
- package/dist/session-oom-stop.js.map +1 -0
- package/dist/session-oom.d.ts +13 -0
- package/dist/session-oom.js +119 -7
- package/dist/session-oom.js.map +1 -1
- package/dist/settings-api.d.ts +27 -1
- package/dist/settings-api.js +108 -2
- package/dist/settings-api.js.map +1 -1
- package/dist/settings-change.js +37 -3
- package/dist/settings-change.js.map +1 -1
- package/dist/settings-effect.js +5 -2
- package/dist/settings-effect.js.map +1 -1
- package/dist/spawn-gate.d.ts +2 -0
- package/dist/spawn-gate.js +17 -4
- package/dist/spawn-gate.js.map +1 -1
- package/dist/start-attempt.d.ts +15 -1
- package/dist/start-attempt.js +35 -5
- package/dist/start-attempt.js.map +1 -1
- package/dist/start-failure-chat.d.ts +41 -0
- package/dist/start-failure-chat.js +159 -0
- package/dist/start-failure-chat.js.map +1 -0
- package/dist/start-failure-notice.d.ts +4 -0
- package/dist/start-failure-notice.js +8 -0
- package/dist/start-failure-notice.js.map +1 -1
- package/dist/start-failure.d.ts +49 -0
- package/dist/start-failure.js +141 -1
- package/dist/start-failure.js.map +1 -1
- package/dist/telegram-api-root.d.ts +1 -0
- package/dist/telegram-api-root.js +40 -0
- package/dist/telegram-api-root.js.map +1 -0
- package/dist/turn-reply-guard.d.ts +25 -0
- package/dist/turn-reply-guard.js +111 -12
- package/dist/turn-reply-guard.js.map +1 -1
- package/dist/types.d.ts +21 -3
- package/dist/ui/backend-signin.js +5 -2
- package/dist/ui/chat-ambient.js +1 -1
- package/dist/ui/chat-effects.js +4 -4
- package/dist/ui/chat-render.js +5 -2
- package/dist/ui/chat-strings.js +2 -2
- package/dist/ui/chat-thread.js +2 -2
- package/dist/ui/first-run.js +2 -2
- package/dist/ui/fleet-canvas-view.js +114 -0
- package/dist/ui/fleet-canvas.js +15 -77
- package/dist/ui/fleet-office-editor.js +80 -5
- package/dist/ui/fleet-office-layout.js +197 -0
- package/dist/ui/fleet-office-model.js +6 -3
- package/dist/ui/fleet-office-pack.d.ts +39 -0
- package/dist/ui/fleet-office-pack.js +240 -29
- package/dist/ui/fleet-office-skin.js +130 -21
- package/dist/ui/fleet-office-store.js +75 -4
- package/dist/ui/fleet-office-view.js +154 -0
- package/dist/ui/fleet-office-zones.js +95 -33
- package/dist/ui/fleet-office.js +530 -94
- package/dist/ui/fleet-usage.js +7 -6
- package/dist/ui/panel-chat.js +20 -7
- package/dist/ui/panel-details.js +2 -2
- package/dist/ui/panel-fleet.js +11 -77
- package/dist/ui/panel-log.js +5 -5
- package/dist/ui/panel-settings.js +586 -164
- package/dist/ui/search-dialog.js +26 -7
- package/dist/ui/settings-appearance.js +82 -29
- package/dist/ui/settings-apply.js +42 -7
- package/dist/ui/settings-dialogs.js +68 -22
- package/dist/ui/settings-draft.js +51 -0
- package/dist/ui/settings-guests.js +244 -78
- package/dist/ui/settings-learn.js +2 -2
- package/dist/ui/settings-model.js +2 -0
- package/dist/ui/settings-registry.js +142 -0
- package/dist/ui/settings-strings.js +108 -44
- package/dist/ui/settings-web-authority.js +34 -23
- package/dist/ui/shared/app-i18n.js +14 -0
- package/dist/ui/shared/app-nav.js +10 -5
- package/dist/ui/shared/app-prefs.js +80 -2
- package/dist/ui/shared/app-route.js +39 -5
- package/dist/ui/shared/app-shell.js +80 -8
- package/dist/ui/shared/app-stream.js +2 -1
- package/dist/ui/shared/app.css +251 -20
- package/dist/ui/shared/app.js +3 -0
- package/dist/ui/shared/effect-stock-trust.js +1 -1
- package/dist/ui/shared/identity-color.js +4 -4
- package/dist/ui/shared/signin.js +2 -2
- package/dist/ui/shared/ui-dialog.js +6 -1
- package/dist/ui/shared/ui-icons.js +2 -0
- package/dist/ui/shared/usage-dialog.js +21 -11
- package/dist/ui/shared/view-strings.js +2 -0
- package/dist/ui/shared/web-authorize.js +13 -12
- package/dist/web-api.d.ts +9 -3
- package/dist/web-api.js +161 -6
- package/dist/web-api.js.map +1 -1
- package/dist/web-auth.js +7 -1
- package/dist/web-auth.js.map +1 -1
- package/dist/web-experimental.d.ts +1 -1
- package/dist/web-experimental.js +1 -1
- package/dist/web-experimental.js.map +1 -1
- package/dist/web-mod-agent.js +2 -2
- package/dist/web-mod-agent.js.map +1 -1
- package/dist/web-office-skins.d.ts +108 -0
- package/dist/web-office-skins.js +298 -0
- package/dist/web-office-skins.js.map +1 -0
- package/dist/web-office-store.d.ts +8 -2
- package/dist/web-office-store.js +37 -7
- package/dist/web-office-store.js.map +1 -1
- package/dist/web-settings-patch.d.ts +6 -0
- package/dist/web-settings-patch.js +61 -4
- package/dist/web-settings-patch.js.map +1 -1
- package/dist/web-shell-routes.d.ts +4 -2
- package/dist/web-shell-routes.js +17 -7
- package/dist/web-shell-routes.js.map +1 -1
- package/dist/working-recovery.d.ts +104 -0
- package/dist/working-recovery.js +263 -0
- package/dist/working-recovery.js.map +1 -0
- package/dist/working-state-journal.d.ts +70 -0
- package/dist/working-state-journal.js +106 -0
- package/dist/working-state-journal.js.map +1 -0
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -10,7 +10,8 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
|
10
10
|
import { EventEmitter } from "node:events";
|
|
11
11
|
import { rotateLogIfNeededAsync } from "./logger.js";
|
|
12
12
|
import { coredumpFilterLaunchPrefix } from "./coredump-filter.js";
|
|
13
|
-
import { observeCliOomIdentity, confirmedCliOom, consecutiveSessionOoms } from "./session-oom.js";
|
|
13
|
+
import { observeCliOomIdentity, confirmedCliOom, consecutiveSessionOoms, describeSelfOom } from "./session-oom.js";
|
|
14
|
+
import { assertNoSessionOomStop, readSessionOomStop, writeSessionOomStop, SessionOomStoppedError, sessionOomStopHint } from "./session-oom-stop.js";
|
|
14
15
|
import { mcpServerState } from "./mcp-liveness.js";
|
|
15
16
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
16
17
|
import { writeBackgroundWorkEndedMarker } from "./background-work-ended.js";
|
|
@@ -45,7 +46,7 @@ import { buildFleetInstructions } from "./instructions.js";
|
|
|
45
46
|
import { formatHumanInboundMessage, prepareDeliveryWire, steerInboundEnvelope, BTW_INBOUND_PREFIX, humanBodyHash, humanOwnerObjectId } from "./human-delivery-receipts.js";
|
|
46
47
|
import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
|
|
47
48
|
import { agendMessageInInput, bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignature, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
|
|
48
|
-
import { TurnReplyGuard } from "./turn-reply-guard.js";
|
|
49
|
+
import { TurnReplyGuard, formatReplyDropDetection, turnExpectsPersonReply } from "./turn-reply-guard.js";
|
|
49
50
|
import { t } from "./locale.js";
|
|
50
51
|
import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
|
|
51
52
|
import { deliveryContentDigest } from "./delivery-outbox.js";
|
|
@@ -58,6 +59,7 @@ import { buildResumeContinuation, clearInFlightTurnMarker, consumeInFlightTurnMa
|
|
|
58
59
|
import { codexLiveRowMatches } from "./backend/codex.js";
|
|
59
60
|
import { codexPackageProblemMessage } from "./backend/codex-package.js";
|
|
60
61
|
import { claudeConfigDirFor, claudeProfileConfigDir, claudeSwitchDialog, claudeSwitchTargetMatches } from "./backend/claude-code.js";
|
|
62
|
+
import { MuseBackend } from "./backend/muse.js";
|
|
61
63
|
import { InteractionObservation, INTERACTION_CONFIRM_MS, sameInteractionOwner } from "./interaction-observation.js";
|
|
62
64
|
import { unrecognisedPromptShape } from "./unrecognised-prompt.js";
|
|
63
65
|
/** #2140: deliveries that may carry an unwritten what's-new notice before it is recorded anyway. */
|
|
@@ -1163,6 +1165,25 @@ export class Daemon extends EventEmitter {
|
|
|
1163
1165
|
crashCount = 0;
|
|
1164
1166
|
cliOomObservation = null;
|
|
1165
1167
|
consecutiveOoms = null;
|
|
1168
|
+
collateralOomRecovery = false;
|
|
1169
|
+
unclearOomStopped = false;
|
|
1170
|
+
sessionOomStopUnpersisted = false;
|
|
1171
|
+
assertSessionOomAdmission() {
|
|
1172
|
+
if (this.sessionOomStopUnpersisted)
|
|
1173
|
+
throw new SessionOomStoppedError(false, { blocked: true, record: null, revision: null, confirmedOom: true });
|
|
1174
|
+
assertNoSessionOomStop(this.instanceDir);
|
|
1175
|
+
}
|
|
1176
|
+
isSessionOomStopped() {
|
|
1177
|
+
return this.sessionOomStopUnpersisted || readSessionOomStop(this.instanceDir).blocked;
|
|
1178
|
+
}
|
|
1179
|
+
isUnclearOomStopped() {
|
|
1180
|
+
return this.unclearOomStopped && this.healthCheckPaused && this.processStatus === "crashed";
|
|
1181
|
+
}
|
|
1182
|
+
stopUnclearOom(exitCode) {
|
|
1183
|
+
this.unclearOomStopped = true;
|
|
1184
|
+
this.healthCheckPaused = true;
|
|
1185
|
+
this.emitSupervisionEnded(t("inst.oom_unclear_stopped"), t("inst.session_oom_remedy"), exitCode, "oom_unclear");
|
|
1186
|
+
}
|
|
1166
1187
|
oomSessionKey() {
|
|
1167
1188
|
// Claude's per-instance statusline owns its ID. Other current getters
|
|
1168
1189
|
// either return null or select by cwd (Muse/Grok): those are not proof
|
|
@@ -1224,6 +1245,8 @@ export class Daemon extends EventEmitter {
|
|
|
1224
1245
|
* skipResume — the #1209 gate reads this, never the resettable flag.
|
|
1225
1246
|
*/
|
|
1226
1247
|
bootSkippedResume = false;
|
|
1248
|
+
recoveryCrashLoopFresh = false;
|
|
1249
|
+
recoveryFreshChoice = false;
|
|
1227
1250
|
startupAborted = false;
|
|
1228
1251
|
/** First time the current on-screen blocking dialog was seen (0 = none); drives the parked report. */
|
|
1229
1252
|
dialogParkedSince = 0;
|
|
@@ -1810,6 +1833,12 @@ export class Daemon extends EventEmitter {
|
|
|
1810
1833
|
model: config.model ?? "default",
|
|
1811
1834
|
};
|
|
1812
1835
|
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
1836
|
+
if (backend instanceof MuseBackend) {
|
|
1837
|
+
const log = this.logger;
|
|
1838
|
+
backend.setPlaceholderDebug((rec) => {
|
|
1839
|
+
log.debug(rec, "muse empty-composer placeholder classified as empty");
|
|
1840
|
+
});
|
|
1841
|
+
}
|
|
1813
1842
|
this.tmuxSessionName = getTmuxSession();
|
|
1814
1843
|
this.messageBus = new MessageBus();
|
|
1815
1844
|
this.messageBus.setLogger(this.logger);
|
|
@@ -1822,7 +1851,10 @@ export class Daemon extends EventEmitter {
|
|
|
1822
1851
|
// finished turn can never resume after a later restart. reset() is
|
|
1823
1852
|
// deliberately NOT hooked: stop/spawn abandon the in-memory turn while
|
|
1824
1853
|
// the on-disk interruption must survive for the resume gate.
|
|
1825
|
-
this.turnReplyGuard.onComplete = () =>
|
|
1854
|
+
this.turnReplyGuard.onComplete = () => {
|
|
1855
|
+
clearInFlightTurnMarker(this.instanceDir);
|
|
1856
|
+
this.publishWorkingState("turn_closed");
|
|
1857
|
+
};
|
|
1826
1858
|
}
|
|
1827
1859
|
/** FleetManager and this daemon share a process, so durable begin/ACK is a direct synchronous store call. */
|
|
1828
1860
|
setDeliveryOutboxPort(port) {
|
|
@@ -2117,6 +2149,11 @@ export class Daemon extends EventEmitter {
|
|
|
2117
2149
|
* daemon's own state only; it never inspects the pane itself.
|
|
2118
2150
|
*/
|
|
2119
2151
|
notAcceptingReason() {
|
|
2152
|
+
if (this.sessionOomStopUnpersisted)
|
|
2153
|
+
return t("inst.session_oom_hold");
|
|
2154
|
+
const oomStop = readSessionOomStop(this.instanceDir);
|
|
2155
|
+
if (oomStop.blocked)
|
|
2156
|
+
return sessionOomStopHint(oomStop);
|
|
2120
2157
|
if (this.pauseWakeState !== "active")
|
|
2121
2158
|
return t("delivery.hold_state", this.pauseWakeState);
|
|
2122
2159
|
if (this.inputBlockedDialogKey !== null)
|
|
@@ -2167,6 +2204,8 @@ export class Daemon extends EventEmitter {
|
|
|
2167
2204
|
}
|
|
2168
2205
|
}
|
|
2169
2206
|
async startAdmitted() {
|
|
2207
|
+
this.assertSessionOomAdmission();
|
|
2208
|
+
this.recoveryBootWorkPending = true;
|
|
2170
2209
|
ensureInstanceDir(this.instanceDir);
|
|
2171
2210
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
2172
2211
|
this.logger.info(`Starting ${this.name}`);
|
|
@@ -2175,6 +2214,8 @@ export class Daemon extends EventEmitter {
|
|
|
2175
2214
|
// even when the JSON is corrupt so it cannot remain forever.
|
|
2176
2215
|
const crashStatePath = join(this.instanceDir, "crash-state.json");
|
|
2177
2216
|
this.bootSkippedResume = false;
|
|
2217
|
+
this.recoveryCrashLoopFresh = false;
|
|
2218
|
+
this.recoveryFreshChoice = false;
|
|
2178
2219
|
try {
|
|
2179
2220
|
if (existsSync(crashStatePath)) {
|
|
2180
2221
|
const content = readFileSync(crashStatePath, "utf-8");
|
|
@@ -2183,6 +2224,10 @@ export class Daemon extends EventEmitter {
|
|
|
2183
2224
|
if (state?.resumeDisabled === true) {
|
|
2184
2225
|
this.skipResume = true;
|
|
2185
2226
|
this.bootSkippedResume = true;
|
|
2227
|
+
this.recoveryCrashLoopFresh = (state.reason === undefined || state.reason === "crash_loop")
|
|
2228
|
+
&& Number.isInteger(state.crashesInWindow) && state.crashesInWindow >= 3
|
|
2229
|
+
&& Number.isFinite(state.lastCrashAt) && state.lastCrashAt > 0;
|
|
2230
|
+
this.recoveryFreshChoice = !this.recoveryCrashLoopFresh;
|
|
2186
2231
|
this.logger.warn("Previous crash loop detected — starting without resume");
|
|
2187
2232
|
}
|
|
2188
2233
|
}
|
|
@@ -2331,12 +2376,19 @@ export class Daemon extends EventEmitter {
|
|
|
2331
2376
|
// #1209: conditional continuation for the turn interrupted by the last
|
|
2332
2377
|
// restart, if the gates allow. Detached like the warmup notice; the gate
|
|
2333
2378
|
// consumes its one-shot marker first and never throws into boot.
|
|
2334
|
-
|
|
2379
|
+
const legacyResume = this.maybeResumeInterruptedTurn().catch(() => "unknown");
|
|
2335
2380
|
// Warmup: wait for CLI idle, then trigger steering reload — but only when
|
|
2336
2381
|
// the instructions actually changed since the agent last saw them.
|
|
2337
2382
|
// Skipping the no-op reload saves 10-30s of agent time on every restart
|
|
2338
2383
|
// where instructions are unchanged.
|
|
2339
|
-
|
|
2384
|
+
const warmup = this.runWarmupInstructionNotice();
|
|
2385
|
+
const recoveryOwner = JSON.stringify(this.interactionOwner());
|
|
2386
|
+
void Promise.allSettled([legacyResume, warmup]).then(() => {
|
|
2387
|
+
if (JSON.stringify(this.interactionOwner()) === recoveryOwner) {
|
|
2388
|
+
this.recoveryBootWorkPending = false;
|
|
2389
|
+
this.recoveryBootSettled = true;
|
|
2390
|
+
}
|
|
2391
|
+
});
|
|
2340
2392
|
if (!this.config.lightweight) {
|
|
2341
2393
|
// 3. Pipe-pane for prompt detection. Rotate first so a ballooned log from a
|
|
2342
2394
|
// previous stuck splash (hundreds of MB of ANSI frames) is truncated before
|
|
@@ -3094,19 +3146,36 @@ export class Daemon extends EventEmitter {
|
|
|
3094
3146
|
return;
|
|
3095
3147
|
this.cliOomObservation = null;
|
|
3096
3148
|
const decisionKey = this.oomSessionKey();
|
|
3097
|
-
this.consecutiveOoms = consecutiveSessionOoms(this.consecutiveOoms, decisionKey, decisionKey === key && oom
|
|
3149
|
+
this.consecutiveOoms = consecutiveSessionOoms(this.consecutiveOoms, decisionKey, decisionKey === key && oom?.attribution === "self");
|
|
3150
|
+
const collateralOom = oom?.attribution === "collateral";
|
|
3151
|
+
if (collateralOom)
|
|
3152
|
+
this.logger.warn({ attribution: oom.attribution, pid: oom.pid }, "OOM victim without self-caused proof — preserving session; waiting for memory recovery");
|
|
3098
3153
|
if (this.consecutiveOoms && this.consecutiveOoms.count >= 2) {
|
|
3099
3154
|
// Stop supervision, not an idle pause: queued work must not auto-wake
|
|
3100
3155
|
// the same session. An explicit restart is the person's retry choice.
|
|
3156
|
+
// No await after the final captured health owner/session check. The
|
|
3157
|
+
// stop is authoritative before observers can see supervision end.
|
|
3158
|
+
try {
|
|
3159
|
+
writeSessionOomStop(this.instanceDir, { owner: healthOwner, sessionKey: decisionKey,
|
|
3160
|
+
configuration: this.getWorkingStateObservation().configuration, stoppedAt: Date.now(),
|
|
3161
|
+
count: this.consecutiveOoms.count, victim: { pid: oom.pid, at: oom.at }, evidence: oom.evidence });
|
|
3162
|
+
}
|
|
3163
|
+
catch {
|
|
3164
|
+
// Storage failure cannot revive this CLI in the current process.
|
|
3165
|
+
// Say explicitly that a fleet restart cannot yet rely on durability.
|
|
3166
|
+
this.sessionOomStopUnpersisted = true;
|
|
3167
|
+
this.logger.error("Could not persist session OOM stop; fix storage before restarting the fleet");
|
|
3168
|
+
}
|
|
3101
3169
|
this.healthCheckPaused = true;
|
|
3102
3170
|
try {
|
|
3103
|
-
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: "session_oom" });
|
|
3171
|
+
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: "session_oom", oomEvidence: oom.evidence });
|
|
3104
3172
|
}
|
|
3105
3173
|
catch (error) {
|
|
3106
3174
|
this.logger.warn({ error }, "Could not append session OOM history");
|
|
3107
3175
|
}
|
|
3108
3176
|
this.logger.error({ pid: oom.pid, count: this.consecutiveOoms.count }, "Repeated confirmed CLI OOM — automatic resume stopped; session preserved");
|
|
3109
|
-
this.emitSupervisionEnded(t("inst.session_oom_stopped")
|
|
3177
|
+
this.emitSupervisionEnded(`${t("inst.session_oom_stopped")} (${describeSelfOom(oom.evidence)})`, t("inst.session_oom_remedy")
|
|
3178
|
+
+ (this.sessionOomStopUnpersisted ? ` ${t("inst.session_oom_save_failed")}` : ""), exitCode, "session_oom");
|
|
3110
3179
|
return;
|
|
3111
3180
|
}
|
|
3112
3181
|
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
@@ -3116,7 +3185,7 @@ export class Daemon extends EventEmitter {
|
|
|
3116
3185
|
return;
|
|
3117
3186
|
}
|
|
3118
3187
|
// Detect claude-code background session conflict — recover without counting as crash
|
|
3119
|
-
if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
3188
|
+
if (!oom && lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
3120
3189
|
if (!this.backgroundSessionRecoveryAttempted) {
|
|
3121
3190
|
this.backgroundSessionRecoveryAttempted = true;
|
|
3122
3191
|
this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
|
|
@@ -3156,7 +3225,7 @@ export class Daemon extends EventEmitter {
|
|
|
3156
3225
|
// --continue again and crash in the same way → loop. Clear the session id
|
|
3157
3226
|
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
3158
3227
|
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
3159
|
-
if (this.paneSaysNoConversation(lastOutput)) {
|
|
3228
|
+
if (!oom && this.paneSaysNoConversation(lastOutput)) {
|
|
3160
3229
|
this.setSessionAside("cli_reported_no_conversation");
|
|
3161
3230
|
this.skipResume = true;
|
|
3162
3231
|
}
|
|
@@ -3177,12 +3246,21 @@ export class Daemon extends EventEmitter {
|
|
|
3177
3246
|
return;
|
|
3178
3247
|
}
|
|
3179
3248
|
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
3180
|
-
|
|
3249
|
+
if (!collateralOom)
|
|
3250
|
+
this.crashTimestamps.push(Date.now());
|
|
3251
|
+
else
|
|
3252
|
+
this.crashTimestamps = [];
|
|
3181
3253
|
const crashWindowMs = 5 * 60_000;
|
|
3182
3254
|
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
3183
3255
|
if (this.crashTimestamps.length >= 3) {
|
|
3184
3256
|
this.healthCheckPaused = true;
|
|
3185
3257
|
this.logger.error({ crashesInWindow: this.crashTimestamps.length, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "3+ crashes in 5 minutes — pausing respawn");
|
|
3258
|
+
// Unknown cause is bounded like an ordinary crash, never a
|
|
3259
|
+
// self-OOM verdict. Preserve the session for the person's choice.
|
|
3260
|
+
if (oom?.attribution === "unknown") {
|
|
3261
|
+
this.stopUnclearOom(exitCode);
|
|
3262
|
+
return;
|
|
3263
|
+
}
|
|
3186
3264
|
// P1: Persist crash state so next process restart skips resume
|
|
3187
3265
|
try {
|
|
3188
3266
|
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
@@ -3199,18 +3277,23 @@ export class Daemon extends EventEmitter {
|
|
|
3199
3277
|
if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
|
|
3200
3278
|
this.crashCount = 0;
|
|
3201
3279
|
}
|
|
3202
|
-
|
|
3280
|
+
if (!collateralOom)
|
|
3281
|
+
this.crashCount++;
|
|
3203
3282
|
this.lastCrashAt = Date.now();
|
|
3204
|
-
if (this.crashCount > max_retries) {
|
|
3283
|
+
if (!collateralOom && this.crashCount > max_retries) {
|
|
3205
3284
|
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "Max crash retries exceeded — not respawning");
|
|
3206
3285
|
this.healthCheckPaused = true;
|
|
3286
|
+
if (oom?.attribution === "unknown") {
|
|
3287
|
+
this.stopUnclearOom(exitCode);
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3207
3290
|
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
3208
3291
|
return; // don't schedule next — given up
|
|
3209
3292
|
}
|
|
3210
3293
|
// Calculate backoff delay
|
|
3211
3294
|
const delay = backoff === "exponential"
|
|
3212
|
-
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
3213
|
-
: 1000 * this.crashCount;
|
|
3295
|
+
? Math.min(1000 * Math.pow(2, Math.max(1, this.crashCount) - 1), 60_000)
|
|
3296
|
+
: 1000 * Math.max(1, this.crashCount);
|
|
3214
3297
|
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
3215
3298
|
await new Promise(r => setTimeout(r, delay));
|
|
3216
3299
|
try {
|
|
@@ -3255,7 +3338,14 @@ export class Daemon extends EventEmitter {
|
|
|
3255
3338
|
// Write snapshot before spawn — consumed only if resume fails
|
|
3256
3339
|
this.writeRotationSnapshot("crash");
|
|
3257
3340
|
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
3258
|
-
|
|
3341
|
+
this.collateralOomRecovery = collateralOom;
|
|
3342
|
+
let resumed;
|
|
3343
|
+
try {
|
|
3344
|
+
resumed = await this.spawnClaudeWindow();
|
|
3345
|
+
}
|
|
3346
|
+
finally {
|
|
3347
|
+
this.collateralOomRecovery = false;
|
|
3348
|
+
}
|
|
3259
3349
|
if (!resumed) {
|
|
3260
3350
|
// Resume failed → fresh session → inject snapshot for context
|
|
3261
3351
|
await this.injectSnapshotMessage();
|
|
@@ -3349,6 +3439,7 @@ export class Daemon extends EventEmitter {
|
|
|
3349
3439
|
* four, carrying a human-readable cause and the operator's next step.
|
|
3350
3440
|
*/
|
|
3351
3441
|
emitSupervisionEnded(reason, remedy, exitCode, cause) {
|
|
3442
|
+
this.publishWorkingState("supervision_ended");
|
|
3352
3443
|
this.emit("supervision_ended", {
|
|
3353
3444
|
name: this.name,
|
|
3354
3445
|
reason,
|
|
@@ -3361,6 +3452,7 @@ export class Daemon extends EventEmitter {
|
|
|
3361
3452
|
if (this.processStatus === status)
|
|
3362
3453
|
return;
|
|
3363
3454
|
this.processStatus = status;
|
|
3455
|
+
this.publishWorkingState("process");
|
|
3364
3456
|
if (status === "running") {
|
|
3365
3457
|
// A successful crash respawn is a new pane generation. It is ready by
|
|
3366
3458
|
// this point, so an error gate inherited from the dead pane must not keep
|
|
@@ -3898,6 +3990,7 @@ export class Daemon extends EventEmitter {
|
|
|
3898
3990
|
clearPendingDeliveries(fleetEpoch) {
|
|
3899
3991
|
// FleetManager calls this synchronously before sendEscape. Cancel only the
|
|
3900
3992
|
// current human turn; leave already-forwarded tool requests to settle.
|
|
3993
|
+
this.recoveryCancelled = true;
|
|
3901
3994
|
this.turnReplyGuard.cancelByUser();
|
|
3902
3995
|
// #1241: a cancelled turn never confirms — its next edge completes it as
|
|
3903
3996
|
// intentional. Drop a pending confirmation so it cannot promote afterwards.
|
|
@@ -3909,6 +4002,8 @@ export class Daemon extends EventEmitter {
|
|
|
3909
4002
|
this.deliveryEpoch = fleetEpoch === undefined
|
|
3910
4003
|
? this.deliveryEpoch + 1
|
|
3911
4004
|
: Math.max(this.deliveryEpoch, fleetEpoch);
|
|
4005
|
+
this.cancelCapacityNudge("the user cancelled");
|
|
4006
|
+
this.publishWorkingState("cancelled");
|
|
3912
4007
|
this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
|
|
3913
4008
|
}
|
|
3914
4009
|
/** Board 907c7899: the layout a web /compact's end can be read from on this pane (kiro classic or TUI V2), or null. */
|
|
@@ -4477,6 +4572,7 @@ export class Daemon extends EventEmitter {
|
|
|
4477
4572
|
return true;
|
|
4478
4573
|
}
|
|
4479
4574
|
async stop() {
|
|
4575
|
+
this.publishWorkingState("before_stop");
|
|
4480
4576
|
this.logger.info("Stopping daemon instance");
|
|
4481
4577
|
this.clearQuitRelaunchWatch();
|
|
4482
4578
|
this.fenceDeliveryWritesForStop();
|
|
@@ -4625,6 +4721,114 @@ export class Daemon extends EventEmitter {
|
|
|
4625
4721
|
getInteractionSnapshot() {
|
|
4626
4722
|
return this.interactionObservation.snapshot(this.interactionOwner());
|
|
4627
4723
|
}
|
|
4724
|
+
ownedWorkingSession = null;
|
|
4725
|
+
recoveryBootSettled = false;
|
|
4726
|
+
recoveryBootWorkPending = false;
|
|
4727
|
+
recoveryLegacyCovered = false;
|
|
4728
|
+
recoveryNativeWork = false;
|
|
4729
|
+
recoveryContinuationId = null;
|
|
4730
|
+
recoveryCancelled = false;
|
|
4731
|
+
workingRecoveryInputOwner = null;
|
|
4732
|
+
noteLegacyRestartReminder() { this.recoveryLegacyCovered = true; }
|
|
4733
|
+
getWorkingRecoveryState() {
|
|
4734
|
+
const observation = this.getWorkingStateObservation();
|
|
4735
|
+
const owner = JSON.stringify(observation.owner), ownsInput = this.workingRecoveryInputOwner === owner;
|
|
4736
|
+
return { owner, configuration: observation.configuration,
|
|
4737
|
+
sessionId: this.ownedWorkingSession, idle: (ownsInput || this.instanceState === "idle") && this.pasteQueueDepth === 0,
|
|
4738
|
+
paused: this.isPaused, stopped: this.isSessionOomStopped(), legacyPending: !this.recoveryBootSettled,
|
|
4739
|
+
legacyCovered: this.recoveryLegacyCovered, nativeWork: !ownsInput && this.recoveryNativeWork,
|
|
4740
|
+
freshChoice: this.recoveryFreshChoice || this.isNewSession && !this.bootSkippedResume, crashLoopFresh: this.recoveryCrashLoopFresh,
|
|
4741
|
+
cancelled: this.recoveryCancelled, background: this.workingBackgroundObservation };
|
|
4742
|
+
}
|
|
4743
|
+
/** Local system work: no user turn, channel reply target, wake or replay of an original request. */
|
|
4744
|
+
async deliverWorkingRecovery(content, delivery, admitted) {
|
|
4745
|
+
const tmux = this.tmux, owner = JSON.stringify(this.interactionOwner()), epoch = this.deliveryEpoch;
|
|
4746
|
+
const current = () => this.tmux === tmux && owner === JSON.stringify(this.interactionOwner())
|
|
4747
|
+
&& this.deliveryEpoch === epoch && !this.deliveryWritesStopping && !this.runtimeMonitorsFrozen
|
|
4748
|
+
&& !this.spawning && !this.startupAborted && !this.isPaused && !this.healthCheckPaused
|
|
4749
|
+
&& this.processStatus === "running" && this.pasteQueueDepth === 0
|
|
4750
|
+
&& (this.workingRecoveryInputOwner === owner || this.instanceState === "idle" && !this.recoveryNativeWork)
|
|
4751
|
+
&& this.recoveryBootSettled && !this.recoveryCancelled && !this.isSessionOomStopped() && admitted();
|
|
4752
|
+
let begun = false;
|
|
4753
|
+
try {
|
|
4754
|
+
await this.paneWriteLock.run(async () => {
|
|
4755
|
+
if (!tmux || !current())
|
|
4756
|
+
return;
|
|
4757
|
+
const pane = await tmux.capturePane();
|
|
4758
|
+
const input = this.backend?.readInputRow?.(pane);
|
|
4759
|
+
if (!current() || !input || input.text !== "" || input.collapsedPastes !== 0 || input.queued
|
|
4760
|
+
|| !this.paneAuthoritativelyIdle(pane) || this.inputTransientInPane(pane)
|
|
4761
|
+
|| this.backend?.backgroundWorkRunning?.(pane) === true)
|
|
4762
|
+
return;
|
|
4763
|
+
if (await tmux.getPaneInputMode?.() !== "raw" || !current())
|
|
4764
|
+
return;
|
|
4765
|
+
const evidence = await this.durableAttemptEvidence(tmux.getWindowId(), false, false);
|
|
4766
|
+
if (!current())
|
|
4767
|
+
return;
|
|
4768
|
+
const signature = this.submissionSignature(content), baseline = this.paneEvidence(pane, signature);
|
|
4769
|
+
const attempt = { ...delivery, contentSha256: deliveryContentDigest(content) };
|
|
4770
|
+
if (!this.beginDurableDelivery(attempt, evidence))
|
|
4771
|
+
return;
|
|
4772
|
+
begun = true;
|
|
4773
|
+
// Own PTY input immediately produces a working observation. It must not
|
|
4774
|
+
// cancel this same fenced write; external queue/cancel/owner guards stay.
|
|
4775
|
+
this.workingRecoveryInputOwner = owner;
|
|
4776
|
+
// Conservatively commit possible input BEFORE paste, not merely before Enter.
|
|
4777
|
+
// A process death at any later boundary must never replay this continuation.
|
|
4778
|
+
if (!this.deliveryOutbox?.markEnterStarted(delivery.deliveryId, this.bootId, delivery.attemptNo))
|
|
4779
|
+
return;
|
|
4780
|
+
if (!current() || !await tmux.pasteBuffer(content, { guard: current }) || !current())
|
|
4781
|
+
return;
|
|
4782
|
+
await new Promise(resolve => setTimeout(resolve, NORMAL_ENTER_SETTLE_MS));
|
|
4783
|
+
if (!current() || !await this.sendDeliveryEnterAfterAvailability("working-recovery", current, attempt))
|
|
4784
|
+
return;
|
|
4785
|
+
const proof = await this.confirmSubmitted(signature, baseline);
|
|
4786
|
+
const committed = this.deliveryOutbox.complete(delivery.deliveryId, this.bootId, delivery.attemptNo, proof === "submitted" ? "delivered" : "uncertain", "local continuation Enter attempted once; no replay");
|
|
4787
|
+
if (committed && proof === "submitted" && current()) {
|
|
4788
|
+
this.recoveryContinuationId = delivery.deliveryId;
|
|
4789
|
+
this.publishWorkingState("execution");
|
|
4790
|
+
}
|
|
4791
|
+
});
|
|
4792
|
+
}
|
|
4793
|
+
catch { /* The durable fence below owns ambiguity; never retry a begun write. */ }
|
|
4794
|
+
finally {
|
|
4795
|
+
if (this.workingRecoveryInputOwner === owner) {
|
|
4796
|
+
this.workingRecoveryInputOwner = null;
|
|
4797
|
+
if (this.tmux === tmux && owner === JSON.stringify(this.interactionOwner()))
|
|
4798
|
+
this.recoveryNativeWork = true;
|
|
4799
|
+
}
|
|
4800
|
+
if (begun)
|
|
4801
|
+
this.deliveryOutbox?.complete(delivery.deliveryId, this.bootId, delivery.attemptNo, "uncertain", "local continuation may have reached input; not replayed");
|
|
4802
|
+
else
|
|
4803
|
+
this.deliveryOutbox?.retryBeforeBegin(delivery.deliveryId, this.bootId, delivery.attemptNo, "working recovery prerequisite changed before input", 5_000);
|
|
4804
|
+
}
|
|
4805
|
+
}
|
|
4806
|
+
workingTurnDelivery = null;
|
|
4807
|
+
workingBackgroundObservation = null;
|
|
4808
|
+
/** Pure metadata read: never discovers sessions, captures a pane or reads backend files. */
|
|
4809
|
+
getWorkingStateObservation() {
|
|
4810
|
+
const turn = this.turnReplyGuard.snapshot();
|
|
4811
|
+
const backend = this.config.backend ?? this.backend?.binaryName ?? "unknown";
|
|
4812
|
+
const profile = resolveCredentialProfile(this.config.backend_options?.[backend]);
|
|
4813
|
+
return {
|
|
4814
|
+
owner: this.interactionOwner(), backend,
|
|
4815
|
+
configuration: createHash("sha256").update(JSON.stringify([backend, this.config.working_directory, profile])).digest("hex"),
|
|
4816
|
+
executionState: this.processStatus === "running" ? this.isPaused ? "paused" : this.instanceState : null,
|
|
4817
|
+
processStatus: this.getProcessStatus(), sessionId: this.ownedWorkingSession, backendTurnId: null,
|
|
4818
|
+
...(this.recoveryContinuationId ? { recoveryDeliveryId: this.recoveryContinuationId } : {}),
|
|
4819
|
+
backgroundWork: this.workingBackgroundObservation,
|
|
4820
|
+
turn: turn ? { generation: turn.generation,
|
|
4821
|
+
deliveryId: this.workingTurnDelivery?.generation === turn.generation ? this.workingTurnDelivery.deliveryId : null,
|
|
4822
|
+
correlationId: turn.target.correlationId ?? null, messageId: turn.target.messageId ?? null,
|
|
4823
|
+
cancelled: turn.cancelledByUser, completionDelivered: turn.completionDelivered } : null,
|
|
4824
|
+
};
|
|
4825
|
+
}
|
|
4826
|
+
publishWorkingState(reason) {
|
|
4827
|
+
try {
|
|
4828
|
+
this.emit("working_state", { reason, observation: this.getWorkingStateObservation() });
|
|
4829
|
+
}
|
|
4830
|
+
catch { /* Journal observers cannot change daemon behavior. */ }
|
|
4831
|
+
}
|
|
4628
4832
|
/** Key of the last interaction snapshot actually emitted + broadcast (#1219). */
|
|
4629
4833
|
lastPublishedInteractionKey = null;
|
|
4630
4834
|
publishInteraction() {
|
|
@@ -4877,6 +5081,7 @@ export class Daemon extends EventEmitter {
|
|
|
4877
5081
|
this.pauseWakeState = "paused";
|
|
4878
5082
|
this.autoPauseController.markPaused();
|
|
4879
5083
|
writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now(), reason);
|
|
5084
|
+
this.publishWorkingState("execution");
|
|
4880
5085
|
this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
|
|
4881
5086
|
this.ipcServer?.broadcast({
|
|
4882
5087
|
type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
|
|
@@ -4902,6 +5107,7 @@ export class Daemon extends EventEmitter {
|
|
|
4902
5107
|
}
|
|
4903
5108
|
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
4904
5109
|
async wake(timeoutMs = 30_000) {
|
|
5110
|
+
this.assertSessionOomAdmission();
|
|
4905
5111
|
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
4906
5112
|
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
4907
5113
|
this.pausePending = false;
|
|
@@ -4969,6 +5175,7 @@ export class Daemon extends EventEmitter {
|
|
|
4969
5175
|
this.clearErrorRecoveryGate();
|
|
4970
5176
|
clearPausedMarker(this.instanceDir);
|
|
4971
5177
|
this.resumeRuntimeMonitors();
|
|
5178
|
+
this.publishWorkingState("execution");
|
|
4972
5179
|
this.logger.info("Instance auto-woke");
|
|
4973
5180
|
this.ipcServer?.broadcast({
|
|
4974
5181
|
type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
|
|
@@ -5068,8 +5275,22 @@ export class Daemon extends EventEmitter {
|
|
|
5068
5275
|
if (unblockedTurn && unblockedTurn.phase === "awaiting")
|
|
5069
5276
|
this.maybeProxyReplyOnTurnEnd(pane);
|
|
5070
5277
|
}
|
|
5278
|
+
if (snapshot.state === "working" && this.recoveryBootSettled
|
|
5279
|
+
&& this.workingRecoveryInputOwner !== JSON.stringify(this.interactionOwner()))
|
|
5280
|
+
this.recoveryNativeWork = true;
|
|
5071
5281
|
this.instanceState = snapshot.state;
|
|
5072
5282
|
this.acceptedInstanceSnapshot = { ...snapshot };
|
|
5283
|
+
// Reuse an already-captured pane and the backend's pure parser, never capture for the journal.
|
|
5284
|
+
if (typeof pane === "string" && this.backend?.backgroundWorkRunning) {
|
|
5285
|
+
try {
|
|
5286
|
+
const background = this.backend.backgroundWorkRunning(pane);
|
|
5287
|
+
if (background !== this.workingBackgroundObservation) {
|
|
5288
|
+
this.workingBackgroundObservation = background;
|
|
5289
|
+
this.publishWorkingState("background");
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
catch { /* Observational parsing cannot change accepted execution state. */ }
|
|
5293
|
+
}
|
|
5073
5294
|
// OpenCode creates its session lazily on the first submitted message.
|
|
5074
5295
|
// Waiting until stop/pause to persist that id loses resume state when the
|
|
5075
5296
|
// fleet is SIGKILLed or the host reboots.
|
|
@@ -5114,6 +5335,7 @@ export class Daemon extends EventEmitter {
|
|
|
5114
5335
|
}
|
|
5115
5336
|
if (snapshot.state !== previous) {
|
|
5116
5337
|
this.logStateEdge(previous, snapshot);
|
|
5338
|
+
this.publishWorkingState("execution");
|
|
5117
5339
|
this.emit("instance_state", { name: this.name, ...snapshot });
|
|
5118
5340
|
this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
|
|
5119
5341
|
if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
|
|
@@ -5172,14 +5394,21 @@ export class Daemon extends EventEmitter {
|
|
|
5172
5394
|
// Its late success must not re-arm the turn that the user just cancelled.
|
|
5173
5395
|
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
5174
5396
|
return;
|
|
5175
|
-
// Channel turns
|
|
5176
|
-
//
|
|
5177
|
-
//
|
|
5178
|
-
//
|
|
5179
|
-
//
|
|
5180
|
-
// web-only sink), but it is a person waiting for an answer all the
|
|
5181
|
-
|
|
5397
|
+
// Channel turns started by a person. Cross-instance, schedules (silent or
|
|
5398
|
+
// not), and AgEnD system notices must not expect a reply to a person (#2544):
|
|
5399
|
+
// a proxy or recovery notice would land in the last USER topic, or flag a
|
|
5400
|
+
// heartbeat that was told not to report. A web message on a fleet with no
|
|
5401
|
+
// chat platform has no chat_id (its reply goes to the web chat through the
|
|
5402
|
+
// fleet's web-only sink), but it is a person waiting for an answer all the
|
|
5403
|
+
// same: the guard arms for it (alpha.2).
|
|
5404
|
+
if (!turnExpectsPersonReply(meta)) {
|
|
5405
|
+
this.logger.debug({
|
|
5406
|
+
source: meta.source || null,
|
|
5407
|
+
user: meta.user || null,
|
|
5408
|
+
from_instance: meta.from_instance || null,
|
|
5409
|
+
}, "Reply guard skipped — turn was not started by a person");
|
|
5182
5410
|
return;
|
|
5411
|
+
}
|
|
5183
5412
|
// The last non-empty line of what we pasted: everything on screen after it
|
|
5184
5413
|
// is the agent's own output.
|
|
5185
5414
|
const inboundMarker = deliveredText.split(/\r?\n/).map(l => l.trim()).filter(Boolean).pop();
|
|
@@ -5196,6 +5425,9 @@ export class Daemon extends EventEmitter {
|
|
|
5196
5425
|
deliveredText,
|
|
5197
5426
|
deliveredAt,
|
|
5198
5427
|
}, this.pendingWork.lastInboundTimestamp());
|
|
5428
|
+
const armed = this.turnReplyGuard.snapshot();
|
|
5429
|
+
this.workingTurnDelivery = { generation: armed.generation, deliveryId: meta.delivery_id || null };
|
|
5430
|
+
this.publishWorkingState("turn_started");
|
|
5199
5431
|
// #1209: persist the in-flight turn with a seam checkpoint. A marker
|
|
5200
5432
|
// that survives a restart is an interrupted turn by construction —
|
|
5201
5433
|
// completion (via the guard's onComplete) and cancel paths delete it.
|
|
@@ -5203,7 +5435,9 @@ export class Daemon extends EventEmitter {
|
|
|
5203
5435
|
// the interruption. Reads are windowed/bounded (see turn-resume.ts).
|
|
5204
5436
|
// A resumed continuation never re-arms (P3a): episode-once means a crash
|
|
5205
5437
|
// during the continuation boots clean instead of chaining resumes.
|
|
5206
|
-
|
|
5438
|
+
// #2544: a reply that settled before this arm already answered the person —
|
|
5439
|
+
// writing a marker would re-inject a continuation for a finished turn.
|
|
5440
|
+
if (!meta.resumedContinuationOf && !armed.completionDelivered) {
|
|
5207
5441
|
const resumeBackend = this.config.backend ?? this.backend?.binaryName ?? this.runtimeIdentity?.backend;
|
|
5208
5442
|
const resumeCwd = this.config.working_directory ?? "";
|
|
5209
5443
|
writeInFlightTurnMarker(this.instanceDir, {
|
|
@@ -5251,6 +5485,7 @@ export class Daemon extends EventEmitter {
|
|
|
5251
5485
|
const marker = consumeInFlightTurnMarker(this.instanceDir);
|
|
5252
5486
|
if (!marker)
|
|
5253
5487
|
return "none";
|
|
5488
|
+
this.recoveryLegacyCovered = true;
|
|
5254
5489
|
// P1 fence: capture before any await. A cancel (deliveryEpoch),
|
|
5255
5490
|
// respawn (spawnGeneration) or stop/pause (launchFenceEpoch) during the
|
|
5256
5491
|
// grace hold must stale this gate — the push below rechecks, and the
|
|
@@ -5617,7 +5852,7 @@ export class Daemon extends EventEmitter {
|
|
|
5617
5852
|
reason,
|
|
5618
5853
|
recoveryStarted: true,
|
|
5619
5854
|
});
|
|
5620
|
-
|
|
5855
|
+
this.deliverReplyDropNotice("inst.reply_drop_retrying", "replydrop", "Reply-drop status", turn, reason, false);
|
|
5621
5856
|
this.queueReplyRecoveryPrompt(turn);
|
|
5622
5857
|
}
|
|
5623
5858
|
reportReplyDropWithoutRetry(turn, reason) {
|
|
@@ -5628,7 +5863,7 @@ export class Daemon extends EventEmitter {
|
|
|
5628
5863
|
reason,
|
|
5629
5864
|
recoveryStarted: false,
|
|
5630
5865
|
});
|
|
5631
|
-
|
|
5866
|
+
this.deliverReplyDropNotice("inst.reply_drop_unknown", "replydrop", "Unconfirmed reply status", turn, reason, true);
|
|
5632
5867
|
}
|
|
5633
5868
|
queueReplyRecoveryPrompt(turn) {
|
|
5634
5869
|
const deliveryEpoch = this.deliveryEpoch;
|
|
@@ -5721,12 +5956,20 @@ export class Daemon extends EventEmitter {
|
|
|
5721
5956
|
generation: turn.generation,
|
|
5722
5957
|
reason,
|
|
5723
5958
|
});
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
|
|
5727
|
-
|
|
5728
|
-
|
|
5729
|
-
|
|
5959
|
+
this.deliverReplyDropNotice("inst.reply_drop_unrecovered", "replydropwarn", "Reply-drop recovery warning", turn, reason, true);
|
|
5960
|
+
}
|
|
5961
|
+
/** Channel notice for a reply-drop. `rateLimit` shares the per-instance cooldown (#2544). */
|
|
5962
|
+
deliverReplyDropNotice(key, kind, title, turn, reason, rateLimit) {
|
|
5963
|
+
if (rateLimit) {
|
|
5964
|
+
const now = Date.now();
|
|
5965
|
+
if (this.lastReplyDropWarningAt !== 0
|
|
5966
|
+
&& now - this.lastReplyDropWarningAt < REPLY_DROP_WARNING_COOLDOWN_MS) {
|
|
5967
|
+
this.logger.debug({ reason, kind }, "Reply-drop channel notice rate-limited");
|
|
5968
|
+
return;
|
|
5969
|
+
}
|
|
5970
|
+
this.lastReplyDropWarningAt = now;
|
|
5971
|
+
}
|
|
5972
|
+
void this.deliverDaemonReply(t(key, formatReplyDropDetection(turn, reason)), kind, title, turn.target, true);
|
|
5730
5973
|
}
|
|
5731
5974
|
/** Relay the pane's final text to the channel, marked as a daemon proxy reply. */
|
|
5732
5975
|
async sendProxyReply(pane, target) {
|
|
@@ -6324,6 +6567,7 @@ export class Daemon extends EventEmitter {
|
|
|
6324
6567
|
}
|
|
6325
6568
|
/** Stop every runtime poller/watcher while preserving IPC and daemon state. */
|
|
6326
6569
|
freezeRuntimeMonitors() {
|
|
6570
|
+
this.cancelCapacityNudge("the instance was stopped, paused or respawned");
|
|
6327
6571
|
this.runtimeMonitorsFrozen = true;
|
|
6328
6572
|
this.launchFenceEpoch++;
|
|
6329
6573
|
this.interactionObservation.reset(this.interactionOwner());
|
|
@@ -6436,6 +6680,10 @@ export class Daemon extends EventEmitter {
|
|
|
6436
6680
|
this.config.warm_cap = update.warm_cap;
|
|
6437
6681
|
else if (update.warm_cap === null)
|
|
6438
6682
|
delete this.config.warm_cap;
|
|
6683
|
+
if (typeof update.resume_interrupted_work === "boolean")
|
|
6684
|
+
this.config.resume_interrupted_work = update.resume_interrupted_work;
|
|
6685
|
+
else if (update.resume_interrupted_work === null)
|
|
6686
|
+
delete this.config.resume_interrupted_work;
|
|
6439
6687
|
const nextAutoPause = typeof update.auto_pause_after === "number"
|
|
6440
6688
|
&& Number.isFinite(update.auto_pause_after)
|
|
6441
6689
|
&& update.auto_pause_after >= 0
|
|
@@ -7708,7 +7956,8 @@ export class Daemon extends EventEmitter {
|
|
|
7708
7956
|
/**
|
|
7709
7957
|
* Arm the "keep going" nudge for the error occurrence just reported (`pattern` must be the one the lifecycle was
|
|
7710
7958
|
* handed). The nudge is sent once, `delayMs` from now on the MONOTONIC clock (a wall-clock step changes nothing),
|
|
7711
|
-
* and only if
|
|
7959
|
+
* and only if the failed turn still owns an empty idle composer — see {@link tickCapacityNudge}.
|
|
7960
|
+
* Cosmetic footer redraws are not new work. Returns false when there is nothing to arm.
|
|
7712
7961
|
*/
|
|
7713
7962
|
armCapacityNudge(pattern, delayMs) {
|
|
7714
7963
|
const episode = this.lastErrorEpisode;
|
|
@@ -7727,9 +7976,12 @@ export class Daemon extends EventEmitter {
|
|
|
7727
7976
|
// lookahead on the counting pattern: whole-pane dedup would max at 1 and miss later hits. See #1215.)
|
|
7728
7977
|
if (!codexLiveRowMatches(episode.pane, pattern))
|
|
7729
7978
|
return false;
|
|
7979
|
+
this.cancelCapacityNudge("the instance has work of its own");
|
|
7730
7980
|
this.capacityNudge = {
|
|
7731
7981
|
pane: episode.pane, pattern, spawn: episode.spawn, fence: episode.fence,
|
|
7732
7982
|
epoch: this.deliveryEpoch, dueAt: performance.now() + delayMs,
|
|
7983
|
+
episode,
|
|
7984
|
+
promiseTarget: null,
|
|
7733
7985
|
};
|
|
7734
7986
|
this.logger.info({ delayMs }, "Model-capacity nudge armed");
|
|
7735
7987
|
return true;
|
|
@@ -7744,7 +7996,7 @@ export class Daemon extends EventEmitter {
|
|
|
7744
7996
|
const nudge = this.capacityNudge, episode = this.lastErrorEpisode;
|
|
7745
7997
|
const tmux = this.tmux, attempt = this.launchAttempt, boot = this.bootId;
|
|
7746
7998
|
this.cancelCapacityNudge("retries exhausted");
|
|
7747
|
-
if (!nudge || !episode || !tmux)
|
|
7999
|
+
if (!nudge || !episode || nudge.episode !== episode || !tmux)
|
|
7748
8000
|
return null;
|
|
7749
8001
|
const window = tmux.getWindowId?.();
|
|
7750
8002
|
const current = () => this.tmux === tmux && tmux.getWindowId?.() === window
|
|
@@ -7752,9 +8004,7 @@ export class Daemon extends EventEmitter {
|
|
|
7752
8004
|
&& episode === this.lastErrorEpisode && this.isDeliveryEpochCurrent(nudge.epoch)
|
|
7753
8005
|
&& !this.startupAborted && !this.spawning && !this.isPaused
|
|
7754
8006
|
&& this.getProcessStatus() === "running" && this.instanceState === "idle" && this.pasteQueueDepth === 0;
|
|
7755
|
-
const accepts = (pane) =>
|
|
7756
|
-
&& this.backend?.isPeriodicRedrawIdlePane?.(pane) === true
|
|
7757
|
-
&& codexLiveRowMatches(pane, nudge.pattern) && !this.backend?.backgroundWorkRunning?.(pane);
|
|
8007
|
+
const accepts = (pane) => this.capacityNudgePaneCurrent(nudge, pane);
|
|
7758
8008
|
const revalidate = () => this.paneWriteLock.run(async () => {
|
|
7759
8009
|
if (!current())
|
|
7760
8010
|
return false;
|
|
@@ -7776,10 +8026,35 @@ export class Daemon extends EventEmitter {
|
|
|
7776
8026
|
}
|
|
7777
8027
|
/** Drop an armed nudge (the user stopped / paused the instance, or the lifecycle gave up on it). */
|
|
7778
8028
|
cancelCapacityNudge(reason) {
|
|
7779
|
-
|
|
8029
|
+
const nudge = this.capacityNudge;
|
|
8030
|
+
if (!nudge)
|
|
7780
8031
|
return;
|
|
7781
8032
|
this.capacityNudge = null;
|
|
7782
8033
|
this.logger.info({ reason }, "Model-capacity nudge cancelled");
|
|
8034
|
+
this.reportCapacityNudgeCancelled(nudge, reason);
|
|
8035
|
+
}
|
|
8036
|
+
/** Only a dispatched channel promise needs a later cancellation notice. */
|
|
8037
|
+
markCapacityNudgePromised(target) {
|
|
8038
|
+
if (this.capacityNudge)
|
|
8039
|
+
this.capacityNudge.promiseTarget = target;
|
|
8040
|
+
}
|
|
8041
|
+
reportCapacityNudgeCancelled(nudge, reason) {
|
|
8042
|
+
const target = nudge.promiseTarget;
|
|
8043
|
+
if (!target)
|
|
8044
|
+
return;
|
|
8045
|
+
nudge.promiseTarget = null;
|
|
8046
|
+
this.emit("capacity_nudge_cancelled", { name: this.name, reason, target });
|
|
8047
|
+
}
|
|
8048
|
+
/**
|
|
8049
|
+
* A failed turn must still be the last live transcript item, with no new
|
|
8050
|
+
* error occurrence, draft, queue, busy status or background work. Codex's
|
|
8051
|
+
* own periodic-redraw idle classifier recognises its empty composer/footer;
|
|
8052
|
+
* never infer inactivity from byte equality or a cached idle state alone.
|
|
8053
|
+
*/
|
|
8054
|
+
capacityNudgePaneCurrent(nudge, pane) {
|
|
8055
|
+
return this.paneAuthoritativelyIdle(pane) && this.backend?.isPeriodicRedrawIdlePane?.(pane) === true
|
|
8056
|
+
&& !this.backend?.backgroundWorkRunning?.(pane) && codexLiveRowMatches(pane, nudge.pattern)
|
|
8057
|
+
&& Daemon.countOccurrences(nudge.pattern, pane) === Daemon.countOccurrences(nudge.pattern, nudge.pane);
|
|
7783
8058
|
}
|
|
7784
8059
|
/** The first live composer of a spawn fixes what is history (see {@link nudgeBaselines}). */
|
|
7785
8060
|
takeNudgeBaselines(pane, patterns) {
|
|
@@ -7802,8 +8077,8 @@ export class Daemon extends EventEmitter {
|
|
|
7802
8077
|
}
|
|
7803
8078
|
/**
|
|
7804
8079
|
* One monitor poll's look at an armed nudge. It is CANCELLED — never postponed — when anything about the instance
|
|
7805
|
-
* moved: a stop / pause / respawn (spawn generation or monitor epoch), a user cancel (delivery epoch), the pane
|
|
7806
|
-
* longer
|
|
8080
|
+
* moved: a stop / pause / respawn (spawn generation or monitor epoch), a user cancel (delivery epoch), the pane
|
|
8081
|
+
* no longer belonging to the failed turn (new output, a draft, another dialog), the
|
|
7807
8082
|
* agent working, or a message waiting for delivery. Only when it is due and still untouched is it fired — once — and
|
|
7808
8083
|
* the injection re-verifies everything under the pane-write lock.
|
|
7809
8084
|
*/
|
|
@@ -7813,10 +8088,11 @@ export class Daemon extends EventEmitter {
|
|
|
7813
8088
|
return;
|
|
7814
8089
|
const gone = nudge.spawn !== this.spawnGeneration || nudge.fence !== this.launchFenceEpoch ? "the instance was stopped, paused or respawned"
|
|
7815
8090
|
: !this.isDeliveryEpochCurrent(nudge.epoch) ? "the user cancelled"
|
|
7816
|
-
:
|
|
7817
|
-
:
|
|
8091
|
+
: nudge.episode !== this.lastErrorEpisode ? "the instance has work of its own"
|
|
8092
|
+
: this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped" ? "the instance is not running"
|
|
7818
8093
|
: this.instanceState === "working" || this.pasteQueueDepth > 0 ? "the instance has work of its own"
|
|
7819
|
-
:
|
|
8094
|
+
: !this.capacityNudgePaneCurrent(nudge, pane) ? "the failed turn is no longer idle with an empty input"
|
|
8095
|
+
: null;
|
|
7820
8096
|
if (gone) {
|
|
7821
8097
|
this.cancelCapacityNudge(gone);
|
|
7822
8098
|
return;
|
|
@@ -7824,29 +8100,38 @@ export class Daemon extends EventEmitter {
|
|
|
7824
8100
|
if (performance.now() < nudge.dueAt)
|
|
7825
8101
|
return;
|
|
7826
8102
|
this.capacityNudge = null; // fired: once per episode
|
|
7827
|
-
void this.sendCapacityNudge(nudge).catch(err =>
|
|
8103
|
+
void this.sendCapacityNudge(nudge).catch(err => {
|
|
8104
|
+
this.logger.warn({ err: err.message }, "Model-capacity nudge failed");
|
|
8105
|
+
this.reportCapacityNudgeCancelled(nudge, "the retry could not be submitted");
|
|
8106
|
+
});
|
|
7828
8107
|
}
|
|
7829
8108
|
async sendCapacityNudge(nudge) {
|
|
7830
8109
|
// Everything the nudge was armed under, asked again after EVERY await that precedes a write — inside the paste
|
|
7831
8110
|
// primitive too (its capture, its paste, its pause, its Enter, its retry): once any of it is false nothing more is
|
|
7832
8111
|
// written. A message queued behind the lock, or the agent starting a turn, is "work of its own" like at the poll.
|
|
7833
8112
|
const current = () => nudge.spawn === this.spawnGeneration && nudge.fence === this.launchFenceEpoch
|
|
8113
|
+
&& nudge.episode === this.lastErrorEpisode
|
|
7834
8114
|
&& this.isDeliveryEpochCurrent(nudge.epoch) && !this.isPaused && !this.healthCheckPaused
|
|
7835
8115
|
&& !this.startupAborted && this.getProcessStatus() !== "stopped"
|
|
7836
8116
|
&& this.pasteQueueDepth === 0 && this.instanceState !== "working";
|
|
7837
8117
|
// Exclusive with every delivery: nothing can paste a turn between the verdict below and the nudge.
|
|
7838
|
-
|
|
7839
|
-
|
|
7840
|
-
|
|
7841
|
-
|
|
7842
|
-
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
|
|
7846
|
-
|
|
8118
|
+
let sent = false;
|
|
8119
|
+
try {
|
|
8120
|
+
await this.paneWriteLock.run(async () => {
|
|
8121
|
+
if (!this.tmux)
|
|
8122
|
+
return;
|
|
8123
|
+
// ONE capture decides failed-turn ownership and empty/idle input.
|
|
8124
|
+
sent = await this.submitSystemPaste(t("inst.codex_capacity_nudge_text"), "capacity-continue", {
|
|
8125
|
+
current,
|
|
8126
|
+
accept: pane => this.capacityNudgePaneCurrent(nudge, pane),
|
|
8127
|
+
});
|
|
8128
|
+
this.logger.info({ sent }, "Model-capacity nudge attempted");
|
|
7847
8129
|
});
|
|
7848
|
-
|
|
7849
|
-
|
|
8130
|
+
}
|
|
8131
|
+
finally {
|
|
8132
|
+
if (!sent)
|
|
8133
|
+
this.reportCapacityNudgeCancelled(nudge, "the retry could not be submitted");
|
|
8134
|
+
}
|
|
7850
8135
|
}
|
|
7851
8136
|
/** What identifies the request on screen: the backend's own identity when it has one, else the whole screen verbatim. */
|
|
7852
8137
|
static screenOf(dialog, pane) {
|
|
@@ -10095,8 +10380,10 @@ export class Daemon extends EventEmitter {
|
|
|
10095
10380
|
// purposes even before the idle edge completes the guard. Without
|
|
10096
10381
|
// this, a restart in the reply-to-idle window would re-inject a
|
|
10097
10382
|
// continuation for an already-answered turn.
|
|
10098
|
-
if (replyAttempt?.completionAction)
|
|
10383
|
+
if (replyAttempt?.completionAction) {
|
|
10099
10384
|
clearInFlightTurnMarker(this.instanceDir);
|
|
10385
|
+
this.publishWorkingState("completion_delivered");
|
|
10386
|
+
}
|
|
10100
10387
|
// #1377: a confirmed completion action (reply/react/edit_message) ends
|
|
10101
10388
|
// the human-facing obligation immediately — the idle edge does not need
|
|
10102
10389
|
// to confirm it. Clear any pending confirmation window so a later
|
|
@@ -10111,7 +10398,9 @@ export class Daemon extends EventEmitter {
|
|
|
10111
10398
|
// false and must not clear the window either.
|
|
10112
10399
|
if (replyAttempt?.completionAction) {
|
|
10113
10400
|
const snap = this.turnReplyGuard.snapshot();
|
|
10114
|
-
|
|
10401
|
+
// generation 0 is a pre-arm token applied to the live generation (#2544).
|
|
10402
|
+
if (snap && (snap.generation === replyAttempt.generation || replyAttempt.generation === 0)
|
|
10403
|
+
&& snap.phase === "awaiting"
|
|
10115
10404
|
&& snap.completionDelivered && !snap.cancelledByUser) {
|
|
10116
10405
|
this.clearReplyGuardConfirm();
|
|
10117
10406
|
}
|
|
@@ -10759,8 +11048,16 @@ export class Daemon extends EventEmitter {
|
|
|
10759
11048
|
* into exactly the window this exists to close.
|
|
10760
11049
|
*/
|
|
10761
11050
|
beginSpawn() {
|
|
11051
|
+
this.unclearOomStopped = false;
|
|
11052
|
+
this.publishWorkingState("before_spawn");
|
|
11053
|
+
this.recoveryContinuationId = null;
|
|
11054
|
+
this.workingBackgroundObservation = null;
|
|
11055
|
+
this.ownedWorkingSession = null;
|
|
11056
|
+
this.recoveryBootSettled = false;
|
|
11057
|
+
this.recoveryNativeWork = false;
|
|
11058
|
+
this.recoveryCancelled = false;
|
|
10762
11059
|
this.dialogAnswers = null;
|
|
10763
|
-
this.
|
|
11060
|
+
this.cancelCapacityNudge("the instance was stopped, paused or respawned");
|
|
10764
11061
|
this.lastErrorEpisode = null;
|
|
10765
11062
|
this.nudgeBaselines = null;
|
|
10766
11063
|
// A relaunch settles any outstanding quit watch: the CLI came back.
|
|
@@ -10791,6 +11088,8 @@ export class Daemon extends EventEmitter {
|
|
|
10791
11088
|
this.spawnDepth = Math.max(0, this.spawnDepth - 1);
|
|
10792
11089
|
if (this.spawnDepth > 0)
|
|
10793
11090
|
return; // an enclosing spawn is still running
|
|
11091
|
+
if (!this.recoveryBootWorkPending)
|
|
11092
|
+
this.recoveryBootSettled = true;
|
|
10794
11093
|
this.spawning = false;
|
|
10795
11094
|
const resolve = this.resolveSpawnSettled;
|
|
10796
11095
|
this.spawnSettled = null;
|
|
@@ -10974,6 +11273,7 @@ export class Daemon extends EventEmitter {
|
|
|
10974
11273
|
catch { /* best effort */ }
|
|
10975
11274
|
}
|
|
10976
11275
|
async spawnClaudeWindow() {
|
|
11276
|
+
this.assertSessionOomAdmission();
|
|
10977
11277
|
this.beginSpawn();
|
|
10978
11278
|
let resumedSuccessfully = false;
|
|
10979
11279
|
try {
|
|
@@ -11364,11 +11664,20 @@ export class Daemon extends EventEmitter {
|
|
|
11364
11664
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
11365
11665
|
*/
|
|
11366
11666
|
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
11367
|
-
const
|
|
11667
|
+
const recoveryOwner = this.collateralOomRecovery ? this.interactionOwner() : null;
|
|
11668
|
+
const enter = () => {
|
|
11669
|
+
if (recoveryOwner && (!sameInteractionOwner(recoveryOwner, this.interactionOwner())
|
|
11670
|
+
|| this.startupAborted || this.deliveryWritesStopping || this.runtimeMonitorsFrozen || this.healthCheckPaused)) {
|
|
11671
|
+
throw new Error("OOM recovery cancelled: the instance was stopped, paused or replaced while memory was unavailable");
|
|
11672
|
+
}
|
|
11673
|
+
return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
|
|
11674
|
+
};
|
|
11675
|
+
const ready = !this.spawnGate ? await enter() : await this.spawnGate.run({
|
|
11368
11676
|
instanceName: this.name,
|
|
11369
11677
|
workingDirectory: this.config.working_directory,
|
|
11370
11678
|
reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
|
|
11371
|
-
|
|
11679
|
+
requireNormalMemory: this.collateralOomRecovery,
|
|
11680
|
+
}, enter);
|
|
11372
11681
|
// #906: a resumed kiro conversation comes back as the agent it was saved under. Every launch path that reaches a
|
|
11373
11682
|
// ready CLI (start, recovery, wake) switches it to this instance's own agent here, while the spawn still holds
|
|
11374
11683
|
// deliveries — and outside the spawn gate, which it does not need.
|
|
@@ -11650,6 +11959,7 @@ export class Daemon extends EventEmitter {
|
|
|
11650
11959
|
}
|
|
11651
11960
|
}
|
|
11652
11961
|
async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
|
|
11962
|
+
this.assertSessionOomAdmission();
|
|
11653
11963
|
this.startupAdmission?.();
|
|
11654
11964
|
const launchFence = this.launchFenceEpoch;
|
|
11655
11965
|
const backendConfig = this.buildBackendConfig();
|
|
@@ -11698,6 +12008,7 @@ export class Daemon extends EventEmitter {
|
|
|
11698
12008
|
// Thrown, not `false`: a false verdict reads as "the CLI failed to start" and sends the startup path
|
|
11699
12009
|
// into its resume-retry / set-the-session-aside handling, for an instance somebody just stopped.
|
|
11700
12010
|
this.startupAdmission?.();
|
|
12011
|
+
this.assertSessionOomAdmission();
|
|
11701
12012
|
if (launchFence !== this.launchFenceEpoch)
|
|
11702
12013
|
throw new Error("Launch cancelled: the instance was stopped or paused while the launch was being prepared");
|
|
11703
12014
|
this.backend.writeConfig(backendConfig);
|
|
@@ -11784,6 +12095,7 @@ export class Daemon extends EventEmitter {
|
|
|
11784
12095
|
}
|
|
11785
12096
|
this.startupAdmission?.();
|
|
11786
12097
|
let windowId;
|
|
12098
|
+
this.assertSessionOomAdmission();
|
|
11787
12099
|
if (reuseWindow) {
|
|
11788
12100
|
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
11789
12101
|
await this.tmux.respawnWindow(cmd, resolvedCwd);
|
|
@@ -12231,6 +12543,13 @@ export class Daemon extends EventEmitter {
|
|
|
12231
12543
|
const sid = this.backend?.getSessionId();
|
|
12232
12544
|
if (!sid)
|
|
12233
12545
|
return;
|
|
12546
|
+
if ((this.config.backend ?? "claude-code") === "claude-code" && Buffer.byteLength(sid) <= 256
|
|
12547
|
+
&& !/[\u0000-\u001f\u007f]/.test(sid) && this.ownedWorkingSession !== sid) {
|
|
12548
|
+
this.ownedWorkingSession = sid;
|
|
12549
|
+
// Reuse this existing owned checkpoint. An idle instance may have no
|
|
12550
|
+
// later state edge before an OOM; its journal must not stay unknown.
|
|
12551
|
+
this.publishWorkingState("process");
|
|
12552
|
+
}
|
|
12234
12553
|
const path = join(this.instanceDir, "session-id");
|
|
12235
12554
|
// Idle observations happen after turns. Avoid needless writes (and mtime
|
|
12236
12555
|
// churn) while still updating the marker when /new changes the session.
|
|
@@ -12418,6 +12737,7 @@ export class Daemon extends EventEmitter {
|
|
|
12418
12737
|
crashType: data.crashType,
|
|
12419
12738
|
exitCode: data.exitCode,
|
|
12420
12739
|
reason: data.reason,
|
|
12740
|
+
...(data.oomEvidence ? { oomEvidence: data.oomEvidence } : {}),
|
|
12421
12741
|
lastOutput: data.lastOutput,
|
|
12422
12742
|
crashCount: this.crashCount + 1,
|
|
12423
12743
|
crashesInWindow: this.crashTimestamps.length,
|