@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +47 -5
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +68 -3
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/types.d.ts +9 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/mcp-operation-response.d.ts +3 -0
- package/dist/channel/mcp-operation-response.js +21 -0
- package/dist/channel/mcp-operation-response.js.map +1 -0
- package/dist/channel/mcp-operation-response.test.d.ts +1 -0
- package/dist/channel/mcp-operation-response.test.js +23 -0
- package/dist/channel/mcp-operation-response.test.js.map +1 -0
- package/dist/channel/mcp-server.js +20 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/daemon.d.ts +23 -0
- package/dist/daemon.js +248 -24
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-outbox.d.ts +90 -0
- package/dist/delivery-outbox.js +562 -0
- package/dist/delivery-outbox.js.map +1 -0
- package/dist/delivery-outbox.test.d.ts +1 -0
- package/dist/delivery-outbox.test.js +343 -0
- package/dist/delivery-outbox.test.js.map +1 -0
- package/dist/fleet-manager.d.ts +38 -1
- package/dist/fleet-manager.js +302 -36
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +5 -0
- package/dist/instance-lifecycle.js +38 -0
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +2 -0
- package/dist/instructions.js.map +1 -1
- package/dist/outbound-handlers.d.ts +22 -1
- package/dist/outbound-handlers.js +81 -7
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/remote-credential-check.d.ts +39 -0
- package/dist/remote-credential-check.js +81 -0
- package/dist/remote-credential-check.js.map +1 -0
- package/dist/usage/providers.d.ts +11 -6
- package/dist/usage/providers.js +17 -10
- package/dist/usage/providers.js.map +1 -1
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
2
|
import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
4
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
6
|
import { rotateLogIfNeeded } from "./logger.js";
|
|
7
7
|
import { mcpServerState } from "./mcp-liveness.js";
|
|
@@ -33,7 +33,7 @@ import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
|
|
|
33
33
|
const __filename = fileURLToPath(import.meta.url);
|
|
34
34
|
const __dirname = dirname(__filename);
|
|
35
35
|
// Tool routing sets — module-level to avoid re-creation on every handleToolCall
|
|
36
|
-
const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "describe_instance"]);
|
|
36
|
+
const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "broadcast", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "describe_instance"]);
|
|
37
37
|
const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
|
|
38
38
|
const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
|
|
39
39
|
const TASK_TOOL = "task";
|
|
@@ -95,6 +95,8 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
|
95
95
|
export const PERIODIC_REDRAW_PROBE_MS = 25;
|
|
96
96
|
/** Do not turn a streaming Codex turn into one capture-pane per burst. */
|
|
97
97
|
export const STRUCTURED_IDLE_PROBE_MS = 500;
|
|
98
|
+
/** How long an unrecognised-but-empty composer must stay unchanged before it counts as idle (#978). */
|
|
99
|
+
export const UNKNOWN_LAYOUT_STABLE_MS = 10_000;
|
|
98
100
|
/** A foreground server/port-forward should hand control back or be acknowledged. */
|
|
99
101
|
export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
|
|
100
102
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
@@ -326,9 +328,12 @@ export class PaneStateMachine {
|
|
|
326
328
|
* the caller has them.
|
|
327
329
|
* @param opts.forceBusy the backend has positively identified a still-running
|
|
328
330
|
* foreground tool even though its TUI keeps an old ready footer visible.
|
|
331
|
+
* @param opts.forceReady the backend has positively proved an idle composer
|
|
332
|
+
* the ready pattern cannot read (an unknown footer layout); the busy
|
|
333
|
+
* pattern still vetoes it.
|
|
329
334
|
*/
|
|
330
335
|
observe(pane, now = Date.now(), opts = {}) {
|
|
331
|
-
const { settled = false, changeAt = now, forceBusy = false } = opts;
|
|
336
|
+
const { settled = false, changeAt = now, forceBusy = false, forceReady = false } = opts;
|
|
332
337
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
333
338
|
const firstObservation = this.lastPaneHash === null;
|
|
334
339
|
const paneChanged = this.lastPaneHash !== paneHash;
|
|
@@ -340,7 +345,9 @@ export class PaneStateMachine {
|
|
|
340
345
|
// Some TUIs keep their ready footer visible while a foreground tool owns
|
|
341
346
|
// stdin. Backends can positively identify that tool from the pane; in that
|
|
342
347
|
// case the old prompt must not clear pending work or retire Cancel.
|
|
343
|
-
|
|
348
|
+
// forceReady: the backend proved readiness itself (#978 stable unknown
|
|
349
|
+
// footer); a visible busy marker still vetoes it.
|
|
350
|
+
const ready = !forceBusy && (forceReady ? !this.busyPattern?.test(pane) : this.isReady(pane));
|
|
344
351
|
const nextState = firstObservation
|
|
345
352
|
? ready ? "idle" : "working"
|
|
346
353
|
: paneChanged && !settled
|
|
@@ -908,6 +915,9 @@ export class Daemon extends EventEmitter {
|
|
|
908
915
|
spawnGate;
|
|
909
916
|
stormWindow;
|
|
910
917
|
backendOutage;
|
|
918
|
+
/** Identity of this live Daemon object; changes on object stop/restart. */
|
|
919
|
+
bootId = randomUUID();
|
|
920
|
+
deliveryOutbox;
|
|
911
921
|
logger;
|
|
912
922
|
tmuxSessionName;
|
|
913
923
|
tmux = null;
|
|
@@ -996,6 +1006,10 @@ export class Daemon extends EventEmitter {
|
|
|
996
1006
|
/** Consecutive structural idle captures for a noisy, periodically-redrawing
|
|
997
1007
|
* pane. Two are required before a working -> idle transition. */
|
|
998
1008
|
instanceStatePeriodicIdleConfirmations = 0;
|
|
1009
|
+
/** #978: last unrecognised-but-empty-composer capture and since when it has been unchanged. */
|
|
1010
|
+
unknownLayoutPaneKey = null;
|
|
1011
|
+
unknownLayoutStableSince = 0;
|
|
1012
|
+
unknownLayoutWarned = false;
|
|
999
1013
|
instanceStateStuckTimer = null;
|
|
1000
1014
|
instanceStateOutputListener = null;
|
|
1001
1015
|
instanceStateOutputEventName = null;
|
|
@@ -1313,6 +1327,65 @@ export class Daemon extends EventEmitter {
|
|
|
1313
1327
|
const autoPauseMinutes = isGeneral ? 0 : (typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0); // default: disabled
|
|
1314
1328
|
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
|
|
1315
1329
|
}
|
|
1330
|
+
/** FleetManager and this daemon share a process, so durable begin/ACK is a direct synchronous store call. */
|
|
1331
|
+
setDeliveryOutboxPort(port) {
|
|
1332
|
+
this.deliveryOutbox = port;
|
|
1333
|
+
}
|
|
1334
|
+
durableDeliveryAttempt(meta) {
|
|
1335
|
+
const deliveryId = meta.delivery_id;
|
|
1336
|
+
if (!deliveryId)
|
|
1337
|
+
return null;
|
|
1338
|
+
const attemptNo = Number(meta.delivery_attempt);
|
|
1339
|
+
if (!Number.isSafeInteger(attemptNo) || attemptNo < 1)
|
|
1340
|
+
return false;
|
|
1341
|
+
return { deliveryId, attemptNo };
|
|
1342
|
+
}
|
|
1343
|
+
beginDurableDelivery(delivery) {
|
|
1344
|
+
if (!this.deliveryOutbox) {
|
|
1345
|
+
this.emit("durable_delivery_deferred", { ...delivery, targetBootId: this.bootId, reason: "outbox port unavailable before pane write" });
|
|
1346
|
+
return false;
|
|
1347
|
+
}
|
|
1348
|
+
const outcome = this.deliveryOutbox.begin(delivery.deliveryId, this.bootId, delivery.attemptNo);
|
|
1349
|
+
// A repeated begin is idempotent at the store but must not authorize a
|
|
1350
|
+
// second pane write. Only the first caller that acquired the permit acts.
|
|
1351
|
+
return outcome === "begun";
|
|
1352
|
+
}
|
|
1353
|
+
abortDurableDelivery(delivery, reason) {
|
|
1354
|
+
if (!delivery)
|
|
1355
|
+
return;
|
|
1356
|
+
this.deliveryOutbox?.abort(delivery.deliveryId, this.bootId, delivery.attemptNo, reason);
|
|
1357
|
+
}
|
|
1358
|
+
retryDurableDeliveryBeforeBegin(delivery, reason) {
|
|
1359
|
+
if (!delivery)
|
|
1360
|
+
return;
|
|
1361
|
+
if (this.deliveryOutbox) {
|
|
1362
|
+
this.deliveryOutbox.retryBeforeBegin(delivery.deliveryId, this.bootId, delivery.attemptNo, reason, 5_000);
|
|
1363
|
+
}
|
|
1364
|
+
else {
|
|
1365
|
+
this.emit("durable_delivery_deferred", { ...delivery, targetBootId: this.bootId, reason });
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
finishDurableDelivery(delivery, outcome, evidence) {
|
|
1369
|
+
if (!delivery)
|
|
1370
|
+
return;
|
|
1371
|
+
this.deliveryOutbox?.complete(delivery.deliveryId, this.bootId, delivery.attemptNo, outcome, evidence);
|
|
1372
|
+
}
|
|
1373
|
+
finishDurableSubmission(delivery, verdict) {
|
|
1374
|
+
// Legacy backends may accept a pane write without enough pane evidence to
|
|
1375
|
+
// claim submission. Keep that row inspectable instead of recording a false
|
|
1376
|
+
// delivered state; reconciliation can classify it in a later phase.
|
|
1377
|
+
if (verdict.proof === "unverified") {
|
|
1378
|
+
this.finishDurableDelivery(delivery, "uncertain", `${verdict.phase ?? "submission"}:unverified`);
|
|
1379
|
+
return;
|
|
1380
|
+
}
|
|
1381
|
+
this.finishDurableDelivery(delivery, "delivered", "positive submission proof");
|
|
1382
|
+
}
|
|
1383
|
+
deferDurableDelivery(meta, reason) {
|
|
1384
|
+
const delivery = this.durableDeliveryAttempt(meta);
|
|
1385
|
+
if (delivery === null || delivery === false)
|
|
1386
|
+
return;
|
|
1387
|
+
this.retryDurableDeliveryBeforeBegin(delivery, reason);
|
|
1388
|
+
}
|
|
1316
1389
|
async start() {
|
|
1317
1390
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
1318
1391
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
@@ -1411,6 +1484,7 @@ export class Daemon extends EventEmitter {
|
|
|
1411
1484
|
void this.wake().then(() => {
|
|
1412
1485
|
this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
|
|
1413
1486
|
}).catch(err => {
|
|
1487
|
+
this.deferDurableDelivery(meta, `target wake failed before pane submission: ${err.message}`);
|
|
1414
1488
|
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
1415
1489
|
});
|
|
1416
1490
|
}
|
|
@@ -3699,9 +3773,39 @@ export class Daemon extends EventEmitter {
|
|
|
3699
3773
|
}
|
|
3700
3774
|
else {
|
|
3701
3775
|
// A live busy marker or an unfamiliar layout outranks the broad Codex
|
|
3702
|
-
// ready pattern during the noisy-redraw path. Unknown is working
|
|
3776
|
+
// ready pattern during the noisy-redraw path. Unknown is working —
|
|
3777
|
+
// but not forever (#978): a footer the proof does not recognise
|
|
3778
|
+
// would otherwise pin an idle CLI in `working` and queue every
|
|
3779
|
+
// delivery indefinitely. If the backend still sees its own live,
|
|
3780
|
+
// empty composer and the pane has not changed (cosmetic glyphs
|
|
3781
|
+
// aside) for UNKNOWN_LAYOUT_STABLE_MS, read it as settled and ready.
|
|
3703
3782
|
this.instanceStatePeriodicIdleConfirmations = 0;
|
|
3704
|
-
|
|
3783
|
+
const stableFallback = this.backend?.isStableUnknownLayoutIdlePane;
|
|
3784
|
+
const now = Date.now();
|
|
3785
|
+
if (stableFallback?.call(this.backend, pane)) {
|
|
3786
|
+
const key = pane.replace(/⋆/gu, "").replace(/[ \t]+$/gmu, "");
|
|
3787
|
+
if (key !== this.unknownLayoutPaneKey) {
|
|
3788
|
+
this.unknownLayoutPaneKey = key;
|
|
3789
|
+
this.unknownLayoutStableSince = now;
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
else {
|
|
3793
|
+
this.unknownLayoutPaneKey = null;
|
|
3794
|
+
}
|
|
3795
|
+
if (this.unknownLayoutPaneKey !== null && now - this.unknownLayoutStableSince >= UNKNOWN_LAYOUT_STABLE_MS) {
|
|
3796
|
+
if (!this.unknownLayoutWarned) {
|
|
3797
|
+
this.unknownLayoutWarned = true;
|
|
3798
|
+
this.logger.warn({ backend: this.backend?.binaryName }, "Unrecognised idle footer layout; falling back to a stable empty composer for readiness");
|
|
3799
|
+
}
|
|
3800
|
+
snapshot = this.instanceStateMachine.observe(pane, now, {
|
|
3801
|
+
settled: true,
|
|
3802
|
+
changeAt: observedChangeAt,
|
|
3803
|
+
forceReady: true,
|
|
3804
|
+
});
|
|
3805
|
+
}
|
|
3806
|
+
else {
|
|
3807
|
+
snapshot = this.instanceStateMachine.recordOutput(observedChangeAt);
|
|
3808
|
+
}
|
|
3705
3809
|
}
|
|
3706
3810
|
}
|
|
3707
3811
|
else if (structuredPeriodicIdle && reason !== "idle_debounce" && !settled) {
|
|
@@ -4182,6 +4286,9 @@ export class Daemon extends EventEmitter {
|
|
|
4182
4286
|
if (meta.reply_to_text) {
|
|
4183
4287
|
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
4184
4288
|
}
|
|
4289
|
+
if (meta.delivery_id) {
|
|
4290
|
+
formatted = `[agend-delivery-id:${meta.delivery_id}]\n${formatted}`;
|
|
4291
|
+
}
|
|
4185
4292
|
return formatted;
|
|
4186
4293
|
}
|
|
4187
4294
|
/**
|
|
@@ -4204,7 +4311,10 @@ export class Daemon extends EventEmitter {
|
|
|
4204
4311
|
verdict.reached = true;
|
|
4205
4312
|
verdict.phase = phase;
|
|
4206
4313
|
verdict.proof = proof;
|
|
4207
|
-
|
|
4314
|
+
// Durable cross-instance rows are still retryable before a pane write. Their
|
|
4315
|
+
// terminal state is announced by the outbox only after its retry/TTL policy
|
|
4316
|
+
// is exhausted; emitting ❌ here would expose a transient gate timeout.
|
|
4317
|
+
if (status && !verdict.durableAttempt)
|
|
4208
4318
|
this.emit("message_failed", status); // ❌
|
|
4209
4319
|
return false;
|
|
4210
4320
|
}
|
|
@@ -4240,8 +4350,10 @@ export class Daemon extends EventEmitter {
|
|
|
4240
4350
|
* normal inbound message, with a steering notice prepended for context.
|
|
4241
4351
|
*/
|
|
4242
4352
|
steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
4243
|
-
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
4353
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4354
|
+
this.deferDurableDelivery(meta, "delivery epoch changed before steer began");
|
|
4244
4355
|
return;
|
|
4356
|
+
}
|
|
4245
4357
|
// The fleet facade prevents a steer from overtaking an item still in its
|
|
4246
4358
|
// idle queue. Close the second race window here: an earlier item may already
|
|
4247
4359
|
// have reached this daemon's paste queue by the time the steer IPC arrives.
|
|
@@ -4256,25 +4368,60 @@ export class Daemon extends EventEmitter {
|
|
|
4256
4368
|
const formatted = "[STEERING — mid-task course correction. Fold this into the CURRENT work if one is active.]\n"
|
|
4257
4369
|
+ this.formatInboundMessage(content, meta);
|
|
4258
4370
|
const status = channelStatus(meta);
|
|
4371
|
+
const durableAttempt = this.durableDeliveryAttempt(meta);
|
|
4372
|
+
if (durableAttempt === false) {
|
|
4373
|
+
this.deferDurableDelivery(meta, "invalid durable delivery attempt metadata");
|
|
4374
|
+
return;
|
|
4375
|
+
}
|
|
4259
4376
|
this.steerLock = this.steerLock.then(async () => {
|
|
4260
|
-
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
4377
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4378
|
+
this.deferDurableDelivery(meta, "delivery epoch changed before steer began");
|
|
4261
4379
|
return;
|
|
4380
|
+
}
|
|
4262
4381
|
await this.wake();
|
|
4263
|
-
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
4382
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4383
|
+
this.deferDurableDelivery(meta, "delivery epoch changed while waking for steer");
|
|
4264
4384
|
return;
|
|
4385
|
+
}
|
|
4265
4386
|
const verdict = { reached: false };
|
|
4266
|
-
|
|
4267
|
-
this.
|
|
4387
|
+
try {
|
|
4388
|
+
if (await this.deliverMessage(formatted, status, {
|
|
4389
|
+
steer: true,
|
|
4390
|
+
deliveryEpoch,
|
|
4391
|
+
submissionId: meta.message_id,
|
|
4392
|
+
verdict,
|
|
4393
|
+
durableAttempt: durableAttempt || undefined,
|
|
4394
|
+
})) {
|
|
4395
|
+
this.finishDurableSubmission(durableAttempt, verdict);
|
|
4396
|
+
this.markTurnStarted(meta, formatted);
|
|
4397
|
+
}
|
|
4398
|
+
else if (durableAttempt && verdict.paneWriteStarted) {
|
|
4399
|
+
this.finishDurableDelivery(durableAttempt, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
|
|
4400
|
+
}
|
|
4401
|
+
else if (durableAttempt && verdict.durableBeginCommitted) {
|
|
4402
|
+
this.abortDurableDelivery(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "no-pane-write"}`);
|
|
4403
|
+
}
|
|
4404
|
+
else if (durableAttempt) {
|
|
4405
|
+
this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "deferred"}`);
|
|
4406
|
+
}
|
|
4407
|
+
else if (verdict.reached && this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4408
|
+
// Legacy non-outbox path retains its existing failure surface.
|
|
4409
|
+
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4410
|
+
}
|
|
4268
4411
|
}
|
|
4269
|
-
|
|
4270
|
-
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4412
|
+
catch (err) {
|
|
4413
|
+
if (durableAttempt && verdict.paneWriteStarted)
|
|
4414
|
+
this.finishDurableDelivery(durableAttempt, "uncertain", err.message);
|
|
4415
|
+
else if (durableAttempt && verdict.durableBeginCommitted)
|
|
4416
|
+
this.abortDurableDelivery(durableAttempt, err.message);
|
|
4417
|
+
else if (durableAttempt)
|
|
4418
|
+
this.retryDurableDeliveryBeforeBegin(durableAttempt, err.message);
|
|
4419
|
+
throw err;
|
|
4274
4420
|
}
|
|
4275
4421
|
}).catch(err => {
|
|
4422
|
+
this.deferDurableDelivery(meta, `target wake/steer failed before pane submission: ${err.message}`);
|
|
4276
4423
|
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
4277
|
-
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4424
|
+
if (!durableAttempt && this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4278
4425
|
this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
|
|
4279
4426
|
}
|
|
4280
4427
|
});
|
|
@@ -4315,10 +4462,12 @@ export class Daemon extends EventEmitter {
|
|
|
4315
4462
|
pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
|
|
4316
4463
|
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4317
4464
|
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
4465
|
+
this.deferDurableDelivery(meta, "delivery epoch changed before queue admission");
|
|
4318
4466
|
return;
|
|
4319
4467
|
}
|
|
4320
4468
|
if (!this.tmux) {
|
|
4321
4469
|
this.logger.warn("Cannot push channel message: tmux not running");
|
|
4470
|
+
this.deferDurableDelivery(meta, "target tmux is unavailable");
|
|
4322
4471
|
return;
|
|
4323
4472
|
}
|
|
4324
4473
|
// Remember (and persist) the reply target. Only real channel messages have a
|
|
@@ -4375,9 +4524,12 @@ export class Daemon extends EventEmitter {
|
|
|
4375
4524
|
this.emit("message_queued", queuedStatus);
|
|
4376
4525
|
}
|
|
4377
4526
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
4527
|
+
let durable = null;
|
|
4528
|
+
let verdict = { reached: false };
|
|
4378
4529
|
try {
|
|
4379
4530
|
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4380
4531
|
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
4532
|
+
this.deferDurableDelivery(meta, "delivery epoch changed while queued");
|
|
4381
4533
|
return;
|
|
4382
4534
|
}
|
|
4383
4535
|
if (this.config.pre_task_command) {
|
|
@@ -4395,18 +4547,60 @@ export class Daemon extends EventEmitter {
|
|
|
4395
4547
|
// A fresh delivery begins a fresh turn — its bubble must not inherit
|
|
4396
4548
|
// the previous turn's tool list.
|
|
4397
4549
|
this.resetToolProgress();
|
|
4398
|
-
|
|
4399
|
-
|
|
4550
|
+
verdict = { reached: false };
|
|
4551
|
+
const durableAttempt = this.durableDeliveryAttempt(meta);
|
|
4552
|
+
if (durableAttempt === false) {
|
|
4553
|
+
this.deferDurableDelivery(meta, "invalid durable delivery attempt metadata");
|
|
4554
|
+
return;
|
|
4555
|
+
}
|
|
4556
|
+
durable = durableAttempt;
|
|
4557
|
+
if (await this.deliverMessage(formatted, status, {
|
|
4558
|
+
deliveryEpoch,
|
|
4559
|
+
submissionId: meta.message_id,
|
|
4560
|
+
verdict,
|
|
4561
|
+
durableAttempt: durableAttempt || undefined,
|
|
4562
|
+
})) {
|
|
4563
|
+
this.finishDurableSubmission(durable, verdict);
|
|
4400
4564
|
this.markTurnStarted(meta, formatted);
|
|
4401
4565
|
}
|
|
4566
|
+
else if (durableAttempt && verdict.paneWriteStarted) {
|
|
4567
|
+
this.finishDurableDelivery(durable, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
|
|
4568
|
+
}
|
|
4569
|
+
else if (durableAttempt && verdict.durableBeginCommitted) {
|
|
4570
|
+
this.abortDurableDelivery(durable, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "no-pane-write"}`);
|
|
4571
|
+
}
|
|
4572
|
+
else if (durableAttempt) {
|
|
4573
|
+
this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "deferred"}`);
|
|
4574
|
+
}
|
|
4402
4575
|
else if (meta.from_instance && verdict.reached
|
|
4403
4576
|
&& this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4404
4577
|
// Only a delivery that reached a verdict is reported. A pane that is
|
|
4405
4578
|
// not ready yet, a cancel, a storm hold or a shutdown all return
|
|
4406
4579
|
// false without one, and telling the sender its message was lost
|
|
4407
4580
|
// there would be the false ❌ of #826 in its other form.
|
|
4581
|
+
this.finishDurableDelivery(durable, "failed", `${verdict.phase ?? "delivery"}:${verdict.proof ?? "failed"}`);
|
|
4408
4582
|
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4409
4583
|
}
|
|
4584
|
+
else if (verdict.reached) {
|
|
4585
|
+
this.finishDurableDelivery(durable, "failed", `${verdict.phase ?? "delivery"}:${verdict.proof ?? "failed"}`);
|
|
4586
|
+
}
|
|
4587
|
+
else if (verdict.paneWriteStarted) {
|
|
4588
|
+
this.finishDurableDelivery(durable, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
|
|
4589
|
+
}
|
|
4590
|
+
else {
|
|
4591
|
+
this.abortDurableDelivery(durable, "delivery did not reach pane submission");
|
|
4592
|
+
}
|
|
4593
|
+
}
|
|
4594
|
+
catch (err) {
|
|
4595
|
+
if (durable && verdict.paneWriteStarted)
|
|
4596
|
+
this.finishDurableDelivery(durable, "uncertain", err.message);
|
|
4597
|
+
else if (durable && verdict.durableBeginCommitted)
|
|
4598
|
+
this.abortDurableDelivery(durable, err.message);
|
|
4599
|
+
else if (durable)
|
|
4600
|
+
this.retryDurableDeliveryBeforeBegin(durable, err.message);
|
|
4601
|
+
else
|
|
4602
|
+
throw err;
|
|
4603
|
+
this.logger.warn({ err: err.message }, "Durable pane delivery deferred after exception");
|
|
4410
4604
|
}
|
|
4411
4605
|
finally {
|
|
4412
4606
|
this.pasteQueueDepth--;
|
|
@@ -4439,6 +4633,8 @@ export class Daemon extends EventEmitter {
|
|
|
4439
4633
|
// The caller passes its own holder when it needs the answer; a system paste
|
|
4440
4634
|
// that ignores the outcome gets a throwaway.
|
|
4441
4635
|
const verdict = opts?.verdict ?? { reached: false };
|
|
4636
|
+
if (opts?.durableAttempt)
|
|
4637
|
+
verdict.durableAttempt = true;
|
|
4442
4638
|
const cancelled = () => opts?.deliveryEpoch !== undefined
|
|
4443
4639
|
&& !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
|
|
4444
4640
|
if (cancelled() || this.stormWindow?.isStopped())
|
|
@@ -4513,7 +4709,9 @@ export class Daemon extends EventEmitter {
|
|
|
4513
4709
|
// The pane never freed up. Report the failure instead of pasting into a
|
|
4514
4710
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
4515
4711
|
// would land on top of it) — and instead of holding the queue silently.
|
|
4516
|
-
this.logger.
|
|
4712
|
+
this.logger[verdict.durableAttempt ? "warn" : "error"](verdict.durableAttempt
|
|
4713
|
+
? "Pane still busy after the idle wait — durable delivery remains retryable"
|
|
4714
|
+
: "Pane still busy after the idle wait — reporting delivery failure");
|
|
4517
4715
|
return this.failDelivery(verdict, status, "readiness", "timeout-before-write");
|
|
4518
4716
|
}
|
|
4519
4717
|
}
|
|
@@ -4591,7 +4789,7 @@ export class Daemon extends EventEmitter {
|
|
|
4591
4789
|
}
|
|
4592
4790
|
if (!(await this.hasPositiveDeliveryInput()))
|
|
4593
4791
|
return "dialog";
|
|
4594
|
-
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict);
|
|
4792
|
+
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict, opts?.durableAttempt);
|
|
4595
4793
|
});
|
|
4596
4794
|
if (outcome === "spawn-started") {
|
|
4597
4795
|
// The pane changed under this delivery: its queued paste must not land
|
|
@@ -5364,17 +5562,31 @@ export class Daemon extends EventEmitter {
|
|
|
5364
5562
|
async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId,
|
|
5365
5563
|
// Last and defaulted so the positional callers that ignore the outcome stay
|
|
5366
5564
|
// readable; deliverMessage, the only one that reports, always passes its own.
|
|
5367
|
-
verdict = { reached: false }) {
|
|
5565
|
+
verdict = { reached: false }, durableAttempt) {
|
|
5368
5566
|
const signature = this.submissionSignature(formatted, submissionId);
|
|
5369
5567
|
let windowId = initialWindowId;
|
|
5370
5568
|
// Bug A: paste with backoff. Transient failures are usually a stale window id
|
|
5371
5569
|
// after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
|
|
5372
5570
|
const maxAttempts = 3;
|
|
5571
|
+
let beginChecked = false;
|
|
5373
5572
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
5374
5573
|
const pasteStartedAt = Date.now();
|
|
5375
5574
|
// Read the pane BEFORE writing to it, so the submission check can require
|
|
5376
5575
|
// evidence this paste ADDED rather than evidence that was already there.
|
|
5377
5576
|
const pasteBaseline = await this.capturePaneEvidence(signature);
|
|
5577
|
+
// Readiness, idle-gate, dialog, spawn and pane-lock waits have all ended.
|
|
5578
|
+
// Commit the submission fence at the last possible point before the
|
|
5579
|
+
// first side effect; a crash during those waits remains safely replayable.
|
|
5580
|
+
if (durableAttempt && !beginChecked) {
|
|
5581
|
+
beginChecked = true;
|
|
5582
|
+
if (!this.beginDurableDelivery(durableAttempt)) {
|
|
5583
|
+
verdict.durableBeginRejected = true;
|
|
5584
|
+
verdict.phase = "submission-begin";
|
|
5585
|
+
verdict.proof = "permit-rejected";
|
|
5586
|
+
return false;
|
|
5587
|
+
}
|
|
5588
|
+
verdict.durableBeginCommitted = true;
|
|
5589
|
+
}
|
|
5378
5590
|
const pasted = await this.tmux.pasteBuffer(formatted);
|
|
5379
5591
|
if (!pasted) {
|
|
5380
5592
|
const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
|
|
@@ -5392,6 +5604,10 @@ export class Daemon extends EventEmitter {
|
|
|
5392
5604
|
await new Promise(r => setTimeout(r, 2000));
|
|
5393
5605
|
continue;
|
|
5394
5606
|
}
|
|
5607
|
+
// From this point a process/window transition can leave the user payload
|
|
5608
|
+
// in the pane even if later proof is cancelled or unreadable. Such a row
|
|
5609
|
+
// must be uncertain, never aborted back to replayable queued state.
|
|
5610
|
+
verdict.paneWriteStarted = true;
|
|
5395
5611
|
// Settle the bracketed paste, then submit. When control mode can observe
|
|
5396
5612
|
// the pane, wait for the paste's own render to go quiet instead of
|
|
5397
5613
|
// trusting a fixed delay (see waitForPasteSettle). The fixed delays —
|
|
@@ -5475,6 +5691,8 @@ export class Daemon extends EventEmitter {
|
|
|
5475
5691
|
// Re-pasting on a backend without structured Codex pane evidence
|
|
5476
5692
|
// would risk duplication; retain its legacy best-effort behavior.
|
|
5477
5693
|
this.logger.warn("Paste reached the pane but could not be verified as submitted — accepting without proof");
|
|
5694
|
+
verdict.phase = "best-effort-submission";
|
|
5695
|
+
verdict.proof = "unverified";
|
|
5478
5696
|
if (status)
|
|
5479
5697
|
this.emit("message_confirmed", status); // ✅ (best-effort)
|
|
5480
5698
|
return true;
|
|
@@ -6081,6 +6299,7 @@ export class Daemon extends EventEmitter {
|
|
|
6081
6299
|
const tool = msg.tool;
|
|
6082
6300
|
const args = (msg.args ?? {});
|
|
6083
6301
|
const requestId = msg.requestId;
|
|
6302
|
+
const operationId = typeof msg.operationId === "string" ? msg.operationId : undefined;
|
|
6084
6303
|
this.logger.debug({ tool, requestId }, "Tool call from MCP server");
|
|
6085
6304
|
// A tool call can only come from a live MCP server of ours — proof of life
|
|
6086
6305
|
// that outranks whatever the pid slot says.
|
|
@@ -6103,10 +6322,13 @@ export class Daemon extends EventEmitter {
|
|
|
6103
6322
|
if (!error && result != null && TURN_OUTBOUND_TOOLS.has(tool)) {
|
|
6104
6323
|
this.turnReplyGuard.settleToolAttempt(replyAttempt, true);
|
|
6105
6324
|
}
|
|
6106
|
-
const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
|
|
6325
|
+
const sent = this.ipcServer?.send(socket, { requestId, result, error, operationId }) ?? false;
|
|
6107
6326
|
if (!sent) {
|
|
6108
6327
|
this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
|
|
6109
6328
|
}
|
|
6329
|
+
else if (operationId) {
|
|
6330
|
+
this.emit("mcp_tool_response_written", { operationId, daemonBootId: this.bootId });
|
|
6331
|
+
}
|
|
6110
6332
|
};
|
|
6111
6333
|
// Repo checkout — handled locally in daemon (no fleet-manager)
|
|
6112
6334
|
if (tool === "checkout_repo") {
|
|
@@ -6231,6 +6453,7 @@ export class Daemon extends EventEmitter {
|
|
|
6231
6453
|
type: "fleet_outbound",
|
|
6232
6454
|
tool,
|
|
6233
6455
|
args,
|
|
6456
|
+
...(operationId ? { operationId, sourceDaemonBootId: this.bootId } : {}),
|
|
6234
6457
|
fleetRequestId: fleetReqId,
|
|
6235
6458
|
senderSessionName,
|
|
6236
6459
|
});
|
|
@@ -6289,6 +6512,7 @@ export class Daemon extends EventEmitter {
|
|
|
6289
6512
|
type: "fleet_outbound",
|
|
6290
6513
|
tool,
|
|
6291
6514
|
args,
|
|
6515
|
+
...(operationId ? { operationId, sourceDaemonBootId: this.bootId } : {}),
|
|
6292
6516
|
fleetRequestId: fleetReqId,
|
|
6293
6517
|
// Preserve the exact adapter world that supplied the chat context. An
|
|
6294
6518
|
// instance binding is only a fallback: persisted/runtime context can be
|