@evident-ai/cli 3.1.1-dev.fe0815c → 3.2.1-dev.04ef8b1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -2
- package/dist/index.js +976 -145
- package/dist/index.js.map +1 -1
- package/package.json +4 -2
package/dist/index.js
CHANGED
|
@@ -212,6 +212,7 @@ var api = {
|
|
|
212
212
|
|
|
213
213
|
// src/lib/keychain.ts
|
|
214
214
|
var SERVICE_NAME = "evident-cli";
|
|
215
|
+
var keytarWarned = false;
|
|
215
216
|
async function getKeytar() {
|
|
216
217
|
try {
|
|
217
218
|
const keytar = await import("keytar");
|
|
@@ -219,7 +220,13 @@ async function getKeytar() {
|
|
|
219
220
|
return null;
|
|
220
221
|
}
|
|
221
222
|
return keytar;
|
|
222
|
-
} catch {
|
|
223
|
+
} catch (err) {
|
|
224
|
+
if (!keytarWarned) {
|
|
225
|
+
keytarWarned = true;
|
|
226
|
+
console.warn(
|
|
227
|
+
`System keychain unavailable, falling back to file-based credential storage: ${err instanceof Error ? err.message : String(err)}`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
223
230
|
return null;
|
|
224
231
|
}
|
|
225
232
|
}
|
|
@@ -368,8 +375,9 @@ async function deviceFlowLogin(options) {
|
|
|
368
375
|
await waitForEnter("Press Enter to open the browser...");
|
|
369
376
|
try {
|
|
370
377
|
await open(verification_uri);
|
|
371
|
-
} catch {
|
|
372
|
-
|
|
378
|
+
} catch (error2) {
|
|
379
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
380
|
+
console.log(chalk2.dim(`Could not open browser (${message}). Please visit the URL manually.`));
|
|
373
381
|
}
|
|
374
382
|
}
|
|
375
383
|
const spinner = ora("Waiting for authentication...").start();
|
|
@@ -797,6 +805,16 @@ async function checkStatus(jsonMode) {
|
|
|
797
805
|
exitCode: 1
|
|
798
806
|
};
|
|
799
807
|
}
|
|
808
|
+
if (response.status === 404) {
|
|
809
|
+
return {
|
|
810
|
+
ok: false,
|
|
811
|
+
endpoint: apiUrl,
|
|
812
|
+
authLabel: authLabelFor(credentials2),
|
|
813
|
+
reason: "endpoint_not_found",
|
|
814
|
+
error: `${apiUrl}/me returned HTTP 404 \u2014 that endpoint has no /me route, so it is probably missing the /v1 prefix. The credentials were NOT validated.`,
|
|
815
|
+
exitCode: 75
|
|
816
|
+
};
|
|
817
|
+
}
|
|
800
818
|
if (response.status >= 500) {
|
|
801
819
|
const serverMessage = await readErrorMessage(response);
|
|
802
820
|
return {
|
|
@@ -899,14 +917,25 @@ function readClaudeCliCredentials() {
|
|
|
899
917
|
{ encoding: "utf-8", timeout: 2e3, stdio: ["pipe", "pipe", "ignore"] }
|
|
900
918
|
);
|
|
901
919
|
return parseClaudeCliCredentials(raw);
|
|
902
|
-
} catch {
|
|
920
|
+
} catch (err) {
|
|
921
|
+
if (err.status !== 44) {
|
|
922
|
+
console.warn(
|
|
923
|
+
`readClaudeCliCredentials: security find-generic-password failed: ${err instanceof Error ? err.message : String(err)}`
|
|
924
|
+
);
|
|
925
|
+
}
|
|
903
926
|
return null;
|
|
904
927
|
}
|
|
905
928
|
}
|
|
906
929
|
try {
|
|
907
930
|
const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
|
|
908
931
|
return parseClaudeCliCredentials(raw);
|
|
909
|
-
} catch {
|
|
932
|
+
} catch (err) {
|
|
933
|
+
const code = err.code;
|
|
934
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
935
|
+
console.warn(
|
|
936
|
+
`readClaudeCliCredentials: reading .claude/.credentials.json failed: ${err instanceof Error ? err.message : String(err)}`
|
|
937
|
+
);
|
|
938
|
+
}
|
|
910
939
|
return null;
|
|
911
940
|
}
|
|
912
941
|
}
|
|
@@ -994,7 +1023,7 @@ async function claudeUsage() {
|
|
|
994
1023
|
|
|
995
1024
|
// src/commands/run.ts
|
|
996
1025
|
import { homedir as homedir3 } from "os";
|
|
997
|
-
import { isAbsolute as isAbsolute2, join as
|
|
1026
|
+
import { isAbsolute as isAbsolute2, join as join4, parse, resolve as resolvePath } from "path";
|
|
998
1027
|
import chalk6 from "chalk";
|
|
999
1028
|
|
|
1000
1029
|
// ../../packages/types/src/agents/index.ts
|
|
@@ -1023,6 +1052,7 @@ var MAX_FILE_SYNC_DIRECTORIES = 16;
|
|
|
1023
1052
|
|
|
1024
1053
|
// ../../packages/types/src/logging/index.ts
|
|
1025
1054
|
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
1055
|
+
var FORWARD_FAILURE_REASON_HEADER = "X-Evident-Failure-Reason";
|
|
1026
1056
|
function log(level, event, fields) {
|
|
1027
1057
|
const method = level === "debug" ? "log" : level;
|
|
1028
1058
|
try {
|
|
@@ -1404,7 +1434,10 @@ function findOpenCodeProcesses() {
|
|
|
1404
1434
|
}
|
|
1405
1435
|
}
|
|
1406
1436
|
}
|
|
1407
|
-
} catch {
|
|
1437
|
+
} catch (err) {
|
|
1438
|
+
console.warn(
|
|
1439
|
+
`findOpenCodeProcesses: ps fallback failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1440
|
+
);
|
|
1408
1441
|
}
|
|
1409
1442
|
}
|
|
1410
1443
|
for (const pid of pids) {
|
|
@@ -1427,7 +1460,10 @@ function findOpenCodeProcesses() {
|
|
|
1427
1460
|
}
|
|
1428
1461
|
}
|
|
1429
1462
|
}
|
|
1430
|
-
} catch {
|
|
1463
|
+
} catch (err) {
|
|
1464
|
+
console.warn(
|
|
1465
|
+
`findOpenCodeProcesses: process detection failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1466
|
+
);
|
|
1431
1467
|
}
|
|
1432
1468
|
return instances;
|
|
1433
1469
|
}
|
|
@@ -1501,7 +1537,12 @@ function stopOpenCode(opencodeProcess) {
|
|
|
1501
1537
|
} else {
|
|
1502
1538
|
process.kill(-opencodeProcess.pid, "SIGTERM");
|
|
1503
1539
|
}
|
|
1504
|
-
} catch {
|
|
1540
|
+
} catch (err) {
|
|
1541
|
+
if (err.code !== "ESRCH") {
|
|
1542
|
+
console.warn(
|
|
1543
|
+
`stopOpenCode: kill failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1544
|
+
);
|
|
1545
|
+
}
|
|
1505
1546
|
}
|
|
1506
1547
|
}
|
|
1507
1548
|
|
|
@@ -2066,7 +2107,9 @@ function messageRunState(messages, userMessageId) {
|
|
|
2066
2107
|
}
|
|
2067
2108
|
if (!reply) return "queued";
|
|
2068
2109
|
if (isAssistantInFlight(reply)) return "running";
|
|
2069
|
-
|
|
2110
|
+
if (errorOf(reply) != null) return "failed";
|
|
2111
|
+
if (isAmbiguousTerminalFinish(reply)) return "running";
|
|
2112
|
+
return "done";
|
|
2070
2113
|
}
|
|
2071
2114
|
function isPreamblePinnedRunning(messages, userMessageId) {
|
|
2072
2115
|
if (messageRunState(messages, userMessageId) !== "running") return false;
|
|
@@ -2076,6 +2119,19 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
2076
2119
|
function isB2AbandonmentConfirmed(params) {
|
|
2077
2120
|
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
2078
2121
|
}
|
|
2122
|
+
function isAmbiguousTerminalFinish(m) {
|
|
2123
|
+
if (completedOf(m) == null) return false;
|
|
2124
|
+
if (errorOf(m) != null) return false;
|
|
2125
|
+
const finish = finishOf(m);
|
|
2126
|
+
return finish !== "tool-calls" && finish !== "stop";
|
|
2127
|
+
}
|
|
2128
|
+
function isAmbiguousFinishPinnedRunning(messages, userMessageId) {
|
|
2129
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2130
|
+
return isAmbiguousTerminalFinish(reply);
|
|
2131
|
+
}
|
|
2132
|
+
function isAmbiguousFinishResolved(params) {
|
|
2133
|
+
return params.sessionOngoing === false || params.pinnedForMs >= params.maxPinnedMs;
|
|
2134
|
+
}
|
|
2079
2135
|
function messageError(messages, userMessageId) {
|
|
2080
2136
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2081
2137
|
const error2 = errorOf(reply);
|
|
@@ -2089,6 +2145,21 @@ function messageError(messages, userMessageId) {
|
|
|
2089
2145
|
}
|
|
2090
2146
|
return "The agent run failed.";
|
|
2091
2147
|
}
|
|
2148
|
+
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2149
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2150
|
+
const error2 = errorOf(reply);
|
|
2151
|
+
if (error2 == null) return false;
|
|
2152
|
+
if (typeof error2 === "string") return error2.trim() === "Aborted";
|
|
2153
|
+
if (typeof error2 === "object") {
|
|
2154
|
+
const e = error2;
|
|
2155
|
+
if (e.name === "MessageAbortedError") return true;
|
|
2156
|
+
if (e.name === "AbortError") return true;
|
|
2157
|
+
const dataMessage = e.data?.message;
|
|
2158
|
+
const rendered = typeof dataMessage === "string" ? dataMessage : typeof e.message === "string" ? e.message : null;
|
|
2159
|
+
return rendered != null && rendered.trim() === "Aborted";
|
|
2160
|
+
}
|
|
2161
|
+
return false;
|
|
2162
|
+
}
|
|
2092
2163
|
function messageFailure(messages, userMessageId) {
|
|
2093
2164
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2094
2165
|
const error2 = errorOf(reply);
|
|
@@ -2267,6 +2338,149 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
2267
2338
|
return { enabled, maxAgeMs, maxCount, intervalMs, warnings };
|
|
2268
2339
|
}
|
|
2269
2340
|
|
|
2341
|
+
// src/lib/opencode/session-db-size.ts
|
|
2342
|
+
import { statSync as statSync2 } from "fs";
|
|
2343
|
+
import { join as join2 } from "path";
|
|
2344
|
+
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
2345
|
+
function statSessionDbBytes(homeDir) {
|
|
2346
|
+
const dbPath = join2(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
2347
|
+
try {
|
|
2348
|
+
return statSync2(dbPath).size;
|
|
2349
|
+
} catch (err) {
|
|
2350
|
+
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
2351
|
+
if (!isMissingFile) {
|
|
2352
|
+
console.error(
|
|
2353
|
+
`[statSessionDbBytes] could not stat ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
2354
|
+
);
|
|
2355
|
+
}
|
|
2356
|
+
return null;
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
function buildSessionStoreSizeWarning(input) {
|
|
2360
|
+
const { dbBytes, cleanupEnabled, reclaimSkipReason } = input;
|
|
2361
|
+
if (dbBytes === null || dbBytes <= LARGE_DB_THRESHOLD_BYTES) return null;
|
|
2362
|
+
const mib = Math.round(dbBytes / 1024 / 1024);
|
|
2363
|
+
if (!cleanupEnabled) {
|
|
2364
|
+
return `Session store is large: opencode.db is ${mib} MiB and automatic session cleanup is off. Enable it with --session-cleanup-max-age 24h (env EVIDENT_SESSION_CLEANUP_MAX_AGE) to stop it growing \u2014 a store much larger than this can exceed a hosted runner's session-history restore budget on the next start, losing this runner's session history.`;
|
|
2365
|
+
}
|
|
2366
|
+
if (reclaimSkipReason === "sqlite-unavailable" || reclaimSkipReason === "insufficient-disk-space") {
|
|
2367
|
+
const reasonText = reclaimSkipReason === "sqlite-unavailable" ? "this Node runtime lacks node:sqlite (needs Node >=22.5)" : "there is not enough free disk space to compact it";
|
|
2368
|
+
return `Session store is large: opencode.db is ${mib} MiB. Automatic session cleanup is on, but space cannot currently be reclaimed because ${reasonText} \u2014 a store this large can exceed a hosted runner's session-history restore budget on the next start, losing this runner's session history.`;
|
|
2369
|
+
}
|
|
2370
|
+
return null;
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
2374
|
+
import { statSync as statSync3, statfsSync } from "fs";
|
|
2375
|
+
import { dirname as dirname2 } from "path";
|
|
2376
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
2377
|
+
try {
|
|
2378
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
2379
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
2380
|
+
if (availableBytes < requiredBytes) {
|
|
2381
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
2382
|
+
}
|
|
2383
|
+
return null;
|
|
2384
|
+
} catch (err) {
|
|
2385
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
2386
|
+
}
|
|
2387
|
+
}
|
|
2388
|
+
function readLogicalBytes(db) {
|
|
2389
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
2390
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
2391
|
+
return pageCount * pageSize;
|
|
2392
|
+
}
|
|
2393
|
+
function readCheckpointResult(db) {
|
|
2394
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
2395
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
2396
|
+
}
|
|
2397
|
+
async function probeReclaimAvailability(input) {
|
|
2398
|
+
const { dbPath, requiredBytes } = input;
|
|
2399
|
+
let sqlite;
|
|
2400
|
+
try {
|
|
2401
|
+
sqlite = await import("sqlite");
|
|
2402
|
+
} catch (err) {
|
|
2403
|
+
console.warn(
|
|
2404
|
+
`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
2405
|
+
);
|
|
2406
|
+
return "sqlite-unavailable";
|
|
2407
|
+
}
|
|
2408
|
+
let autoVacuum = null;
|
|
2409
|
+
try {
|
|
2410
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
2411
|
+
try {
|
|
2412
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2413
|
+
} finally {
|
|
2414
|
+
db.close();
|
|
2415
|
+
}
|
|
2416
|
+
} catch (err) {
|
|
2417
|
+
console.warn(
|
|
2418
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
2419
|
+
);
|
|
2420
|
+
}
|
|
2421
|
+
if (autoVacuum !== 0) return null;
|
|
2422
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? "insufficient-disk-space" : null;
|
|
2423
|
+
}
|
|
2424
|
+
async function reclaimSessionDbSpace(input) {
|
|
2425
|
+
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
2426
|
+
let sqlite;
|
|
2427
|
+
try {
|
|
2428
|
+
sqlite = await import("sqlite");
|
|
2429
|
+
} catch (err) {
|
|
2430
|
+
console.warn(
|
|
2431
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
2432
|
+
);
|
|
2433
|
+
return { ok: false, skipped: "sqlite-unavailable" };
|
|
2434
|
+
}
|
|
2435
|
+
const { DatabaseSync } = sqlite;
|
|
2436
|
+
let db;
|
|
2437
|
+
try {
|
|
2438
|
+
db = new DatabaseSync(dbPath);
|
|
2439
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
2440
|
+
if (autoVacuum === 0) {
|
|
2441
|
+
if (!allowFullVacuum) {
|
|
2442
|
+
console.warn(
|
|
2443
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
2444
|
+
);
|
|
2445
|
+
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
2446
|
+
}
|
|
2447
|
+
const fileBytesForGuard = statSync3(dbPath).size;
|
|
2448
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
2449
|
+
if (skipReason !== null) {
|
|
2450
|
+
console.warn(
|
|
2451
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
2452
|
+
);
|
|
2453
|
+
return { ok: false, skipped: "insufficient-disk-space" };
|
|
2454
|
+
}
|
|
2455
|
+
const beforeBytes = readLogicalBytes(db);
|
|
2456
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
2457
|
+
db.exec("VACUUM");
|
|
2458
|
+
const afterBytes = readLogicalBytes(db);
|
|
2459
|
+
const checkpoint = readCheckpointResult(db);
|
|
2460
|
+
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
2461
|
+
}
|
|
2462
|
+
if (autoVacuum === 2) {
|
|
2463
|
+
const beforeBytes = readLogicalBytes(db);
|
|
2464
|
+
const bound = Math.max(0, Math.trunc(maxPages));
|
|
2465
|
+
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
2466
|
+
const afterBytes = readLogicalBytes(db);
|
|
2467
|
+
const checkpoint = readCheckpointResult(db);
|
|
2468
|
+
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
2469
|
+
}
|
|
2470
|
+
console.warn(
|
|
2471
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
2472
|
+
);
|
|
2473
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
2474
|
+
} catch (err) {
|
|
2475
|
+
console.error(
|
|
2476
|
+
`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
2477
|
+
);
|
|
2478
|
+
return { ok: false, skipped: "reclaim-error" };
|
|
2479
|
+
} finally {
|
|
2480
|
+
db?.close();
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
|
|
2270
2484
|
// src/lib/tunnel/connection.ts
|
|
2271
2485
|
import WebSocket2 from "ws";
|
|
2272
2486
|
|
|
@@ -2432,6 +2646,17 @@ var StreamForwarder = class {
|
|
|
2432
2646
|
};
|
|
2433
2647
|
|
|
2434
2648
|
// src/lib/tunnel/connection.ts
|
|
2649
|
+
var FAILURE_REASON_HEADER_LC = FORWARD_FAILURE_REASON_HEADER.toLowerCase();
|
|
2650
|
+
var TunnelUpgradeRejectedError = class extends Error {
|
|
2651
|
+
constructor(message, reason) {
|
|
2652
|
+
super(message);
|
|
2653
|
+
this.reason = reason;
|
|
2654
|
+
}
|
|
2655
|
+
};
|
|
2656
|
+
function classifyUpgradeRejection(headers) {
|
|
2657
|
+
const value = headers[FAILURE_REASON_HEADER_LC];
|
|
2658
|
+
return value === "do_code_updated" ? "do_code_updated" : "unknown";
|
|
2659
|
+
}
|
|
2435
2660
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
2436
2661
|
var BASE_RECONNECT_DELAY = 500;
|
|
2437
2662
|
function getReconnectDelay(attempt) {
|
|
@@ -2476,6 +2701,7 @@ function connectTunnel(options) {
|
|
|
2476
2701
|
onError,
|
|
2477
2702
|
onResponse,
|
|
2478
2703
|
onInfo,
|
|
2704
|
+
onWarning,
|
|
2479
2705
|
onDrainPing
|
|
2480
2706
|
} = options;
|
|
2481
2707
|
const tunnelUrl = getTunnelUrlConfig();
|
|
@@ -2495,8 +2721,11 @@ function connectTunnel(options) {
|
|
|
2495
2721
|
reject(new Error("Connection timeout"));
|
|
2496
2722
|
}, 3e4);
|
|
2497
2723
|
let upgradeRejection = null;
|
|
2724
|
+
let upgradeRejectionReason = null;
|
|
2498
2725
|
ws.on("unexpected-response", (_req, res) => {
|
|
2499
2726
|
clearTimeout(connectionTimeout);
|
|
2727
|
+
const reason = classifyUpgradeRejection(res.headers);
|
|
2728
|
+
upgradeRejectionReason = reason;
|
|
2500
2729
|
const chunks = [];
|
|
2501
2730
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
2502
2731
|
res.on("end", () => {
|
|
@@ -2510,8 +2739,14 @@ function connectTunnel(options) {
|
|
|
2510
2739
|
}
|
|
2511
2740
|
const statusLine = `HTTP ${res.statusCode}${res.statusMessage ? ` ${res.statusMessage}` : ""}`;
|
|
2512
2741
|
upgradeRejection = detail ? `${statusLine}: ${detail}` : statusLine;
|
|
2513
|
-
|
|
2514
|
-
|
|
2742
|
+
if (reason === "do_code_updated") {
|
|
2743
|
+
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
2744
|
+
} else {
|
|
2745
|
+
onError?.(`Tunnel refused by relay (${upgradeRejection})`);
|
|
2746
|
+
}
|
|
2747
|
+
reject(
|
|
2748
|
+
new TunnelUpgradeRejectedError(`Tunnel handshake rejected: ${upgradeRejection}`, reason)
|
|
2749
|
+
);
|
|
2515
2750
|
});
|
|
2516
2751
|
});
|
|
2517
2752
|
ws.on("open", () => {
|
|
@@ -2557,8 +2792,14 @@ function connectTunnel(options) {
|
|
|
2557
2792
|
ws.on("error", (error2) => {
|
|
2558
2793
|
clearTimeout(connectionTimeout);
|
|
2559
2794
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
2560
|
-
|
|
2561
|
-
|
|
2795
|
+
if (upgradeRejectionReason === "do_code_updated") {
|
|
2796
|
+
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
2797
|
+
} else {
|
|
2798
|
+
onError?.(`Connection error: ${detail}`);
|
|
2799
|
+
}
|
|
2800
|
+
reject(
|
|
2801
|
+
upgradeRejectionReason !== null ? new TunnelUpgradeRejectedError(detail, upgradeRejectionReason) : new Error(detail)
|
|
2802
|
+
);
|
|
2562
2803
|
});
|
|
2563
2804
|
ws.on("close", (code, reason) => {
|
|
2564
2805
|
const reasonStr = reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
@@ -2634,7 +2875,8 @@ var RunnerConnection = class {
|
|
|
2634
2875
|
onError: (error2) => events.onError?.(error2),
|
|
2635
2876
|
onResponse: () => events.onResponse?.(),
|
|
2636
2877
|
onDrainPing: () => events.onDrainPing?.(),
|
|
2637
|
-
onInfo: (message) => events.onInfo?.(message)
|
|
2878
|
+
onInfo: (message) => events.onInfo?.(message),
|
|
2879
|
+
onWarning: (message) => events.onWarning?.(message)
|
|
2638
2880
|
});
|
|
2639
2881
|
return;
|
|
2640
2882
|
} catch (error2) {
|
|
@@ -2645,7 +2887,12 @@ var RunnerConnection = class {
|
|
|
2645
2887
|
}
|
|
2646
2888
|
const delay = getReconnectDelay(this.reconnectAttempt);
|
|
2647
2889
|
events.onReconnecting?.(this.reconnectAttempt);
|
|
2648
|
-
|
|
2890
|
+
const retryMessage = `Connection failed, retrying in ${Math.round(delay / 1e3)}s...`;
|
|
2891
|
+
if (error2 instanceof TunnelUpgradeRejectedError && error2.reason === "do_code_updated") {
|
|
2892
|
+
events.onWarning?.(retryMessage);
|
|
2893
|
+
} else {
|
|
2894
|
+
events.onError?.(retryMessage);
|
|
2895
|
+
}
|
|
2649
2896
|
await this.sleep(delay);
|
|
2650
2897
|
}
|
|
2651
2898
|
}
|
|
@@ -2702,7 +2949,7 @@ import { homedir as homedir2 } from "os";
|
|
|
2702
2949
|
// src/lib/file-push.ts
|
|
2703
2950
|
import { randomUUID } from "crypto";
|
|
2704
2951
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2705
|
-
import { basename, dirname as
|
|
2952
|
+
import { basename, dirname as dirname3, isAbsolute, join as join3, relative, resolve as resolve2, sep } from "path";
|
|
2706
2953
|
var FILE_MODE = 384;
|
|
2707
2954
|
var DIRECTORY_MODE = 448;
|
|
2708
2955
|
async function writePushedFile(request) {
|
|
@@ -2733,9 +2980,9 @@ async function writePushedFile(request) {
|
|
|
2733
2980
|
}
|
|
2734
2981
|
try {
|
|
2735
2982
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2736
|
-
|
|
2983
|
+
dirname3(candidate)
|
|
2737
2984
|
);
|
|
2738
|
-
const realTarget =
|
|
2985
|
+
const realTarget = join3(existingAncestor, ...missingSegments, basename(candidate));
|
|
2739
2986
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
2740
2987
|
if (allowedDirectory === null) {
|
|
2741
2988
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
@@ -2745,8 +2992,8 @@ async function writePushedFile(request) {
|
|
|
2745
2992
|
}
|
|
2746
2993
|
if (missingSegments.length > 0) {
|
|
2747
2994
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2748
|
-
const realParent = await realpath(
|
|
2749
|
-
if (realParent !==
|
|
2995
|
+
const realParent = await realpath(dirname3(realTarget));
|
|
2996
|
+
if (realParent !== dirname3(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2750
2997
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2751
2998
|
path: realTarget,
|
|
2752
2999
|
bytes,
|
|
@@ -2771,7 +3018,7 @@ function expandAndValidate(requestedPath, homeDir) {
|
|
|
2771
3018
|
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
2772
3019
|
return null;
|
|
2773
3020
|
}
|
|
2774
|
-
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ?
|
|
3021
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join3(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
2775
3022
|
if (expanded.split(/[/\\]/).includes("..")) {
|
|
2776
3023
|
return null;
|
|
2777
3024
|
}
|
|
@@ -2789,7 +3036,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
2789
3036
|
try {
|
|
2790
3037
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
2791
3038
|
} catch (err) {
|
|
2792
|
-
const parent =
|
|
3039
|
+
const parent = dirname3(current);
|
|
2793
3040
|
if (err.code !== "ENOENT" || parent === current) {
|
|
2794
3041
|
throw err;
|
|
2795
3042
|
}
|
|
@@ -2844,13 +3091,13 @@ function contains(realDirectory, realTarget) {
|
|
|
2844
3091
|
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
2845
3092
|
let current = existingAncestor;
|
|
2846
3093
|
for (const segment of missingSegments) {
|
|
2847
|
-
current =
|
|
3094
|
+
current = join3(current, segment);
|
|
2848
3095
|
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
2849
3096
|
await chmod(current, DIRECTORY_MODE);
|
|
2850
3097
|
}
|
|
2851
3098
|
}
|
|
2852
3099
|
async function writeAtomically(realTarget, content) {
|
|
2853
|
-
const temporaryPath =
|
|
3100
|
+
const temporaryPath = join3(dirname3(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2854
3101
|
let handle;
|
|
2855
3102
|
try {
|
|
2856
3103
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -3123,9 +3370,11 @@ var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
|
3123
3370
|
var HEARTBEAT_MS = 6e4;
|
|
3124
3371
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
3125
3372
|
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
3373
|
+
var AMBIGUOUS_FINISH_MAX_PINNED_MS = 3 * 6e4;
|
|
3126
3374
|
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
3127
3375
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3128
3376
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3377
|
+
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
3129
3378
|
var ChannelAuthError = class extends Error {
|
|
3130
3379
|
constructor(message) {
|
|
3131
3380
|
super(message);
|
|
@@ -3148,6 +3397,10 @@ function backoffDelay(attempt, policy) {
|
|
|
3148
3397
|
function isRetryableStatus(status2) {
|
|
3149
3398
|
return status2 === 429 || status2 >= 500 && status2 <= 599;
|
|
3150
3399
|
}
|
|
3400
|
+
var VOLATILE_BODY_FIELD_PATTERN = /("(?:ref|requestId|request_id|traceId|trace_id)"\s*:\s*)"[^"]*"/gi;
|
|
3401
|
+
function normalizeRedrivePollFailureBody(body) {
|
|
3402
|
+
return body.replace(VOLATILE_BODY_FIELD_PATTERN, '$1"<redacted>"').replace(/\s+/g, " ").trim().slice(0, 200);
|
|
3403
|
+
}
|
|
3151
3404
|
var ChannelDriver = class _ChannelDriver {
|
|
3152
3405
|
agentId;
|
|
3153
3406
|
port;
|
|
@@ -3164,6 +3417,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3164
3417
|
now;
|
|
3165
3418
|
fileSyncDirectories;
|
|
3166
3419
|
homeDir;
|
|
3420
|
+
maxActiveSessions;
|
|
3167
3421
|
/** Cache of conversationId → opencode sessionId. */
|
|
3168
3422
|
sessions = /* @__PURE__ */ new Map();
|
|
3169
3423
|
/**
|
|
@@ -3279,6 +3533,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3279
3533
|
* ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
|
|
3280
3534
|
*/
|
|
3281
3535
|
redriveUnresolvedSince = /* @__PURE__ */ new Map();
|
|
3536
|
+
/**
|
|
3537
|
+
* Consecutive-identical-poll-failure streak for the re-drive fence (#1348),
|
|
3538
|
+
* keyed by Evident **message id** (not session) so `clearRedriveUnresolved`
|
|
3539
|
+
* can drop it with the other two trackers and it cannot leak. `sessionId` is
|
|
3540
|
+
* carried inside the entry, not the key: a session change is a different
|
|
3541
|
+
* situation and resets the streak, which gives the `(sessionId, message.id)`
|
|
3542
|
+
* pairing #1348 asks for without a composite map key.
|
|
3543
|
+
*/
|
|
3544
|
+
redrivePollFailures = /* @__PURE__ */ new Map();
|
|
3545
|
+
/**
|
|
3546
|
+
* "Already emitted `redrive_outcome_unreported` for THIS (message, outcome)
|
|
3547
|
+
* streak" (Class B, #1340: the runner DECIDED reattach/settle/fail_permanent
|
|
3548
|
+
* but its own PATCH to record it failed — distinct from Class A's
|
|
3549
|
+
* `redrive_poll_failed`, where opencode itself can't be observed). Keyed by
|
|
3550
|
+
* message id, valued by the outcome currently failing to report, so a
|
|
3551
|
+
* change of outcome starts a fresh signal. Cleared by
|
|
3552
|
+
* `clearRedriveUnresolved` the instant either PATCH succeeds.
|
|
3553
|
+
*/
|
|
3554
|
+
redriveOutcomeUnreportedSignalled = /* @__PURE__ */ new Map();
|
|
3555
|
+
/**
|
|
3556
|
+
* First `now()` a Class B outcome PATCH (reattach/settle/fail_permanent) was
|
|
3557
|
+
* observed to fail for this message (#1366's failure-window trip arm,
|
|
3558
|
+
* `boundRedriveOutcome`). Duration, not a tick count — bounded by the
|
|
3559
|
+
* existing `pausedMaxWaitMs` window (reusing the knob, not a new constant).
|
|
3560
|
+
* Cleared by `clearRedriveUnresolved` the instant the original PATCH
|
|
3561
|
+
* succeeds.
|
|
3562
|
+
*/
|
|
3563
|
+
redriveOutcomeFailingSince = /* @__PURE__ */ new Map();
|
|
3564
|
+
/**
|
|
3565
|
+
* "Already posted `redrive_outcome_abandoned` with `reported: false` for this
|
|
3566
|
+
* row" (#1366) — the bound tripped but the terminal `markFailed` fallback ALSO
|
|
3567
|
+
* failed (the route-level fault of G2), so every following tick re-attempts
|
|
3568
|
+
* the same terminal PATCH. Guards that quiet retry from re-signalling on
|
|
3569
|
+
* every tick. Cleared by `clearRedriveUnresolved`.
|
|
3570
|
+
*/
|
|
3571
|
+
redriveOutcomeAbandonedSignalled = /* @__PURE__ */ new Set();
|
|
3572
|
+
/**
|
|
3573
|
+
* "Already emitted `dispatch_not_started` for THIS (message, branch) streak"
|
|
3574
|
+
* (#1340). Valued by the branch currently firing, so a row that moves between
|
|
3575
|
+
* exits re-signals — the move IS the finding. Cleared only on a CONFIRMED
|
|
3576
|
+
* dispatch, never on the fence's decision to dispatch: `clearRedriveUnresolved`
|
|
3577
|
+
* runs on that decision (`resolveRedriveUnresolved`), so clearing there would
|
|
3578
|
+
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
3579
|
+
*/
|
|
3580
|
+
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
3581
|
+
/**
|
|
3582
|
+
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
3583
|
+
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
3584
|
+
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
3585
|
+
* read-back retries can never confirm the assigned id when the session's
|
|
3586
|
+
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
3587
|
+
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
3588
|
+
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
3589
|
+
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
3590
|
+
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
3591
|
+
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
3592
|
+
* after abandonment) starts a new streak rather than inheriting the old
|
|
3593
|
+
* session's count — same shape as `redrivePollFailures` above.
|
|
3594
|
+
*/
|
|
3595
|
+
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
3282
3596
|
/**
|
|
3283
3597
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
3284
3598
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -3384,6 +3698,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3384
3698
|
this.now = config.now ?? (() => Date.now());
|
|
3385
3699
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3386
3700
|
this.homeDir = config.homeDir ?? homedir2();
|
|
3701
|
+
this.maxActiveSessions = config.maxActiveSessions;
|
|
3387
3702
|
}
|
|
3388
3703
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
3389
3704
|
get opencodeBase() {
|
|
@@ -3463,10 +3778,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3463
3778
|
message: `Found ${total} pending message(s) across ${conversations.length} conversation(s) \u2014 draining`
|
|
3464
3779
|
});
|
|
3465
3780
|
}
|
|
3781
|
+
let cappedSkips = 0;
|
|
3466
3782
|
for (const conv of conversations) {
|
|
3467
3783
|
if (this.stopped) break;
|
|
3784
|
+
if (this.maxActiveSessions !== void 0) {
|
|
3785
|
+
const activeSessionIds = this.activeSessionIdsForCap();
|
|
3786
|
+
const resolvedSessionId = this.sessions.get(conv.id) ?? conv.opencode_session_id;
|
|
3787
|
+
const alreadyActive = resolvedSessionId != null && activeSessionIds.has(resolvedSessionId);
|
|
3788
|
+
if (activeSessionIds.size >= this.maxActiveSessions && !alreadyActive) {
|
|
3789
|
+
cappedSkips++;
|
|
3790
|
+
continue;
|
|
3791
|
+
}
|
|
3792
|
+
}
|
|
3468
3793
|
dispatched += await this.processConversation(conv);
|
|
3469
3794
|
}
|
|
3795
|
+
if (cappedSkips > 0) {
|
|
3796
|
+
this.log({
|
|
3797
|
+
level: "warn",
|
|
3798
|
+
message: `max-active-sessions cap (${this.maxActiveSessions}) reached \u2014 skipped ${cappedSkips} pending conversation(s) this tick`
|
|
3799
|
+
});
|
|
3800
|
+
}
|
|
3470
3801
|
await this.readoptProcessing();
|
|
3471
3802
|
} finally {
|
|
3472
3803
|
this.draining = false;
|
|
@@ -3485,6 +3816,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3485
3816
|
}
|
|
3486
3817
|
return false;
|
|
3487
3818
|
}
|
|
3819
|
+
/**
|
|
3820
|
+
* Session ids active *for the `--max-active-sessions` cap*: in-flight work AND
|
|
3821
|
+
* a live watcher loop. Unlike `hasInFlightWatchers()` / `protectedSessionIds()`,
|
|
3822
|
+
* a ZOMBIE watcher (in-flight but `loop === null`, left by a non-auth failure
|
|
3823
|
+
* inside `runWatcherLoop`) does not count here — under a cap it would
|
|
3824
|
+
* permanently consume a slot, whereas cleanup/idle-exit should still treat it
|
|
3825
|
+
* as protected. One call per drain iteration serves both the cap check
|
|
3826
|
+
* (`.size`) and the already-active exemption (`.has`).
|
|
3827
|
+
*/
|
|
3828
|
+
activeSessionIdsForCap() {
|
|
3829
|
+
const ids = /* @__PURE__ */ new Set();
|
|
3830
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
3831
|
+
if (watcher.inFlight.size > 0 && watcher.loop !== null) ids.add(sessionId);
|
|
3832
|
+
}
|
|
3833
|
+
return ids;
|
|
3834
|
+
}
|
|
3488
3835
|
/**
|
|
3489
3836
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
3490
3837
|
*
|
|
@@ -3603,7 +3950,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3603
3950
|
* @returns the count of messages NEWLY dispatched (not already in-flight).
|
|
3604
3951
|
*/
|
|
3605
3952
|
async processConversation(conv) {
|
|
3606
|
-
const { sessionId, refusedSessionId } = await this.ensureSession(conv);
|
|
3953
|
+
const { sessionId, refusedSessionId, created: sessionCreated } = await this.ensureSession(conv);
|
|
3607
3954
|
const messages = await this.getPendingMessages(conv.id);
|
|
3608
3955
|
let dispatched = 0;
|
|
3609
3956
|
let skippedAlreadyDispatched = 0;
|
|
@@ -3619,7 +3966,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3619
3966
|
continue;
|
|
3620
3967
|
}
|
|
3621
3968
|
if (message.opencode_message_id) {
|
|
3622
|
-
const outcome = await this.resolveRedrive(conv, sessionId, message,
|
|
3969
|
+
const outcome = await this.resolveRedrive(conv, sessionId, message, sessionCreated);
|
|
3970
|
+
if (outcome === "abandoned") {
|
|
3971
|
+
continue;
|
|
3972
|
+
}
|
|
3623
3973
|
if (outcome !== "dispatch") {
|
|
3624
3974
|
break;
|
|
3625
3975
|
}
|
|
@@ -3653,6 +4003,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3653
4003
|
conversation_id: conv.id,
|
|
3654
4004
|
message_id: message.id
|
|
3655
4005
|
});
|
|
4006
|
+
this.signalDispatchNotStarted(conv, message, "session_deleted_race");
|
|
3656
4007
|
break;
|
|
3657
4008
|
}
|
|
3658
4009
|
if (exists === null) {
|
|
@@ -3662,6 +4013,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3662
4013
|
conversation_id: conv.id,
|
|
3663
4014
|
message_id: message.id
|
|
3664
4015
|
});
|
|
4016
|
+
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
3665
4017
|
break;
|
|
3666
4018
|
}
|
|
3667
4019
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
@@ -3680,6 +4032,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3680
4032
|
conversation_id: conv.id,
|
|
3681
4033
|
message_id: message.id
|
|
3682
4034
|
});
|
|
4035
|
+
this.signalDispatchNotStarted(conv, message, "failure_unreported");
|
|
3683
4036
|
});
|
|
3684
4037
|
this.log({
|
|
3685
4038
|
level: "error",
|
|
@@ -3690,14 +4043,40 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3690
4043
|
break;
|
|
3691
4044
|
}
|
|
3692
4045
|
if (opencodeMessageId === null) {
|
|
4046
|
+
const streak = this.recordUnconfirmedDispatch(message.id, sessionId);
|
|
4047
|
+
if (streak < MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
4048
|
+
this.log({
|
|
4049
|
+
level: "warn",
|
|
4050
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next tick`,
|
|
4051
|
+
conversation_id: conv.id,
|
|
4052
|
+
message_id: message.id
|
|
4053
|
+
});
|
|
4054
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
4055
|
+
continue;
|
|
4056
|
+
}
|
|
4057
|
+
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4058
|
+
this.sessions.delete(conv.id);
|
|
4059
|
+
this.supersede(conv.id, sessionId);
|
|
4060
|
+
const errorMessage = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
3693
4061
|
this.log({
|
|
3694
|
-
level: "
|
|
3695
|
-
message:
|
|
4062
|
+
level: "error",
|
|
4063
|
+
message: errorMessage,
|
|
3696
4064
|
conversation_id: conv.id,
|
|
3697
4065
|
message_id: message.id
|
|
3698
4066
|
});
|
|
3699
|
-
|
|
4067
|
+
await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
|
|
4068
|
+
this.log({
|
|
4069
|
+
level: "warn",
|
|
4070
|
+
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
4071
|
+
conversation_id: conv.id,
|
|
4072
|
+
message_id: message.id
|
|
4073
|
+
});
|
|
4074
|
+
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
4075
|
+
});
|
|
4076
|
+
break;
|
|
3700
4077
|
}
|
|
4078
|
+
this.unconfirmedDispatchFailures.delete(message.id);
|
|
4079
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
3701
4080
|
this.dispatched.add(message.id);
|
|
3702
4081
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
3703
4082
|
dispatched += 1;
|
|
@@ -3718,21 +4097,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3718
4097
|
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
3719
4098
|
* hits the global `fetch` and would bypass the same override every other
|
|
3720
4099
|
* opencode poll in this file respects. Mirrors `readoptProcessing`'s own
|
|
3721
|
-
* snapshot fetch (`:3081-3111`).
|
|
3722
|
-
*
|
|
3723
|
-
*
|
|
4100
|
+
* snapshot fetch (`:3081-3111`).
|
|
4101
|
+
*
|
|
4102
|
+
* Returns `{ ok: true, messages }` on a readable snapshot, or
|
|
4103
|
+
* `{ ok: false, signature }` on failure — `signature` is a string that
|
|
4104
|
+
* repeats across attempts for the SAME underlying fault (used by the
|
|
4105
|
+
* consecutive-identical-failure bound, #1348), or `null` for a thrown
|
|
4106
|
+
* exception, which is NOT countable toward that bound (a network blip / an
|
|
4107
|
+
* opencode restart also throws identically every tick, and must keep
|
|
4108
|
+
* retrying unbounded rather than ever being treated as permanent).
|
|
3724
4109
|
*/
|
|
3725
4110
|
async pollSessionMessagesForRedrive(conv, message, sessionId) {
|
|
3726
4111
|
try {
|
|
3727
4112
|
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
3728
4113
|
if (!res.ok) {
|
|
4114
|
+
const rawBody = await res.text();
|
|
4115
|
+
const normalized = normalizeRedrivePollFailureBody(rawBody);
|
|
3729
4116
|
this.log({
|
|
3730
4117
|
level: "warn",
|
|
3731
|
-
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status} \u2014 treating as unreadable this tick`,
|
|
4118
|
+
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status}${normalized ? `: ${normalized}` : ""} \u2014 treating as unreadable this tick`,
|
|
3732
4119
|
conversation_id: conv.id,
|
|
3733
4120
|
message_id: message.id
|
|
3734
4121
|
});
|
|
3735
|
-
return
|
|
4122
|
+
return { ok: false, signature: `HTTP ${res.status}${normalized ? `: ${normalized}` : ""}` };
|
|
3736
4123
|
}
|
|
3737
4124
|
const body = await res.json();
|
|
3738
4125
|
if (!Array.isArray(body)) {
|
|
@@ -3742,9 +4129,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3742
4129
|
conversation_id: conv.id,
|
|
3743
4130
|
message_id: message.id
|
|
3744
4131
|
});
|
|
3745
|
-
return
|
|
4132
|
+
return { ok: false, signature: "non-array message body" };
|
|
3746
4133
|
}
|
|
3747
|
-
return body;
|
|
4134
|
+
return { ok: true, messages: body };
|
|
3748
4135
|
} catch (err) {
|
|
3749
4136
|
this.log({
|
|
3750
4137
|
level: "warn",
|
|
@@ -3752,7 +4139,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3752
4139
|
conversation_id: conv.id,
|
|
3753
4140
|
message_id: message.id
|
|
3754
4141
|
});
|
|
3755
|
-
return null;
|
|
4142
|
+
return { ok: false, signature: null };
|
|
3756
4143
|
}
|
|
3757
4144
|
}
|
|
3758
4145
|
/**
|
|
@@ -3764,24 +4151,51 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3764
4151
|
* without this fence the drain loop would re-`prompt_async` the SAME turn a
|
|
3765
4152
|
* second time against live GitHub state. Mirrors `readoptOne`'s job for the
|
|
3766
4153
|
* `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
|
|
3767
|
-
* needed here because `
|
|
3768
|
-
*
|
|
4154
|
+
* needed here because `sessionCreated` already handles the cases (a #553
|
|
4155
|
+
* abandoned session, a #190 vanished one) that path exists for.
|
|
3769
4156
|
*
|
|
3770
|
-
* Only `ChannelAuthError` propagates
|
|
3771
|
-
* `
|
|
4157
|
+
* Only `ChannelAuthError` propagates. A poll that fails identically
|
|
4158
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row reports the message
|
|
4159
|
+
* failed instead of retrying it (#1348) — SEPARATE from, not a replacement
|
|
4160
|
+
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
4161
|
+
* other failure resolves to `unresolved` and is retried whole on the next
|
|
4162
|
+
* ~2s drain tick.
|
|
3772
4163
|
*/
|
|
3773
|
-
async resolveRedrive(conv, sessionId, message,
|
|
4164
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
3774
4165
|
const ocId = message.opencode_message_id ?? null;
|
|
3775
|
-
if (
|
|
4166
|
+
if (sessionCreated) {
|
|
3776
4167
|
this.clearRedriveUnresolved(message.id);
|
|
3777
4168
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3778
4169
|
return "dispatch";
|
|
3779
4170
|
}
|
|
3780
|
-
const
|
|
3781
|
-
if (
|
|
4171
|
+
const polled = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
|
|
4172
|
+
if (!polled.ok) {
|
|
4173
|
+
const streak = this.recordRedrivePollFailure(message.id, sessionId, polled.signature);
|
|
4174
|
+
if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES && polled.signature !== null) {
|
|
4175
|
+
return this.failRedrivePollPermanent(conv, sessionId, message, polled.signature, streak);
|
|
4176
|
+
}
|
|
4177
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
4178
|
+
}
|
|
4179
|
+
this.redrivePollFailures.delete(message.id);
|
|
4180
|
+
const messages = polled.messages;
|
|
4181
|
+
if (messages.length === 0) {
|
|
3782
4182
|
return this.resolveRedriveUnresolved(conv, message);
|
|
3783
4183
|
}
|
|
3784
4184
|
const state = messageRunState(messages, ocId ?? "");
|
|
4185
|
+
if (state === "failed" && isAbortedTerminalReply(messages, ocId ?? "")) {
|
|
4186
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
4187
|
+
if (ongoing === false) {
|
|
4188
|
+
this.log({
|
|
4189
|
+
level: "info",
|
|
4190
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
4191
|
+
conversation_id: conv.id,
|
|
4192
|
+
message_id: message.id
|
|
4193
|
+
});
|
|
4194
|
+
this.clearRedriveUnresolved(message.id);
|
|
4195
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
4196
|
+
return "dispatch";
|
|
4197
|
+
}
|
|
4198
|
+
}
|
|
3785
4199
|
if (state === "done" || state === "failed") {
|
|
3786
4200
|
return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
|
|
3787
4201
|
}
|
|
@@ -3791,6 +4205,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3791
4205
|
return this.reattachRedrive(conv, sessionId, message, ocId);
|
|
3792
4206
|
}
|
|
3793
4207
|
if (ongoing === false) {
|
|
4208
|
+
if (state === "running" && isAmbiguousFinishPinnedRunning(messages, ocId ?? "")) {
|
|
4209
|
+
return this.settleRedrive(conv, sessionId, message, ocId, messages, "done");
|
|
4210
|
+
}
|
|
3794
4211
|
this.clearRedriveUnresolved(message.id);
|
|
3795
4212
|
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3796
4213
|
return "dispatch";
|
|
@@ -3825,13 +4242,23 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3825
4242
|
await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
|
|
3826
4243
|
} catch (err) {
|
|
3827
4244
|
if (err instanceof ChannelAuthError) throw err;
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
4245
|
+
if (err instanceof ChannelTerminalError) {
|
|
4246
|
+
this.log({
|
|
4247
|
+
level: "error",
|
|
4248
|
+
message: `Re-drive: the server definitively refused to restore message ${message.id.slice(0, 8)} to processing (terminal HTTP ${err.status} \u2014 the row is gone or the update was rejected); NOT reporting a re-attach`,
|
|
4249
|
+
conversation_id: conv.id,
|
|
4250
|
+
message_id: message.id
|
|
4251
|
+
});
|
|
4252
|
+
} else {
|
|
4253
|
+
this.log({
|
|
4254
|
+
level: "warn",
|
|
4255
|
+
message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4256
|
+
conversation_id: conv.id,
|
|
4257
|
+
message_id: message.id
|
|
4258
|
+
});
|
|
4259
|
+
}
|
|
4260
|
+
const bound = await this.boundRedriveOutcome(conv, message, "reattach");
|
|
4261
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
3835
4262
|
}
|
|
3836
4263
|
this.clearRedriveUnresolved(message.id);
|
|
3837
4264
|
this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
|
|
@@ -3855,7 +4282,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3855
4282
|
* errored) while nobody was watching — deliver/report it instead of re-running.
|
|
3856
4283
|
* Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
|
|
3857
4284
|
* (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
|
|
3858
|
-
* drain, same as any other non-auth failure).
|
|
4285
|
+
* drain, same as any other non-auth failure). The restart-abort carve-out that
|
|
4286
|
+
* keeps the two in step for `failed` lives in the caller (`resolveRedrive`, #1310),
|
|
4287
|
+
* so a row reaching this `failed` branch is a GENUINE failure.
|
|
3859
4288
|
*/
|
|
3860
4289
|
async settleRedrive(conv, sessionId, message, ocId, messages, state) {
|
|
3861
4290
|
try {
|
|
@@ -3889,19 +4318,62 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3889
4318
|
conversation_id: conv.id,
|
|
3890
4319
|
message_id: message.id
|
|
3891
4320
|
});
|
|
3892
|
-
|
|
4321
|
+
const bound = await this.boundRedriveOutcome(conv, message, "settle");
|
|
4322
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
3893
4323
|
}
|
|
3894
4324
|
this.clearRedriveUnresolved(message.id);
|
|
3895
4325
|
void this.postSignal(conv.id, message.id, "redrive_settled");
|
|
3896
4326
|
return "settled";
|
|
3897
4327
|
}
|
|
4328
|
+
/**
|
|
4329
|
+
* The permanent-failure outcome (#1348): the fence's own poll of this session
|
|
4330
|
+
* failed with the SAME opencode-answered signature
|
|
4331
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row — a transient blip
|
|
4332
|
+
* would have varied or eventually cleared (see `pollSessionMessagesForRedrive`
|
|
4333
|
+
* and `recordRedrivePollFailure`), so this is a durable fault (e.g. #1345's
|
|
4334
|
+
* corrupted opencode session) rather than something worth retrying forever.
|
|
4335
|
+
* Mirrors `settleRedrive`'s error discipline: no `usage`/`failure` args to
|
|
4336
|
+
* `markFailed` (no opencode snapshot to extract them from — this poll never
|
|
4337
|
+
* got a readable one).
|
|
4338
|
+
*/
|
|
4339
|
+
async failRedrivePollPermanent(conv, sessionId, message, signature, streak) {
|
|
4340
|
+
this.log({
|
|
4341
|
+
level: "error",
|
|
4342
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) failed to poll with the identical signature "${signature}" ${streak} times in a row \u2014 reporting the message failed instead of retrying forever`,
|
|
4343
|
+
conversation_id: conv.id,
|
|
4344
|
+
message_id: message.id
|
|
4345
|
+
});
|
|
4346
|
+
try {
|
|
4347
|
+
await this.markFailed(
|
|
4348
|
+
conv.id,
|
|
4349
|
+
message.id,
|
|
4350
|
+
sessionId,
|
|
4351
|
+
`The runner could not read this conversation's state from OpenCode (${signature}). The same failure repeated ${streak} times in a row, so the message was not retried further.`
|
|
4352
|
+
);
|
|
4353
|
+
} catch (err) {
|
|
4354
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4355
|
+
this.log({
|
|
4356
|
+
level: "warn",
|
|
4357
|
+
message: `Re-drive: failed to report message ${message.id.slice(0, 8)} permanently failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4358
|
+
conversation_id: conv.id,
|
|
4359
|
+
message_id: message.id
|
|
4360
|
+
});
|
|
4361
|
+
const bound = await this.boundRedriveOutcome(conv, message, "fail_permanent");
|
|
4362
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
4363
|
+
}
|
|
4364
|
+
this.clearRedriveUnresolved(message.id);
|
|
4365
|
+
void this.postSignal(conv.id, message.id, "redrive_poll_failed");
|
|
4366
|
+
return "settled";
|
|
4367
|
+
}
|
|
3898
4368
|
/**
|
|
3899
4369
|
* The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
|
|
3900
4370
|
* observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
|
|
3901
|
-
* A `pending` row is
|
|
3902
|
-
*
|
|
3903
|
-
*
|
|
3904
|
-
*
|
|
4371
|
+
* A `pending` row is swept by the server's own `PENDING_MAX_AGE_MS` (24h,
|
|
4372
|
+
* #1368) cron arm, but that is a day-scale backstop — this local bound acts
|
|
4373
|
+
* in minutes so the row (and the conversation it starves, per the ordering
|
|
4374
|
+
* invariant below) isn't left stranded for that long. Bound to the existing
|
|
4375
|
+
* `pausedMaxWaitMs` window (reusing the knob, not a new constant); takes
|
|
4376
|
+
* `dispatch` once elapsed.
|
|
3905
4377
|
*/
|
|
3906
4378
|
resolveRedriveUnresolved(conv, message) {
|
|
3907
4379
|
const now = this.now();
|
|
@@ -3920,10 +4392,153 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3920
4392
|
}
|
|
3921
4393
|
return "unresolved";
|
|
3922
4394
|
}
|
|
3923
|
-
/** Clear
|
|
4395
|
+
/** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
|
|
3924
4396
|
clearRedriveUnresolved(messageId) {
|
|
3925
4397
|
this.redriveUnresolvedSince.delete(messageId);
|
|
3926
4398
|
this.redriveUnresolvedSignalled.delete(messageId);
|
|
4399
|
+
this.redrivePollFailures.delete(messageId);
|
|
4400
|
+
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4401
|
+
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4402
|
+
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4403
|
+
}
|
|
4404
|
+
/**
|
|
4405
|
+
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
4406
|
+
* most once per (message, branch) streak — a wedged row is re-tried every tick,
|
|
4407
|
+
* and the per-tick count is already carried by the co-occurring
|
|
4408
|
+
* `redrive_unresolved`/`redrive_redispatched` signals.
|
|
4409
|
+
*/
|
|
4410
|
+
signalDispatchNotStarted(conv, message, branch) {
|
|
4411
|
+
if (this.dispatchNotStartedSignalled.get(message.id) === branch) return;
|
|
4412
|
+
this.dispatchNotStartedSignalled.set(message.id, branch);
|
|
4413
|
+
void this.postSignal(conv.id, message.id, "dispatch_not_started", { branch });
|
|
4414
|
+
}
|
|
4415
|
+
/**
|
|
4416
|
+
* Class B (#1340): the runner DECIDED an outcome (reattach/settle/fail_permanent)
|
|
4417
|
+
* but its own PATCH to record it failed. Fires at most once per (message,
|
|
4418
|
+
* outcome) streak, and only while `boundRedriveOutcome` has not yet tripped —
|
|
4419
|
+
* once it trips, `redrive_outcome_abandoned` takes over reporting for the row
|
|
4420
|
+
* (#1366).
|
|
4421
|
+
*/
|
|
4422
|
+
signalRedriveOutcomeUnreported(conv, message, outcome) {
|
|
4423
|
+
if (this.redriveOutcomeUnreportedSignalled.get(message.id) === outcome) return;
|
|
4424
|
+
this.redriveOutcomeUnreportedSignalled.set(message.id, outcome);
|
|
4425
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_unreported", {
|
|
4426
|
+
attempted_outcome: outcome
|
|
4427
|
+
});
|
|
4428
|
+
}
|
|
4429
|
+
/**
|
|
4430
|
+
* The runner-authored, honest error text for the terminal fallback a tripped
|
|
4431
|
+
* `boundRedriveOutcome` sends. Distinguishable per outcome and truthful about
|
|
4432
|
+
* what actually happened — the `settle`/done case must say the turn finished
|
|
4433
|
+
* but its result could not be recorded, never that the runner stopped
|
|
4434
|
+
* responding (that would be a lie for this shape, see #1366's "why this ships").
|
|
4435
|
+
*/
|
|
4436
|
+
static REDRIVE_ABANDON_ERROR = {
|
|
4437
|
+
reattach: "your runner could not record that this message had started, so it was given up on",
|
|
4438
|
+
settle: "your runner finished this message but could not record the result, so the reply could not be delivered",
|
|
4439
|
+
fail_permanent: "the runner could not read this conversation's state from OpenCode, and could not record that failure either, so the message was given up on"
|
|
4440
|
+
};
|
|
4441
|
+
/**
|
|
4442
|
+
* Bound for Class B (#1340, #1366): the runner DECIDED an outcome but its own
|
|
4443
|
+
* PATCH to record it failed. Two independent trip arms (either sufficient):
|
|
4444
|
+
* (1) this failure streak has lasted `pausedMaxWaitMs` — DURATION, not a tick
|
|
4445
|
+
* count, reusing the knob `resolveRedriveUnresolved` already established; (2)
|
|
4446
|
+
* the turn's `processing_started_at` age has crossed
|
|
4447
|
+
* `ABSOLUTE_MAX_PROCESSING_MS` — durable and restart-surviving, since arm (1)'s
|
|
4448
|
+
* in-memory streak resets on a scale-to-zero restart.
|
|
4449
|
+
*
|
|
4450
|
+
* INVARIANT — a tripped bound never suppresses the original outcome attempt;
|
|
4451
|
+
* it only adds a fallback after that attempt has failed again. This is only
|
|
4452
|
+
* ever reached from inside the catch of the ORIGINAL outcome PATCH, which is
|
|
4453
|
+
* attempted first on every tick whether or not this bound tripped before —
|
|
4454
|
+
* there is no give-up latch that would short-circuit it. That is what lets a
|
|
4455
|
+
* route-level fault that heals later still deliver the turn's real
|
|
4456
|
+
* `done`/`failed` payload: once the original PATCH succeeds again, this
|
|
4457
|
+
* helper is never entered and the row settles with its real result.
|
|
4458
|
+
*/
|
|
4459
|
+
async boundRedriveOutcome(conv, message, outcome) {
|
|
4460
|
+
const now = this.now();
|
|
4461
|
+
const since = this.redriveOutcomeFailingSince.get(message.id);
|
|
4462
|
+
if (since === void 0) this.redriveOutcomeFailingSince.set(message.id, now);
|
|
4463
|
+
const durationTripped = now - (since ?? now) >= this.pausedMaxWaitMs;
|
|
4464
|
+
const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
|
|
4465
|
+
const absoluteAgeTripped = !Number.isNaN(parsed) && now - parsed >= ABSOLUTE_MAX_PROCESSING_MS;
|
|
4466
|
+
if (!durationTripped && !absoluteAgeTripped) {
|
|
4467
|
+
this.signalRedriveOutcomeUnreported(conv, message, outcome);
|
|
4468
|
+
return "retry";
|
|
4469
|
+
}
|
|
4470
|
+
const arm = durationTripped ? "failure_window" : "absolute_age";
|
|
4471
|
+
try {
|
|
4472
|
+
await this.markFailed(
|
|
4473
|
+
conv.id,
|
|
4474
|
+
message.id,
|
|
4475
|
+
void 0,
|
|
4476
|
+
_ChannelDriver.REDRIVE_ABANDON_ERROR[outcome]
|
|
4477
|
+
);
|
|
4478
|
+
} catch (err) {
|
|
4479
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4480
|
+
this.log({
|
|
4481
|
+
level: "warn",
|
|
4482
|
+
message: `Re-drive bound: fallback markFailed for message ${message.id.slice(0, 8)} also failed (arm ${arm}, will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4483
|
+
conversation_id: conv.id,
|
|
4484
|
+
message_id: message.id
|
|
4485
|
+
});
|
|
4486
|
+
if (!this.redriveOutcomeAbandonedSignalled.has(message.id)) {
|
|
4487
|
+
this.redriveOutcomeAbandonedSignalled.add(message.id);
|
|
4488
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_abandoned", {
|
|
4489
|
+
attempted_outcome: outcome,
|
|
4490
|
+
reported: false,
|
|
4491
|
+
arm
|
|
4492
|
+
});
|
|
4493
|
+
}
|
|
4494
|
+
return "retry";
|
|
4495
|
+
}
|
|
4496
|
+
this.clearRedriveUnresolved(message.id);
|
|
4497
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_abandoned", {
|
|
4498
|
+
attempted_outcome: outcome,
|
|
4499
|
+
reported: true,
|
|
4500
|
+
arm
|
|
4501
|
+
});
|
|
4502
|
+
return "abandoned";
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4505
|
+
* Record one poll outcome toward the re-drive fence's consecutive-identical-
|
|
4506
|
+
* failure streak (#1348) and return the resulting count. `signature === null`
|
|
4507
|
+
* (a thrown exception, H1) always clears the streak and returns `0` — it is
|
|
4508
|
+
* never countable. Otherwise the streak continues only when BOTH the session
|
|
4509
|
+
* and the signature match the previous failure; anything else (a different
|
|
4510
|
+
* session, or the same session failing a DIFFERENT way) starts a fresh streak
|
|
4511
|
+
* at `1`.
|
|
4512
|
+
*/
|
|
4513
|
+
recordRedrivePollFailure(messageId, sessionId, signature) {
|
|
4514
|
+
if (signature === null) {
|
|
4515
|
+
this.redrivePollFailures.delete(messageId);
|
|
4516
|
+
return 0;
|
|
4517
|
+
}
|
|
4518
|
+
const existing = this.redrivePollFailures.get(messageId);
|
|
4519
|
+
if (existing && existing.sessionId === sessionId && existing.signature === signature) {
|
|
4520
|
+
existing.count += 1;
|
|
4521
|
+
return existing.count;
|
|
4522
|
+
}
|
|
4523
|
+
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
4524
|
+
return 1;
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
4528
|
+
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
4529
|
+
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
4530
|
+
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
4531
|
+
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
4532
|
+
* count, since a new session is a genuinely different attempt.
|
|
4533
|
+
*/
|
|
4534
|
+
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
4535
|
+
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
4536
|
+
if (existing && existing.sessionId === sessionId) {
|
|
4537
|
+
existing.count += 1;
|
|
4538
|
+
return existing.count;
|
|
4539
|
+
}
|
|
4540
|
+
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
4541
|
+
return 1;
|
|
3927
4542
|
}
|
|
3928
4543
|
/**
|
|
3929
4544
|
* Record that `sessionId` is no longer a valid binding for `conversationId`
|
|
@@ -3949,6 +4564,13 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3949
4564
|
* `refusedSessionId` is set when the #553 guard fired — i.e. the persisted
|
|
3950
4565
|
* binding was an id this runner had abandoned, so a resurrection genuinely
|
|
3951
4566
|
* happened and a fresh session was bound instead. The caller reports it.
|
|
4567
|
+
*
|
|
4568
|
+
* `created` says the returned session was made JUST NOW, so it provably holds
|
|
4569
|
+
* no prior turn. The re-drive fence needs that as CONTRARY evidence ("nothing
|
|
4570
|
+
* to reconcile against") — distinct from the ambiguous "I polled and saw an
|
|
4571
|
+
* empty transcript", which stays a deferral. Keep it separate from
|
|
4572
|
+
* `refusedSessionId`: only the latter means a #553 resurrection happened, and
|
|
4573
|
+
* only it may drive the `session_superseded` signal.
|
|
3952
4574
|
*/
|
|
3953
4575
|
async ensureSession(conv) {
|
|
3954
4576
|
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
@@ -3959,7 +4581,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3959
4581
|
conversation_id: conv.id
|
|
3960
4582
|
});
|
|
3961
4583
|
this.sessions.delete(conv.id);
|
|
3962
|
-
return {
|
|
4584
|
+
return {
|
|
4585
|
+
sessionId: await this.createAndBindSession(conv.id),
|
|
4586
|
+
refusedSessionId: bound,
|
|
4587
|
+
created: true
|
|
4588
|
+
};
|
|
3963
4589
|
}
|
|
3964
4590
|
if (bound) {
|
|
3965
4591
|
const exists = await sessionExists(this.port, bound);
|
|
@@ -3970,12 +4596,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3970
4596
|
conversation_id: conv.id
|
|
3971
4597
|
});
|
|
3972
4598
|
this.sessions.delete(conv.id);
|
|
3973
|
-
return { sessionId: await this.createAndBindSession(conv.id) };
|
|
4599
|
+
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
3974
4600
|
}
|
|
3975
4601
|
this.sessions.set(conv.id, bound);
|
|
3976
|
-
return { sessionId: bound };
|
|
4602
|
+
return { sessionId: bound, created: false };
|
|
3977
4603
|
}
|
|
3978
|
-
return { sessionId: await this.createAndBindSession(conv.id) };
|
|
4604
|
+
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
3979
4605
|
}
|
|
3980
4606
|
/**
|
|
3981
4607
|
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
@@ -4193,7 +4819,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4193
4819
|
deliveryDeadlineAnchored: false,
|
|
4194
4820
|
b2PinnedSinceMs: 0,
|
|
4195
4821
|
b2LastDescendantCheckMs: 0,
|
|
4196
|
-
b2AbandonedSignalled: false
|
|
4822
|
+
b2AbandonedSignalled: false,
|
|
4823
|
+
ambiguousPinnedSinceMs: 0,
|
|
4824
|
+
ambiguousResolved: false
|
|
4197
4825
|
});
|
|
4198
4826
|
}
|
|
4199
4827
|
/**
|
|
@@ -4272,7 +4900,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4272
4900
|
deliveryDeadlineAnchored: false,
|
|
4273
4901
|
b2PinnedSinceMs: 0,
|
|
4274
4902
|
b2LastDescendantCheckMs: 0,
|
|
4275
|
-
b2AbandonedSignalled: false
|
|
4903
|
+
b2AbandonedSignalled: false,
|
|
4904
|
+
ambiguousPinnedSinceMs: 0,
|
|
4905
|
+
ambiguousResolved: false
|
|
4276
4906
|
});
|
|
4277
4907
|
}
|
|
4278
4908
|
/**
|
|
@@ -4404,9 +5034,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4404
5034
|
const awaitingHuman = observedOpen || latchedPaused;
|
|
4405
5035
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
4406
5036
|
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
4407
|
-
let claimed;
|
|
4408
5037
|
try {
|
|
4409
|
-
|
|
5038
|
+
await this.markProcessing(
|
|
4410
5039
|
conv.id,
|
|
4411
5040
|
inFlight.evidentMessageId,
|
|
4412
5041
|
sessionId,
|
|
@@ -4415,23 +5044,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4415
5044
|
);
|
|
4416
5045
|
} catch (err) {
|
|
4417
5046
|
if (err instanceof ChannelAuthError) throw err;
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
5047
|
+
if (err instanceof ChannelTerminalError) {
|
|
5048
|
+
this.log({
|
|
5049
|
+
level: "error",
|
|
5050
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (terminal HTTP ${err.status}) \u2014 the server definitively refused the swap`,
|
|
5051
|
+
conversation_id: conv.id,
|
|
5052
|
+
message_id: inFlight.evidentMessageId
|
|
5053
|
+
});
|
|
5054
|
+
} else {
|
|
5055
|
+
this.log({
|
|
5056
|
+
level: "warn",
|
|
5057
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
5058
|
+
conversation_id: conv.id,
|
|
5059
|
+
message_id: inFlight.evidentMessageId
|
|
5060
|
+
});
|
|
5061
|
+
return;
|
|
5062
|
+
}
|
|
4425
5063
|
}
|
|
4426
5064
|
inFlight.started = true;
|
|
4427
|
-
if (!claimed) {
|
|
4428
|
-
this.log({
|
|
4429
|
-
level: "debug",
|
|
4430
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
4431
|
-
conversation_id: conv.id,
|
|
4432
|
-
message_id: inFlight.evidentMessageId
|
|
4433
|
-
});
|
|
4434
|
-
}
|
|
4435
5065
|
}
|
|
4436
5066
|
if (state === "done") {
|
|
4437
5067
|
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
@@ -4540,6 +5170,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4540
5170
|
}
|
|
4541
5171
|
}
|
|
4542
5172
|
}
|
|
5173
|
+
const ambiguousPinnedNow = activelyRunning && isAmbiguousFinishPinnedRunning(messages, inFlight.opencodeMessageId);
|
|
5174
|
+
if (!ambiguousPinnedNow) {
|
|
5175
|
+
if (snapshotReadable) {
|
|
5176
|
+
inFlight.ambiguousPinnedSinceMs = 0;
|
|
5177
|
+
inFlight.ambiguousResolved = false;
|
|
5178
|
+
}
|
|
5179
|
+
} else {
|
|
5180
|
+
if (inFlight.ambiguousResolved) {
|
|
5181
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5182
|
+
return;
|
|
5183
|
+
}
|
|
5184
|
+
if (inFlight.ambiguousPinnedSinceMs === 0) {
|
|
5185
|
+
inFlight.ambiguousPinnedSinceMs = this.now();
|
|
5186
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
5187
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
5188
|
+
this.log({
|
|
5189
|
+
level: "warn",
|
|
5190
|
+
message: `Message ${id.slice(0, 8)} pinned running by an unrecognised finish ("${finish ?? "(absent)"}") \u2014 corroborating against opencode's session status before settling (issue #1493)`,
|
|
5191
|
+
conversation_id: conv.id,
|
|
5192
|
+
message_id: id
|
|
5193
|
+
});
|
|
5194
|
+
}
|
|
5195
|
+
const pinnedForMs = this.now() - inFlight.ambiguousPinnedSinceMs;
|
|
5196
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
5197
|
+
if (isAmbiguousFinishResolved({
|
|
5198
|
+
pinnedForMs,
|
|
5199
|
+
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
5200
|
+
sessionOngoing: ongoing
|
|
5201
|
+
})) {
|
|
5202
|
+
inFlight.ambiguousResolved = true;
|
|
5203
|
+
this.log({
|
|
5204
|
+
level: "warn",
|
|
5205
|
+
message: `Message ${id.slice(0, 8)} ambiguous-finish-pinned for ${Math.round(pinnedForMs / 1e3)}s \u2014 resolved (${ongoing === false ? "session confirmed not-ongoing" : "pin exceeded the no-hang cap"}) \u2014 settling done`,
|
|
5206
|
+
conversation_id: conv.id,
|
|
5207
|
+
message_id: id
|
|
5208
|
+
});
|
|
5209
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
5210
|
+
watched_for_ms: pinnedForMs
|
|
5211
|
+
});
|
|
5212
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
5213
|
+
return;
|
|
5214
|
+
}
|
|
5215
|
+
}
|
|
4543
5216
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
4544
5217
|
this.log({
|
|
4545
5218
|
level: "warn",
|
|
@@ -4771,7 +5444,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4771
5444
|
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
4772
5445
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
4773
5446
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
4774
|
-
* errored turn is reported failed on restart, NOT re-dispatched
|
|
5447
|
+
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
5448
|
+
* EXCEPT a restart-ABORTED turn under a not-ongoing session,
|
|
5449
|
+
* which is a restart orphan wearing a terminal error and is
|
|
5450
|
+
* re-dispatched instead (issue #1310, see the branch below);
|
|
4775
5451
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
4776
5452
|
* tracking the stored id so the reply correlates by it;
|
|
4777
5453
|
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
@@ -4791,51 +5467,19 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4791
5467
|
const ocId = row.opencode_message_id;
|
|
4792
5468
|
const state = messageRunState(messages, ocId ?? "");
|
|
4793
5469
|
if (state === "done") {
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
message_id: row.id
|
|
4800
|
-
});
|
|
4801
|
-
return;
|
|
4802
|
-
}
|
|
5470
|
+
await this.deliverReadoptedDone(sessionId, row, messages, ocId);
|
|
5471
|
+
return;
|
|
5472
|
+
}
|
|
5473
|
+
const restartAborted = state === "failed" && sessionOngoing === false && isAbortedTerminalReply(messages, ocId ?? "");
|
|
5474
|
+
if (restartAborted) {
|
|
4803
5475
|
this.log({
|
|
4804
5476
|
level: "info",
|
|
4805
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
5477
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
4806
5478
|
conversation_id: row.conversation_id,
|
|
4807
5479
|
message_id: row.id
|
|
4808
5480
|
});
|
|
4809
|
-
try {
|
|
4810
|
-
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
4811
|
-
const usage = messageUsage(messages, ocId ?? "");
|
|
4812
|
-
await this.markDone(row.conversation_id, row.id, sessionId, ocId, title, usage);
|
|
4813
|
-
} catch (err) {
|
|
4814
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
4815
|
-
if (err instanceof ChannelTerminalError) {
|
|
4816
|
-
this.doneUndeliverable.add(row.id);
|
|
4817
|
-
this.log({
|
|
4818
|
-
level: "warn",
|
|
4819
|
-
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
4820
|
-
conversation_id: row.conversation_id,
|
|
4821
|
-
message_id: row.id
|
|
4822
|
-
});
|
|
4823
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
4824
|
-
return;
|
|
4825
|
-
}
|
|
4826
|
-
this.log({
|
|
4827
|
-
level: "warn",
|
|
4828
|
-
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4829
|
-
conversation_id: row.conversation_id,
|
|
4830
|
-
message_id: row.id
|
|
4831
|
-
});
|
|
4832
|
-
return;
|
|
4833
|
-
}
|
|
4834
|
-
this.dontRedispatch.delete(row.id);
|
|
4835
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
4836
|
-
return;
|
|
4837
5481
|
}
|
|
4838
|
-
if (state === "failed") {
|
|
5482
|
+
if (state === "failed" && !restartAborted) {
|
|
4839
5483
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
4840
5484
|
const usage = messageUsage(messages, ocId ?? "");
|
|
4841
5485
|
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
@@ -4888,6 +5532,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4888
5532
|
const ongoing = sessionOngoing;
|
|
4889
5533
|
statusReadableOngoing = ongoing;
|
|
4890
5534
|
if (ongoing === false) {
|
|
5535
|
+
if (isAmbiguousFinishPinnedRunning(messages, ocId ?? "")) {
|
|
5536
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
5537
|
+
this.log({
|
|
5538
|
+
level: "info",
|
|
5539
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} has an ambiguous finish ("${finish ?? "(absent)"}") but session ${sessionId.slice(0, 8)} is confirmed not-ongoing per GET /session/status \u2014 delivering the existing reply instead of re-dispatching`,
|
|
5540
|
+
conversation_id: row.conversation_id,
|
|
5541
|
+
message_id: row.id
|
|
5542
|
+
});
|
|
5543
|
+
await this.deliverReadoptedDone(sessionId, row, messages, ocId);
|
|
5544
|
+
return;
|
|
5545
|
+
}
|
|
4891
5546
|
this.log({
|
|
4892
5547
|
level: "info",
|
|
4893
5548
|
message: `Re-adopt: message ${row.id.slice(0, 8)} running/${shape} but session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status (absent/idle) \u2014 re-dispatching from scratch (status-gated recovery)`,
|
|
@@ -4964,6 +5619,63 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4964
5619
|
}
|
|
4965
5620
|
await this.forceReadoptRun(sessionId, row);
|
|
4966
5621
|
}
|
|
5622
|
+
/**
|
|
5623
|
+
* Deliver a `processing` row whose correlated reply already completed while
|
|
5624
|
+
* nobody was watching (ADR-0046) — the `readoptOne` `state === 'done'` body,
|
|
5625
|
+
* extracted (#1493 Task 2.4) so the ambiguous-finish guard above can call the
|
|
5626
|
+
* SAME delivery instead of duplicating it.
|
|
5627
|
+
*
|
|
5628
|
+
* EVEN IF the row was previously parked in `dontRedispatch` (a give-up stops
|
|
5629
|
+
* re-dispatch, not delivery — Bugbot #202). Guarded EXACTLY like the watcher's
|
|
5630
|
+
* `settleMessageDone`: auth re-throws; terminal → park in `doneUndeliverable` +
|
|
5631
|
+
* leave for cron; transient → log + leave for the next drain (the still-
|
|
5632
|
+
* `processing` row is re-read and retried). markDone is idempotent server-side
|
|
5633
|
+
* (status-gated), so a repeat can never double-post.
|
|
5634
|
+
*/
|
|
5635
|
+
async deliverReadoptedDone(sessionId, row, messages, ocId) {
|
|
5636
|
+
if (this.doneUndeliverable.has(row.id)) {
|
|
5637
|
+
this.log({
|
|
5638
|
+
level: "debug",
|
|
5639
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} markDone is terminally undeliverable \u2014 left to the cron; skipping until it leaves processing`,
|
|
5640
|
+
conversation_id: row.conversation_id,
|
|
5641
|
+
message_id: row.id
|
|
5642
|
+
});
|
|
5643
|
+
return;
|
|
5644
|
+
}
|
|
5645
|
+
this.log({
|
|
5646
|
+
level: "info",
|
|
5647
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} completed while unwatched \u2014 marking done`,
|
|
5648
|
+
conversation_id: row.conversation_id,
|
|
5649
|
+
message_id: row.id
|
|
5650
|
+
});
|
|
5651
|
+
try {
|
|
5652
|
+
const title = await this.resolveSessionTitle(sessionId, row.conversation_id);
|
|
5653
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
5654
|
+
await this.markDone(row.conversation_id, row.id, sessionId, ocId, title, usage);
|
|
5655
|
+
} catch (err) {
|
|
5656
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
5657
|
+
if (err instanceof ChannelTerminalError) {
|
|
5658
|
+
this.doneUndeliverable.add(row.id);
|
|
5659
|
+
this.log({
|
|
5660
|
+
level: "warn",
|
|
5661
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 parking until it leaves processing; leaving for the cron safety net: ${err.message}`,
|
|
5662
|
+
conversation_id: row.conversation_id,
|
|
5663
|
+
message_id: row.id
|
|
5664
|
+
});
|
|
5665
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_undeliverable");
|
|
5666
|
+
return;
|
|
5667
|
+
}
|
|
5668
|
+
this.log({
|
|
5669
|
+
level: "warn",
|
|
5670
|
+
message: `Re-adopt: failed to mark message ${row.id.slice(0, 8)} done (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
5671
|
+
conversation_id: row.conversation_id,
|
|
5672
|
+
message_id: row.id
|
|
5673
|
+
});
|
|
5674
|
+
return;
|
|
5675
|
+
}
|
|
5676
|
+
this.dontRedispatch.delete(row.id);
|
|
5677
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
5678
|
+
}
|
|
4967
5679
|
/**
|
|
4968
5680
|
* Re-dispatch an orphaned (`unknown`/null-id) `processing` row (ADR-0046 §2).
|
|
4969
5681
|
*
|
|
@@ -5049,15 +5761,39 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5049
5761
|
}
|
|
5050
5762
|
if (ocId === null) {
|
|
5051
5763
|
this.awaitingReadopt.delete(row.id);
|
|
5764
|
+
const streak = this.recordUnconfirmedDispatch(row.id, sessionId);
|
|
5765
|
+
if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
5766
|
+
this.unconfirmedDispatchFailures.delete(row.id);
|
|
5767
|
+
this.sessions.delete(readoptConv.id);
|
|
5768
|
+
this.supersede(readoptConv.id, sessionId);
|
|
5769
|
+
const errorMessage = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
5770
|
+
this.log({
|
|
5771
|
+
level: "error",
|
|
5772
|
+
message: errorMessage,
|
|
5773
|
+
conversation_id: row.conversation_id,
|
|
5774
|
+
message_id: row.id
|
|
5775
|
+
});
|
|
5776
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage).catch((markErr) => {
|
|
5777
|
+
this.log({
|
|
5778
|
+
level: "warn",
|
|
5779
|
+
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
5780
|
+
conversation_id: row.conversation_id,
|
|
5781
|
+
message_id: row.id
|
|
5782
|
+
});
|
|
5783
|
+
});
|
|
5784
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
5785
|
+
return;
|
|
5786
|
+
}
|
|
5052
5787
|
this.log({
|
|
5053
5788
|
level: "warn",
|
|
5054
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
5789
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next drain`,
|
|
5055
5790
|
conversation_id: row.conversation_id,
|
|
5056
5791
|
message_id: row.id
|
|
5057
5792
|
});
|
|
5058
5793
|
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
5059
5794
|
return;
|
|
5060
5795
|
}
|
|
5796
|
+
this.unconfirmedDispatchFailures.delete(row.id);
|
|
5061
5797
|
this.registerReadopted(readoptConv, sessionId, readoptMessage, ocId, this.processedAtMs(row));
|
|
5062
5798
|
this.dispatched.add(row.id);
|
|
5063
5799
|
this.readopted.add(row.id);
|
|
@@ -5569,17 +6305,25 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5569
6305
|
* the aborted-in-flight production bug after a restart.
|
|
5570
6306
|
* - `b2` — a COMPLETED reply pinned running only by `finish === "tool-calls"`
|
|
5571
6307
|
* (the sub-agent preamble — #253's shape).
|
|
5572
|
-
* - `
|
|
5573
|
-
*
|
|
5574
|
-
*
|
|
5575
|
-
* `
|
|
6308
|
+
* - `ambiguous` — a COMPLETED, non-errored reply whose `finish` is neither
|
|
6309
|
+
* "tool-calls" nor "stop" (issue #1493, class 4 — see
|
|
6310
|
+
* `isAmbiguousFinishPinnedRunning`/Task 2.4).
|
|
6311
|
+
* - `other` — any other shape (defensive; a running row is normally b1, b2 or
|
|
6312
|
+
* ambiguous).
|
|
6313
|
+
* Reads `info.time.completed` / `info.finish` / `info.error` (tolerating the
|
|
6314
|
+
* legacy top-level shape) directly rather than re-importing the module-private
|
|
6315
|
+
* `completedOf`/`finishOf`/`errorOf` — this is a display label only, not a
|
|
6316
|
+
* correctness predicate (that is `isAmbiguousFinishPinnedRunning`'s job).
|
|
5576
6317
|
*/
|
|
5577
6318
|
replyCompletionShape(reply) {
|
|
5578
6319
|
if (!reply) return "other";
|
|
5579
6320
|
const completed = reply.info?.time?.completed ?? reply.time?.completed;
|
|
5580
6321
|
if (completed == null) return "b1";
|
|
5581
6322
|
const finish = reply.info?.finish ?? reply.finish;
|
|
5582
|
-
|
|
6323
|
+
if (finish === "tool-calls") return "b2";
|
|
6324
|
+
const error2 = reply.info?.error ?? reply.error;
|
|
6325
|
+
if (finish !== "stop" && error2 == null) return "ambiguous";
|
|
6326
|
+
return "other";
|
|
5583
6327
|
}
|
|
5584
6328
|
/**
|
|
5585
6329
|
* Attribute a surfaced interaction to the in-flight message it paused on (M-1).
|
|
@@ -5722,18 +6466,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5722
6466
|
* opencode_session_id}` → `notifyMessageStarted` (hourglass→runner swap +
|
|
5723
6467
|
* deep-linked "View in Evident" notice).
|
|
5724
6468
|
*
|
|
5725
|
-
*
|
|
5726
|
-
* -
|
|
5727
|
-
*
|
|
5728
|
-
*
|
|
5729
|
-
*
|
|
5730
|
-
*
|
|
5731
|
-
*
|
|
5732
|
-
*
|
|
5733
|
-
*
|
|
5734
|
-
*
|
|
5735
|
-
*
|
|
5736
|
-
*
|
|
6469
|
+
* Outcome contract (consumed by the watcher's swap-to-running guard):
|
|
6470
|
+
* - resolves (`void`) → the server transitioned the row to
|
|
6471
|
+
* processing (or idempotently confirmed
|
|
6472
|
+
* already-processing — that answer is
|
|
6473
|
+
* still a 200, never a refusal);
|
|
6474
|
+
* - throws `ChannelAuthError` → 401/403 (terminal auth failure);
|
|
6475
|
+
* - throws `ChannelTerminalError` → a definitive non-retryable, non-auth 4xx
|
|
6476
|
+
* (404 the row or its conversation is
|
|
6477
|
+
* gone, 400 the update was rejected).
|
|
6478
|
+
* Retrying cannot help;
|
|
6479
|
+
* - throws a plain `Error` → a TRANSIENT failure (retryable 5xx/429
|
|
6480
|
+
* status, or a network-level error from
|
|
6481
|
+
* `fetch`) — i.e. NO definitive server
|
|
6482
|
+
* response — so the caller leaves the
|
|
6483
|
+
* message un-started and retries the swap
|
|
6484
|
+
* on the next tick.
|
|
5737
6485
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
5738
6486
|
* retry vehicle for the swap-to-running.
|
|
5739
6487
|
*/
|
|
@@ -5752,11 +6500,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
5752
6500
|
}
|
|
5753
6501
|
);
|
|
5754
6502
|
this.assertAuth(res, "marking message as processing");
|
|
5755
|
-
if (res.ok) return
|
|
6503
|
+
if (res.ok) return;
|
|
5756
6504
|
if (isRetryableStatus(res.status)) {
|
|
5757
6505
|
throw new Error(`marking message as processing: HTTP ${res.status}`);
|
|
5758
6506
|
}
|
|
5759
|
-
|
|
6507
|
+
throw new ChannelTerminalError(`marking message as processing: HTTP ${res.status}`, res.status);
|
|
5760
6508
|
}
|
|
5761
6509
|
/**
|
|
5762
6510
|
* EXISTING combinedAuth completion route — idempotent (WI-CHAN-2). `PATCH
|
|
@@ -6192,7 +6940,7 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
6192
6940
|
if (trimmed === "") {
|
|
6193
6941
|
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6194
6942
|
}
|
|
6195
|
-
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ?
|
|
6943
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join4(homeDir, trimmed.slice(2)) : trimmed;
|
|
6196
6944
|
if (!isAbsolute2(expanded)) {
|
|
6197
6945
|
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6198
6946
|
}
|
|
@@ -6242,6 +6990,32 @@ function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
|
6242
6990
|
}
|
|
6243
6991
|
return { timeoutMs: seconds * 1e3, warnings: [] };
|
|
6244
6992
|
}
|
|
6993
|
+
var MAX_ACTIVE_SESSIONS_ENV = "EVIDENT_MAX_ACTIVE_SESSIONS";
|
|
6994
|
+
function resolveMaxActiveSessions(options, env = process.env) {
|
|
6995
|
+
let raw;
|
|
6996
|
+
let source;
|
|
6997
|
+
if (options.maxActiveSessions !== void 0) {
|
|
6998
|
+
raw = options.maxActiveSessions;
|
|
6999
|
+
source = "--max-active-sessions";
|
|
7000
|
+
} else if (env[MAX_ACTIVE_SESSIONS_ENV] !== void 0 && env[MAX_ACTIVE_SESSIONS_ENV] !== "") {
|
|
7001
|
+
raw = env[MAX_ACTIVE_SESSIONS_ENV];
|
|
7002
|
+
source = MAX_ACTIVE_SESSIONS_ENV;
|
|
7003
|
+
} else {
|
|
7004
|
+
return { value: void 0, warnings: [] };
|
|
7005
|
+
}
|
|
7006
|
+
const trimmed = raw.trim();
|
|
7007
|
+
const count = Number(trimmed);
|
|
7008
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(count) && count > 0;
|
|
7009
|
+
if (!isPositiveInteger) {
|
|
7010
|
+
return {
|
|
7011
|
+
value: void 0,
|
|
7012
|
+
warnings: [
|
|
7013
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer; using unlimited`
|
|
7014
|
+
]
|
|
7015
|
+
};
|
|
7016
|
+
}
|
|
7017
|
+
return { value: count, warnings: [] };
|
|
7018
|
+
}
|
|
6245
7019
|
function meetsThreshold(state, level) {
|
|
6246
7020
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
6247
7021
|
}
|
|
@@ -6360,7 +7134,9 @@ async function handleAuthError(state, error2) {
|
|
|
6360
7134
|
);
|
|
6361
7135
|
const newAuthHeader = getAuthHeader(credentials2);
|
|
6362
7136
|
return { success: true, newAuthHeader };
|
|
6363
|
-
} catch {
|
|
7137
|
+
} catch (error3) {
|
|
7138
|
+
const message = error3 instanceof Error ? error3.message : String(error3);
|
|
7139
|
+
logActivity(state, { type: "error", error: `Re-authentication failed: ${message}` });
|
|
6364
7140
|
return { success: false };
|
|
6365
7141
|
}
|
|
6366
7142
|
}
|
|
@@ -6465,6 +7241,10 @@ async function driveChannels(state, driver) {
|
|
|
6465
7241
|
}
|
|
6466
7242
|
}
|
|
6467
7243
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
7244
|
+
var SESSION_DB_RECLAIM_MAX_PAGES = 2e3;
|
|
7245
|
+
function sessionDbPath() {
|
|
7246
|
+
return join4(homedir3(), ".local", "share", "opencode", "opencode.db");
|
|
7247
|
+
}
|
|
6468
7248
|
async function runSweep(state, driver, config) {
|
|
6469
7249
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
6470
7250
|
try {
|
|
@@ -6507,6 +7287,25 @@ async function runSweep(state, driver, config) {
|
|
|
6507
7287
|
type: "info",
|
|
6508
7288
|
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote} (${mode})`
|
|
6509
7289
|
});
|
|
7290
|
+
const reclaimResult = await reclaimSessionDbSpace({
|
|
7291
|
+
dbPath: sessionDbPath(),
|
|
7292
|
+
maxPages: SESSION_DB_RECLAIM_MAX_PAGES,
|
|
7293
|
+
allowFullVacuum: protectedNow.size === 0
|
|
7294
|
+
});
|
|
7295
|
+
if (reclaimResult.ok) {
|
|
7296
|
+
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
7297
|
+
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
7298
|
+
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
7299
|
+
logActivity(state, {
|
|
7300
|
+
type: "info",
|
|
7301
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote}`
|
|
7302
|
+
});
|
|
7303
|
+
} else {
|
|
7304
|
+
logActivity(state, {
|
|
7305
|
+
type: "info",
|
|
7306
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})`
|
|
7307
|
+
});
|
|
7308
|
+
}
|
|
6510
7309
|
} catch (error2) {
|
|
6511
7310
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
6512
7311
|
logActivity(state, {
|
|
@@ -6527,6 +7326,22 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
6527
7326
|
for (const warning2 of config.warnings) {
|
|
6528
7327
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
6529
7328
|
}
|
|
7329
|
+
const dbBytes = statSessionDbBytes(homedir3());
|
|
7330
|
+
void (async () => {
|
|
7331
|
+
const reclaimSkipReason = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
7332
|
+
const sizeWarning = buildSessionStoreSizeWarning({
|
|
7333
|
+
dbBytes,
|
|
7334
|
+
cleanupEnabled: config.enabled,
|
|
7335
|
+
reclaimSkipReason
|
|
7336
|
+
});
|
|
7337
|
+
if (sizeWarning !== null) {
|
|
7338
|
+
logActivity(state, { type: "info", level: "warn", message: sizeWarning });
|
|
7339
|
+
}
|
|
7340
|
+
})().catch((err) => {
|
|
7341
|
+
console.error(
|
|
7342
|
+
`[scheduleSessionCleanup] size-warning preflight failed: ${err instanceof Error ? err.message : String(err)}`
|
|
7343
|
+
);
|
|
7344
|
+
});
|
|
6530
7345
|
if (!config.enabled) return;
|
|
6531
7346
|
logActivity(state, {
|
|
6532
7347
|
type: "info",
|
|
@@ -6967,6 +7782,10 @@ async function run(options) {
|
|
|
6967
7782
|
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
6968
7783
|
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
6969
7784
|
}
|
|
7785
|
+
const { value: maxActiveSessions, warnings: maxActiveSessionsWarnings } = resolveMaxActiveSessions(options, process.env);
|
|
7786
|
+
for (const warning2 of maxActiveSessionsWarnings) {
|
|
7787
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
7788
|
+
}
|
|
6970
7789
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
6971
7790
|
try {
|
|
6972
7791
|
const oc = await ensureOpenCodeRunning({
|
|
@@ -7027,6 +7846,7 @@ async function run(options) {
|
|
|
7027
7846
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
7028
7847
|
fileSyncDirectories,
|
|
7029
7848
|
homeDir: homedir3(),
|
|
7849
|
+
maxActiveSessions,
|
|
7030
7850
|
log: (entry) => (
|
|
7031
7851
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
7032
7852
|
// survive the sink filter (they no longer collapse to info). `type`
|
|
@@ -7103,6 +7923,13 @@ async function run(options) {
|
|
|
7103
7923
|
logActivity(state, { type: "error", error: error2 });
|
|
7104
7924
|
if (state.interactive) displayStatus(state);
|
|
7105
7925
|
},
|
|
7926
|
+
// `warn`, not `info`: `forwardRunnerActivity`'s FORWARDED_LEVELS floor is
|
|
7927
|
+
// {'warn','error'}, so an `info` entry would never leave the machine and
|
|
7928
|
+
// an operator couldn't correlate a reconnect storm with a relay deploy.
|
|
7929
|
+
onWarning: (message) => {
|
|
7930
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
7931
|
+
if (state.interactive) displayStatus(state);
|
|
7932
|
+
},
|
|
7106
7933
|
// Web traffic is proxied transparently; note opencode is live and stamp
|
|
7107
7934
|
// proxied activity so the idle loop treats interactive proxy use as work.
|
|
7108
7935
|
// Fires per forwarded response head (incl. every SSE open) and excludes
|
|
@@ -7227,6 +8054,9 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
7227
8054
|
).option(
|
|
7228
8055
|
"--session-cleanup-max-count <n>",
|
|
7229
8056
|
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
8057
|
+
).option(
|
|
8058
|
+
"--max-active-sessions <n>",
|
|
8059
|
+
"Cap how many sessions this runner works on at once (default: unlimited). Env: EVIDENT_MAX_ACTIVE_SESSIONS"
|
|
7230
8060
|
).option(
|
|
7231
8061
|
"--session-cleanup-interval <duration>",
|
|
7232
8062
|
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
@@ -7260,6 +8090,7 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
7260
8090
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
7261
8091
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
7262
8092
|
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
8093
|
+
maxActiveSessions: options.maxActiveSessions,
|
|
7263
8094
|
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
7264
8095
|
// Raw string — the resolver in run.ts single-sources parsing
|
|
7265
8096
|
// (resolveClaudeUsageReportingMode).
|