@evident-ai/cli 3.5.2-dev.58beb33 → 3.5.2-dev.60daa5f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1386 -402
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -1063,13 +1062,83 @@ async function claudeUsage() {
|
|
|
1063
1062
|
|
|
1064
1063
|
// src/commands/run.ts
|
|
1065
1064
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1065
|
+
import { homedir as homedir7 } from "node:os";
|
|
1067
1066
|
import { isAbsolute as isAbsolute3, join as join11, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1067
|
import chalk7 from "chalk";
|
|
1069
1068
|
|
|
1070
1069
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1070
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1071
|
|
|
1072
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1073
|
+
function roleOf(m) {
|
|
1074
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1075
|
+
if (typeof m.role === "string") return m.role;
|
|
1076
|
+
const infoRole = m.info?.role;
|
|
1077
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1078
|
+
}
|
|
1079
|
+
function idOf(m) {
|
|
1080
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1081
|
+
if (typeof m.id === "string") return m.id;
|
|
1082
|
+
const infoId = m.info?.id;
|
|
1083
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1084
|
+
}
|
|
1085
|
+
function parentIdOf(m) {
|
|
1086
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1087
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1088
|
+
const infoParent = m.info?.parentID;
|
|
1089
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1090
|
+
}
|
|
1091
|
+
function textPartsOf(m) {
|
|
1092
|
+
if (!m || typeof m !== "object") return [];
|
|
1093
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1094
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1095
|
+
}
|
|
1096
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1097
|
+
if (!messages || messages.length === 0) return [];
|
|
1098
|
+
const byParent = messages.filter(
|
|
1099
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1100
|
+
);
|
|
1101
|
+
if (byParent.length > 0) return byParent;
|
|
1102
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1103
|
+
if (userIndex === -1) return [];
|
|
1104
|
+
const run2 = [];
|
|
1105
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1106
|
+
const role = roleOf(messages[i]);
|
|
1107
|
+
if (role === "user") break;
|
|
1108
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1109
|
+
}
|
|
1110
|
+
return run2;
|
|
1111
|
+
}
|
|
1112
|
+
function lastText(assistantRun) {
|
|
1113
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1114
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1115
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1116
|
+
}
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1120
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1121
|
+
if (lastText(run2) !== null) return run2;
|
|
1122
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1123
|
+
if (userIndex === -1) return run2;
|
|
1124
|
+
let continuation;
|
|
1125
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1126
|
+
if (roleOf(messages[i]) === "user") {
|
|
1127
|
+
continuation = messages[i];
|
|
1128
|
+
break;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const continuationId = idOf(continuation);
|
|
1132
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1133
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1134
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1135
|
+
}
|
|
1136
|
+
return run2;
|
|
1137
|
+
}
|
|
1138
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1139
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1073
1142
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1143
|
var TelemetryEventTypes = {
|
|
1075
1144
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1227,7 @@ import ora4 from "ora";
|
|
|
1158
1227
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1228
|
|
|
1160
1229
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1230
|
+
var CLI_VERSION = (true ? "3.5.2-dev.60daa5f" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1231
|
function getCliVersion() {
|
|
1163
1232
|
return CLI_VERSION;
|
|
1164
1233
|
}
|
|
@@ -1172,6 +1241,23 @@ var authProvider = null;
|
|
|
1172
1241
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1242
|
authProvider = provider;
|
|
1174
1243
|
}
|
|
1244
|
+
var diagnosticsReporter = null;
|
|
1245
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1246
|
+
diagnosticsReporter = reporter;
|
|
1247
|
+
}
|
|
1248
|
+
function reportTelemetryDiagnostic(message) {
|
|
1249
|
+
if (!diagnosticsReporter) {
|
|
1250
|
+
console.error(message);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
try {
|
|
1254
|
+
diagnosticsReporter(message);
|
|
1255
|
+
} catch (error2) {
|
|
1256
|
+
console.error(
|
|
1257
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1175
1261
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1262
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1263
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1323,9 @@ async function flushEvents() {
|
|
|
1237
1323
|
signal: controller.signal
|
|
1238
1324
|
});
|
|
1239
1325
|
if (!response.ok) {
|
|
1240
|
-
|
|
1326
|
+
reportTelemetryDiagnostic(
|
|
1327
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1328
|
+
);
|
|
1241
1329
|
}
|
|
1242
1330
|
} finally {
|
|
1243
1331
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1335,9 @@ async function flushEvents() {
|
|
|
1247
1335
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1336
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1337
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1338
|
+
reportTelemetryDiagnostic(
|
|
1339
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1340
|
+
);
|
|
1251
1341
|
lastFlushFailureLoggedAt = now;
|
|
1252
1342
|
suppressedFlushFailureCount = 0;
|
|
1253
1343
|
} else {
|
|
@@ -1299,8 +1389,227 @@ var EventTypes = {
|
|
|
1299
1389
|
};
|
|
1300
1390
|
|
|
1301
1391
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
|
|
1392
|
+
import { homedir as homedir2 } from "node:os";
|
|
1393
|
+
|
|
1394
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1395
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1396
|
+
import { dirname as dirname2 } from "node:path";
|
|
1397
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1398
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1399
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1400
|
+
function errorMessage(error2) {
|
|
1401
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1402
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1403
|
+
}
|
|
1404
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1405
|
+
const messages = [];
|
|
1406
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1407
|
+
let current = error2;
|
|
1408
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1409
|
+
visited.add(current);
|
|
1410
|
+
if (typeof current === "string") {
|
|
1411
|
+
messages.push(current);
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
if (current instanceof Error) {
|
|
1415
|
+
messages.push(current.name, current.message);
|
|
1416
|
+
current = current.cause;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
if (typeof current === "object") {
|
|
1420
|
+
const record = current;
|
|
1421
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1422
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1423
|
+
current = record.cause;
|
|
1424
|
+
continue;
|
|
1425
|
+
}
|
|
1426
|
+
break;
|
|
1427
|
+
}
|
|
1428
|
+
return messages.some(
|
|
1429
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1430
|
+
message
|
|
1431
|
+
)
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1435
|
+
try {
|
|
1436
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1437
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1438
|
+
if (availableBytes < requiredBytes) {
|
|
1439
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1440
|
+
}
|
|
1441
|
+
return null;
|
|
1442
|
+
} catch (err) {
|
|
1443
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
function readLogicalBytes(db) {
|
|
1447
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1448
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1449
|
+
return pageCount * pageSize;
|
|
1450
|
+
}
|
|
1451
|
+
function readFreelistCount(db) {
|
|
1452
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1453
|
+
}
|
|
1454
|
+
function readCheckpointResult(db) {
|
|
1455
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1456
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1457
|
+
}
|
|
1458
|
+
async function probeReclaimAvailability(input) {
|
|
1459
|
+
const { dbPath, requiredBytes } = input;
|
|
1460
|
+
let sqlite;
|
|
1461
|
+
try {
|
|
1462
|
+
sqlite = await import("node:sqlite");
|
|
1463
|
+
} catch (err) {
|
|
1464
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1465
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1466
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1467
|
+
}
|
|
1468
|
+
let autoVacuum = null;
|
|
1469
|
+
try {
|
|
1470
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1471
|
+
try {
|
|
1472
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1473
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1474
|
+
} finally {
|
|
1475
|
+
db.close();
|
|
1476
|
+
}
|
|
1477
|
+
} catch (err) {
|
|
1478
|
+
console.warn(
|
|
1479
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1480
|
+
);
|
|
1481
|
+
}
|
|
1482
|
+
if (autoVacuum !== 0) return null;
|
|
1483
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1484
|
+
}
|
|
1485
|
+
async function reclaimSessionDbSpace(input) {
|
|
1486
|
+
const {
|
|
1487
|
+
dbPath,
|
|
1488
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1489
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1490
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1491
|
+
now = () => performance.now(),
|
|
1492
|
+
allowFullVacuum = true
|
|
1493
|
+
} = input;
|
|
1494
|
+
let sqlite;
|
|
1495
|
+
try {
|
|
1496
|
+
sqlite = await import("node:sqlite");
|
|
1497
|
+
} catch (err) {
|
|
1498
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1499
|
+
console.warn(
|
|
1500
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1501
|
+
);
|
|
1502
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1503
|
+
}
|
|
1504
|
+
const { DatabaseSync } = sqlite;
|
|
1505
|
+
let db;
|
|
1506
|
+
try {
|
|
1507
|
+
db = new DatabaseSync(dbPath);
|
|
1508
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1509
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1510
|
+
if (autoVacuum === 0) {
|
|
1511
|
+
if (!allowFullVacuum) {
|
|
1512
|
+
console.warn(
|
|
1513
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1514
|
+
);
|
|
1515
|
+
return {
|
|
1516
|
+
ok: false,
|
|
1517
|
+
skipped: "full-vacuum-blocked",
|
|
1518
|
+
detail: "a session turn is live"
|
|
1519
|
+
};
|
|
1520
|
+
}
|
|
1521
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1522
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1523
|
+
if (skipReason !== null) {
|
|
1524
|
+
console.warn(
|
|
1525
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1526
|
+
);
|
|
1527
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1528
|
+
}
|
|
1529
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1530
|
+
const freelistBefore = readFreelistCount(db);
|
|
1531
|
+
const startedAt = now();
|
|
1532
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1533
|
+
db.exec("VACUUM");
|
|
1534
|
+
const afterBytes = readLogicalBytes(db);
|
|
1535
|
+
const freelistAfter = readFreelistCount(db);
|
|
1536
|
+
const checkpoint = readCheckpointResult(db);
|
|
1537
|
+
return {
|
|
1538
|
+
ok: true,
|
|
1539
|
+
mode: "convert",
|
|
1540
|
+
beforeBytes,
|
|
1541
|
+
afterBytes,
|
|
1542
|
+
freelistBefore,
|
|
1543
|
+
freelistAfter,
|
|
1544
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1545
|
+
stoppedBy: "freelist-drained",
|
|
1546
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1547
|
+
checkpoint
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
if (autoVacuum === 2) {
|
|
1551
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1552
|
+
const freelistBefore = readFreelistCount(db);
|
|
1553
|
+
const startedAt = now();
|
|
1554
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1555
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1556
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1557
|
+
let pagesAttempted = 0;
|
|
1558
|
+
let freelistAfter = freelistBefore;
|
|
1559
|
+
let stoppedBy = "freelist-drained";
|
|
1560
|
+
while (freelistAfter > 0) {
|
|
1561
|
+
if (pagesAttempted >= pageLimit) {
|
|
1562
|
+
stoppedBy = "page-ceiling";
|
|
1563
|
+
break;
|
|
1564
|
+
}
|
|
1565
|
+
if (now() - startedAt >= budget) {
|
|
1566
|
+
stoppedBy = "time-budget";
|
|
1567
|
+
break;
|
|
1568
|
+
}
|
|
1569
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1570
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1571
|
+
pagesAttempted += pages;
|
|
1572
|
+
freelistAfter = readFreelistCount(db);
|
|
1573
|
+
}
|
|
1574
|
+
const afterBytes = readLogicalBytes(db);
|
|
1575
|
+
const checkpoint = readCheckpointResult(db);
|
|
1576
|
+
return {
|
|
1577
|
+
ok: true,
|
|
1578
|
+
mode: "incremental",
|
|
1579
|
+
beforeBytes,
|
|
1580
|
+
afterBytes,
|
|
1581
|
+
freelistBefore,
|
|
1582
|
+
freelistAfter,
|
|
1583
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1584
|
+
stoppedBy,
|
|
1585
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1586
|
+
checkpoint
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
console.warn(
|
|
1590
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1591
|
+
);
|
|
1592
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1595
|
+
return {
|
|
1596
|
+
ok: false,
|
|
1597
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1598
|
+
detail: errorMessage(err)
|
|
1599
|
+
};
|
|
1600
|
+
} finally {
|
|
1601
|
+
db?.close();
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1606
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1607
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1608
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1609
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
1610
|
+
};
|
|
1303
1611
|
var SEVERITY_BY_LEVEL = {
|
|
1612
|
+
info: "info",
|
|
1304
1613
|
warn: "warning",
|
|
1305
1614
|
error: "error"
|
|
1306
1615
|
};
|
|
@@ -1310,26 +1619,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
1619
|
if (!levelMatch) return null;
|
|
1311
1620
|
const level = levelMatch[1].toUpperCase();
|
|
1312
1621
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
1622
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1623
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1624
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1625
|
+
const parsed = {
|
|
1626
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1627
|
+
sessionID: sessionMatch?.[1],
|
|
1628
|
+
messageID: messageIDMatch?.[1]
|
|
1629
|
+
};
|
|
1630
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1631
|
+
if (message !== void 0) parsed.message = message;
|
|
1632
|
+
return parsed;
|
|
1315
1633
|
}
|
|
1316
1634
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
1635
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1636
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1318
1637
|
let buffer = Buffer.alloc(0);
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1638
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1639
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1640
|
+
const metadata = {
|
|
1641
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1642
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1643
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1644
|
+
};
|
|
1322
1645
|
forwardRunnerActivity(
|
|
1323
1646
|
{
|
|
1324
1647
|
level: parsed.level,
|
|
1325
1648
|
error: line,
|
|
1326
|
-
metadata:
|
|
1649
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1327
1650
|
source: "opencode"
|
|
1328
1651
|
},
|
|
1329
1652
|
getContext()
|
|
1330
1653
|
);
|
|
1331
1654
|
};
|
|
1332
|
-
|
|
1655
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1656
|
+
for (const [key, state] of repeatStates) {
|
|
1657
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1658
|
+
continue;
|
|
1659
|
+
}
|
|
1660
|
+
if (state.repeatsSuppressed > 0) {
|
|
1661
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1662
|
+
}
|
|
1663
|
+
repeatStates.delete(key);
|
|
1664
|
+
}
|
|
1665
|
+
};
|
|
1666
|
+
const flushLine = (line) => {
|
|
1667
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1668
|
+
try {
|
|
1669
|
+
onSessionStoreWriteFailure();
|
|
1670
|
+
} catch (err) {
|
|
1671
|
+
reportTelemetryDiagnostic(
|
|
1672
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1677
|
+
if (!parsed) return;
|
|
1678
|
+
const now = Date.now();
|
|
1679
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1680
|
+
expireRepeatStates(now, key);
|
|
1681
|
+
if (key === void 0) {
|
|
1682
|
+
forwardLine(line, parsed);
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
const previous = repeatStates.get(key);
|
|
1686
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1687
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1688
|
+
repeatStates.set(key, {
|
|
1689
|
+
line,
|
|
1690
|
+
parsed,
|
|
1691
|
+
lastForwardedAt: now,
|
|
1692
|
+
repeatsSuppressed: 0
|
|
1693
|
+
});
|
|
1694
|
+
return;
|
|
1695
|
+
}
|
|
1696
|
+
previous.line = line;
|
|
1697
|
+
previous.parsed = parsed;
|
|
1698
|
+
previous.repeatsSuppressed++;
|
|
1699
|
+
};
|
|
1700
|
+
const forward = ((chunk) => {
|
|
1333
1701
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
1702
|
let newlineIndex;
|
|
1335
1703
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,14 +1708,44 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
1708
|
flushLine(buffer.toString("utf-8"));
|
|
1341
1709
|
buffer = Buffer.alloc(0);
|
|
1342
1710
|
}
|
|
1711
|
+
});
|
|
1712
|
+
forward.flush = () => {
|
|
1713
|
+
for (const state of repeatStates.values()) {
|
|
1714
|
+
if (state.repeatsSuppressed > 0) {
|
|
1715
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
repeatStates.clear();
|
|
1343
1719
|
};
|
|
1720
|
+
return forward;
|
|
1344
1721
|
}
|
|
1345
1722
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
1723
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
1724
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
1725
|
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1726
|
+
function redactHomeDirectory(message) {
|
|
1727
|
+
const home = homedir2();
|
|
1728
|
+
if (!home || home === "/") {
|
|
1729
|
+
return message;
|
|
1730
|
+
}
|
|
1731
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1732
|
+
let result = "";
|
|
1733
|
+
let searchFrom = 0;
|
|
1734
|
+
let homeIndex;
|
|
1735
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1736
|
+
const homeEnd = homeIndex + home.length;
|
|
1737
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1738
|
+
const nextCharacter = message[homeEnd];
|
|
1739
|
+
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1740
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1741
|
+
result += atPathBoundary ? "~" : home;
|
|
1742
|
+
searchFrom = homeEnd;
|
|
1743
|
+
}
|
|
1744
|
+
return result + message.slice(searchFrom);
|
|
1745
|
+
}
|
|
1746
|
+
function redactDiagnosticText(message) {
|
|
1747
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1748
|
+
return redactHomeDirectory(redacted);
|
|
1351
1749
|
}
|
|
1352
1750
|
function truncate(message) {
|
|
1353
1751
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1357,14 +1755,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1357
1755
|
if (!metadata) return {};
|
|
1358
1756
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
1757
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
1758
|
+
reportTelemetryDiagnostic(
|
|
1759
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
1760
|
);
|
|
1363
1761
|
}
|
|
1364
1762
|
const sanitised = [];
|
|
1365
1763
|
for (const [key, value] of entries) {
|
|
1366
1764
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
1765
|
+
sanitised.push([
|
|
1766
|
+
key,
|
|
1767
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1768
|
+
]);
|
|
1368
1769
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
1770
|
sanitised.push([key, value]);
|
|
1370
1771
|
}
|
|
@@ -1382,8 +1783,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
1783
|
}
|
|
1383
1784
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
1785
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
1786
|
+
reportTelemetryDiagnostic(
|
|
1787
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
1788
|
);
|
|
1388
1789
|
}
|
|
1389
1790
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +1794,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
1794
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
1795
|
window.windowDroppedCount++;
|
|
1395
1796
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
1797
|
+
reportTelemetryDiagnostic(
|
|
1798
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
1799
|
);
|
|
1399
1800
|
}
|
|
1400
1801
|
return false;
|
|
@@ -1404,12 +1805,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
1805
|
}
|
|
1405
1806
|
function forwardRunnerActivity(entry, context) {
|
|
1406
1807
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
1808
|
const source = entry.source ?? "cli.run";
|
|
1809
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
1810
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
1811
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
1812
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
1813
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
1814
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
1815
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
1816
|
message,
|
|
@@ -1417,8 +1818,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
1818
|
agentId: context.agentId
|
|
1418
1819
|
});
|
|
1419
1820
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
1821
|
+
reportTelemetryDiagnostic(
|
|
1822
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
1823
|
);
|
|
1423
1824
|
}
|
|
1424
1825
|
}
|
|
@@ -1708,8 +2109,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1708
2109
|
|
|
1709
2110
|
// src/lib/http-timeout.ts
|
|
1710
2111
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2112
|
+
var RequestTimeoutError = class extends Error {
|
|
2113
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2114
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2115
|
+
this.operation = operation;
|
|
2116
|
+
this.timeoutMs = timeoutMs;
|
|
2117
|
+
this.elapsedMs = elapsedMs;
|
|
2118
|
+
this.name = "TimeoutError";
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
function describeRequestOperation(input, init) {
|
|
2122
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2123
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2124
|
+
let path;
|
|
2125
|
+
try {
|
|
2126
|
+
path = new URL(rawUrl).pathname;
|
|
2127
|
+
} catch (error2) {
|
|
2128
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2129
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2130
|
+
}
|
|
2131
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2132
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2133
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2134
|
+
return `${method} ${maskedPath}`;
|
|
2135
|
+
}
|
|
1711
2136
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2137
|
+
return (async (input, init) => {
|
|
2138
|
+
const startedAt = Date.now();
|
|
2139
|
+
try {
|
|
2140
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2141
|
+
} catch (error2) {
|
|
2142
|
+
if (error2?.name === "TimeoutError") {
|
|
2143
|
+
throw new RequestTimeoutError(
|
|
2144
|
+
describeRequestOperation(input, init),
|
|
2145
|
+
timeoutMs,
|
|
2146
|
+
Date.now() - startedAt
|
|
2147
|
+
);
|
|
2148
|
+
}
|
|
2149
|
+
throw error2;
|
|
2150
|
+
}
|
|
2151
|
+
});
|
|
1713
2152
|
}
|
|
1714
2153
|
|
|
1715
2154
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2214,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2214
|
return null;
|
|
1776
2215
|
}
|
|
1777
2216
|
}
|
|
1778
|
-
function
|
|
2217
|
+
function roleOf2(m) {
|
|
1779
2218
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2219
|
if (typeof m.role === "string") return m.role;
|
|
1781
2220
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2228,13 @@ function createdOf(m) {
|
|
|
1789
2228
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2229
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2230
|
}
|
|
1792
|
-
function
|
|
2231
|
+
function idOf2(m) {
|
|
1793
2232
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2233
|
if (typeof m.id === "string") return m.id;
|
|
1795
2234
|
const infoId = m.info?.id;
|
|
1796
2235
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2236
|
}
|
|
1798
|
-
function
|
|
2237
|
+
function parentIdOf2(m) {
|
|
1799
2238
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2239
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2240
|
const infoParent = m.info?.parentID;
|
|
@@ -1860,9 +2299,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2299
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2300
|
if (!messages || messages.length === 0) return false;
|
|
1862
2301
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2302
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2303
|
return completedOf(last) == null;
|
|
1865
2304
|
}
|
|
2305
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2306
|
function sessionLastActivityMs(session) {
|
|
1867
2307
|
const candidates = [
|
|
1868
2308
|
session.time?.updated,
|
|
@@ -1877,30 +2317,60 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2317
|
}
|
|
1878
2318
|
return null;
|
|
1879
2319
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2320
|
+
function sessionListPath(options) {
|
|
2321
|
+
const params = new URLSearchParams();
|
|
2322
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2323
|
+
const query = params.toString();
|
|
2324
|
+
return query ? `/session?${query}` : "/session";
|
|
2325
|
+
}
|
|
2326
|
+
async function listSessionPage(port, client, options) {
|
|
2327
|
+
const path = sessionListPath(options);
|
|
1882
2328
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2329
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2330
|
if (!res.ok) return null;
|
|
1885
2331
|
const body = await res.json();
|
|
1886
2332
|
return Array.isArray(body) ? body : null;
|
|
1887
2333
|
} catch (error2) {
|
|
1888
2334
|
console.error(
|
|
1889
|
-
`[
|
|
2335
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2336
|
);
|
|
1891
2337
|
return null;
|
|
1892
2338
|
}
|
|
1893
2339
|
}
|
|
2340
|
+
async function listSessions(port, client, options) {
|
|
2341
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2342
|
+
return listSessionPage(port, client, options);
|
|
2343
|
+
}
|
|
2344
|
+
async function listAllSessions(port, client) {
|
|
2345
|
+
if (client?.version === "v2") {
|
|
2346
|
+
const sessions2 = await listV2Sessions(client);
|
|
2347
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2348
|
+
}
|
|
2349
|
+
const sessions = await listSessionPage(port, client, {
|
|
2350
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2351
|
+
});
|
|
2352
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2353
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2354
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2355
|
+
}
|
|
2356
|
+
return { sessions };
|
|
2357
|
+
}
|
|
1894
2358
|
async function deleteSession(port, id, client) {
|
|
1895
2359
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2360
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2361
|
+
const res = await requestWithClient(
|
|
2362
|
+
port,
|
|
2363
|
+
client,
|
|
2364
|
+
`/session/${id}`,
|
|
2365
|
+
{ method: "DELETE" },
|
|
2366
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2367
|
+
);
|
|
2368
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1899
2369
|
} catch (error2) {
|
|
1900
2370
|
console.error(
|
|
1901
2371
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2372
|
);
|
|
1903
|
-
return false;
|
|
2373
|
+
return { ok: false, status: "threw" };
|
|
1904
2374
|
}
|
|
1905
2375
|
}
|
|
1906
2376
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +2608,31 @@ function messageText(m) {
|
|
|
2138
2608
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
2609
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
2610
|
}
|
|
2141
|
-
|
|
2611
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2612
|
+
if (!messages || messages.length === 0) return null;
|
|
2613
|
+
let best = null;
|
|
2614
|
+
for (const message of messages) {
|
|
2615
|
+
if (roleOf2(message) !== "user") continue;
|
|
2616
|
+
const id = idOf2(message);
|
|
2617
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2618
|
+
if (messageText(message) !== content) continue;
|
|
2619
|
+
const created = createdOf(message) ?? 0;
|
|
2620
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2621
|
+
}
|
|
2622
|
+
return best?.id ?? null;
|
|
2623
|
+
}
|
|
2624
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2142
2625
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
2626
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
2627
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
2628
|
);
|
|
2629
|
+
if (callbacks) {
|
|
2630
|
+
try {
|
|
2631
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2632
|
+
} catch (error2) {
|
|
2633
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2146
2636
|
const parts = [{ type: "text", text: content }];
|
|
2147
2637
|
let pendingOutcomes = null;
|
|
2148
2638
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,22 +2671,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
2671
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
2672
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
2673
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
2674
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2675
|
+
if (dispatchedUserMessageId !== null) {
|
|
2676
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2677
|
+
return dispatchedUserMessageId;
|
|
2200
2678
|
}
|
|
2201
2679
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
2680
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2207,13 +2685,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
2685
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
2686
|
if (!messages || messages.length === 0) return null;
|
|
2209
2687
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
2688
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
2689
|
);
|
|
2212
2690
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
2691
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
2692
|
if (userIndex === -1) return null;
|
|
2215
2693
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
2694
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
2695
|
}
|
|
2218
2696
|
return null;
|
|
2219
2697
|
}
|
|
@@ -2223,7 +2701,7 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2223
2701
|
let lastNonErrored = null;
|
|
2224
2702
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
2703
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2704
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2227
2705
|
if (lastCorrelated === null) lastCorrelated = m;
|
|
2228
2706
|
if (errorOf(m) == null) {
|
|
2229
2707
|
lastNonErrored = m;
|
|
@@ -2231,12 +2709,12 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2231
2709
|
}
|
|
2232
2710
|
}
|
|
2233
2711
|
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2234
|
-
const userIndex = messages.findIndex((m) =>
|
|
2712
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
2713
|
if (userIndex === -1) return null;
|
|
2236
2714
|
let last = null;
|
|
2237
2715
|
let lastOk = null;
|
|
2238
2716
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
2717
|
+
const role = roleOf2(messages[i]);
|
|
2240
2718
|
if (role === "user") break;
|
|
2241
2719
|
if (role === "assistant") {
|
|
2242
2720
|
last = messages[i];
|
|
@@ -2248,16 +2726,16 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2248
2726
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
2727
|
if (!messages || messages.length === 0) return [];
|
|
2250
2728
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
2729
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
2730
|
);
|
|
2253
2731
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
2732
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
2733
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
2734
|
if (userIndex === -1) return [];
|
|
2257
2735
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
2736
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
2737
|
+
if (roleOf2(message) === "user") break;
|
|
2738
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
2739
|
}
|
|
2262
2740
|
}
|
|
2263
2741
|
const refs = [];
|
|
@@ -2297,7 +2775,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
2775
|
if (!messages || messages.length === 0) return [];
|
|
2298
2776
|
const calls = [];
|
|
2299
2777
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
2778
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
2779
|
for (const part of message.parts ?? []) {
|
|
2302
2780
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
2781
|
continue;
|
|
@@ -2327,7 +2805,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
2805
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
2806
|
const unattributed = [];
|
|
2329
2807
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
2808
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
2809
|
const created = finiteNumber(createdOf(message));
|
|
2332
2810
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
2811
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2404,7 +2882,7 @@ function sumAssistantUsage(messages) {
|
|
|
2404
2882
|
function messageUsage(messages, userMessageId) {
|
|
2405
2883
|
if (!messages || messages.length === 0) return null;
|
|
2406
2884
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
2885
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
2886
|
);
|
|
2409
2887
|
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
2888
|
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
@@ -2420,7 +2898,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
2898
|
}
|
|
2421
2899
|
function messageRunState(messages, userMessageId) {
|
|
2422
2900
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
2901
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
2902
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
2903
|
if (!hasUser) {
|
|
2426
2904
|
if (!reply) return "unknown";
|
|
@@ -2512,7 +2990,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
2990
|
if (!messages || messages.length === 0) return null;
|
|
2513
2991
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
2992
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
2993
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
2994
|
const created = createdOf(message);
|
|
2517
2995
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
2996
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3022,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3022
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3023
|
if (!messages || messages.length === 0) return false;
|
|
2546
3024
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3025
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3026
|
);
|
|
2549
3027
|
}
|
|
2550
3028
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3029
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3030
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3031
|
if (userIndex === -1) return false;
|
|
2554
3032
|
let hasLaterUser = false;
|
|
2555
3033
|
let hasStartedLaterUser = false;
|
|
2556
3034
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3035
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3036
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3037
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3038
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3039
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3040
|
return false;
|
|
2563
3041
|
}
|
|
2564
3042
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3043
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3044
|
)) {
|
|
2567
3045
|
hasStartedLaterUser = true;
|
|
2568
3046
|
}
|
|
@@ -3033,13 +3511,17 @@ async function listV2Sessions(client) {
|
|
|
3033
3511
|
}
|
|
3034
3512
|
async function deleteV2Session(client, sessionId) {
|
|
3035
3513
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
3514
|
+
const response = await client.request(
|
|
3515
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3516
|
+
{ method: "DELETE" },
|
|
3517
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3518
|
+
);
|
|
3519
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3038
3520
|
} catch (error2) {
|
|
3039
3521
|
console.error(
|
|
3040
3522
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
3523
|
);
|
|
3042
|
-
return false;
|
|
3524
|
+
return { ok: false, status: "threw" };
|
|
3043
3525
|
}
|
|
3044
3526
|
}
|
|
3045
3527
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3214,7 +3696,7 @@ import { join as join4 } from "node:path";
|
|
|
3214
3696
|
|
|
3215
3697
|
// src/lib/openai-usage.ts
|
|
3216
3698
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3699
|
+
import { homedir as homedir3 } from "node:os";
|
|
3218
3700
|
import { join as join3 } from "node:path";
|
|
3219
3701
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
3702
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
@@ -3229,7 +3711,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
3711
|
}
|
|
3230
3712
|
function readOpenCodeChatGptCredentials() {
|
|
3231
3713
|
try {
|
|
3232
|
-
const raw = readFileSync3(join3(
|
|
3714
|
+
const raw = readFileSync3(join3(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
3715
|
let parsed;
|
|
3234
3716
|
try {
|
|
3235
3717
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +3852,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
3852
|
].some((name) => headers.has(name));
|
|
3371
3853
|
}
|
|
3372
3854
|
async function getOpenAiUsage(port, client) {
|
|
3855
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
3856
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
3857
|
if (!credentials2) {
|
|
3375
3858
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3859
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
3860
|
"no_credentials"
|
|
3378
3861
|
);
|
|
3379
3862
|
}
|
|
3380
3863
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
3864
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3865
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
3866
|
"credentials_expired"
|
|
3384
3867
|
);
|
|
3385
3868
|
}
|
|
@@ -3660,9 +4143,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4143
|
|
|
3661
4144
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4145
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4146
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4147
|
+
import { homedir as homedir4 } from "node:os";
|
|
4148
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3666
4149
|
|
|
3667
4150
|
// src/lib/runner-synchroniser.ts
|
|
3668
4151
|
import { spawn } from "node:child_process";
|
|
@@ -3787,7 +4270,7 @@ function clearMarker(options) {
|
|
|
3787
4270
|
function markNoReplicate(options, message) {
|
|
3788
4271
|
if (options.noReplicateMarker) {
|
|
3789
4272
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4273
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3791
4274
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4275
|
} catch (error2) {
|
|
3793
4276
|
options.log(
|
|
@@ -3912,7 +4395,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4395
|
return null;
|
|
3913
4396
|
}
|
|
3914
4397
|
try {
|
|
3915
|
-
if (
|
|
4398
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4399
|
} catch (error2) {
|
|
3917
4400
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4401
|
options.log(
|
|
@@ -3942,7 +4425,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
4425
|
return null;
|
|
3943
4426
|
}
|
|
3944
4427
|
try {
|
|
3945
|
-
mkdirSync(
|
|
4428
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3946
4429
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
4430
|
} catch (error2) {
|
|
3948
4431
|
options.log(
|
|
@@ -4017,14 +4500,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
4500
|
case 0:
|
|
4018
4501
|
return;
|
|
4019
4502
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4503
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
4504
|
options.log(
|
|
4022
4505
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
4506
|
"warn"
|
|
4024
4507
|
);
|
|
4025
4508
|
return;
|
|
4026
4509
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4510
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
4511
|
discardSessionDbDebris(options);
|
|
4029
4512
|
markNoReplicate(
|
|
4030
4513
|
options,
|
|
@@ -4103,7 +4586,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4103
4586
|
}
|
|
4104
4587
|
function fileExists(path) {
|
|
4105
4588
|
try {
|
|
4106
|
-
|
|
4589
|
+
statSync3(path);
|
|
4107
4590
|
return true;
|
|
4108
4591
|
} catch (error2) {
|
|
4109
4592
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +4596,7 @@ function fileExists(path) {
|
|
|
4113
4596
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
4597
|
const env = options.env ?? process.env;
|
|
4115
4598
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4599
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
4600
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
4601
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
4602
|
env
|
|
@@ -4192,7 +4675,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4192
4675
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
4676
|
import { createRequire } from "node:module";
|
|
4194
4677
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
4678
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4196
4679
|
var require2 = createRequire(import.meta.url);
|
|
4197
4680
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
4681
|
let db;
|
|
@@ -4258,7 +4741,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
4741
|
}
|
|
4259
4742
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
4743
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
4744
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4262
4745
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
4746
|
`, "utf8");
|
|
4264
4747
|
} catch (error2) {
|
|
@@ -4839,7 +5322,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4839
5322
|
toDelete.push(s.id);
|
|
4840
5323
|
}
|
|
4841
5324
|
}
|
|
4842
|
-
|
|
5325
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5326
|
+
return toDelete.sort(
|
|
5327
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5328
|
+
);
|
|
4843
5329
|
}
|
|
4844
5330
|
var DEFAULT_INTERVAL = "1h";
|
|
4845
5331
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4899,13 +5385,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4899
5385
|
}
|
|
4900
5386
|
|
|
4901
5387
|
// src/lib/opencode/session-db-size.ts
|
|
4902
|
-
import { statSync as
|
|
5388
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4903
5389
|
import { join as join6 } from "node:path";
|
|
4904
5390
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4905
5391
|
function statSessionDbBytes(homeDir) {
|
|
4906
5392
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4907
5393
|
try {
|
|
4908
|
-
return
|
|
5394
|
+
return statSync4(dbPath).size;
|
|
4909
5395
|
} catch (err) {
|
|
4910
5396
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4911
5397
|
if (!isMissingFile) {
|
|
@@ -4931,12 +5417,12 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4931
5417
|
}
|
|
4932
5418
|
|
|
4933
5419
|
// src/lib/opencode/log-tail.ts
|
|
4934
|
-
import { statSync as
|
|
4935
|
-
import { homedir as
|
|
5420
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5421
|
+
import { homedir as homedir5 } from "node:os";
|
|
4936
5422
|
import { join as join7 } from "node:path";
|
|
4937
5423
|
import { open as open2, stat } from "node:fs/promises";
|
|
4938
5424
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4939
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
5425
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
4940
5426
|
const dataDir = env.XDG_DATA_HOME || join7(homeDir, ".local", "share");
|
|
4941
5427
|
return join7(dataDir, "opencode", "log", "opencode.log");
|
|
4942
5428
|
}
|
|
@@ -4954,7 +5440,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4954
5440
|
let inode = null;
|
|
4955
5441
|
let baselineReady = true;
|
|
4956
5442
|
try {
|
|
4957
|
-
const initial =
|
|
5443
|
+
const initial = statSync5(logPath);
|
|
4958
5444
|
offset = initial.size;
|
|
4959
5445
|
inode = initial.ino;
|
|
4960
5446
|
} catch (error2) {
|
|
@@ -5013,125 +5499,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5013
5499
|
};
|
|
5014
5500
|
}
|
|
5015
5501
|
|
|
5016
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5017
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5018
|
-
import { dirname as dirname4 } from "node:path";
|
|
5019
|
-
function errorMessage(error2) {
|
|
5020
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5021
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5022
|
-
}
|
|
5023
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5024
|
-
try {
|
|
5025
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5026
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5027
|
-
if (availableBytes < requiredBytes) {
|
|
5028
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5029
|
-
}
|
|
5030
|
-
return null;
|
|
5031
|
-
} catch (err) {
|
|
5032
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5033
|
-
}
|
|
5034
|
-
}
|
|
5035
|
-
function readLogicalBytes(db) {
|
|
5036
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5037
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5038
|
-
return pageCount * pageSize;
|
|
5039
|
-
}
|
|
5040
|
-
function readCheckpointResult(db) {
|
|
5041
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5042
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5043
|
-
}
|
|
5044
|
-
async function probeReclaimAvailability(input) {
|
|
5045
|
-
const { dbPath, requiredBytes } = input;
|
|
5046
|
-
let sqlite;
|
|
5047
|
-
try {
|
|
5048
|
-
sqlite = await import("node:sqlite");
|
|
5049
|
-
} catch (err) {
|
|
5050
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5051
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5052
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5053
|
-
}
|
|
5054
|
-
let autoVacuum = null;
|
|
5055
|
-
try {
|
|
5056
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5057
|
-
try {
|
|
5058
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5059
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5060
|
-
} finally {
|
|
5061
|
-
db.close();
|
|
5062
|
-
}
|
|
5063
|
-
} catch (err) {
|
|
5064
|
-
console.warn(
|
|
5065
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5066
|
-
);
|
|
5067
|
-
}
|
|
5068
|
-
if (autoVacuum !== 0) return null;
|
|
5069
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5070
|
-
}
|
|
5071
|
-
async function reclaimSessionDbSpace(input) {
|
|
5072
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(
|
|
5079
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5080
|
-
);
|
|
5081
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5082
|
-
}
|
|
5083
|
-
const { DatabaseSync } = sqlite;
|
|
5084
|
-
let db;
|
|
5085
|
-
try {
|
|
5086
|
-
db = new DatabaseSync(dbPath);
|
|
5087
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5088
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5089
|
-
if (autoVacuum === 0) {
|
|
5090
|
-
if (!allowFullVacuum) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5093
|
-
);
|
|
5094
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5095
|
-
}
|
|
5096
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5097
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5098
|
-
if (skipReason !== null) {
|
|
5099
|
-
console.warn(
|
|
5100
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5101
|
-
);
|
|
5102
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5103
|
-
}
|
|
5104
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5105
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5106
|
-
db.exec("VACUUM");
|
|
5107
|
-
const afterBytes = readLogicalBytes(db);
|
|
5108
|
-
const checkpoint = readCheckpointResult(db);
|
|
5109
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5110
|
-
}
|
|
5111
|
-
if (autoVacuum === 2) {
|
|
5112
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5113
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5114
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5115
|
-
const afterBytes = readLogicalBytes(db);
|
|
5116
|
-
const checkpoint = readCheckpointResult(db);
|
|
5117
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5118
|
-
}
|
|
5119
|
-
console.warn(
|
|
5120
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5121
|
-
);
|
|
5122
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5123
|
-
} catch (err) {
|
|
5124
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5125
|
-
return {
|
|
5126
|
-
ok: false,
|
|
5127
|
-
skipped: "reclaim-error",
|
|
5128
|
-
detail: errorMessage(err)
|
|
5129
|
-
};
|
|
5130
|
-
} finally {
|
|
5131
|
-
db?.close();
|
|
5132
|
-
}
|
|
5133
|
-
}
|
|
5134
|
-
|
|
5135
5502
|
// src/lib/tunnel/connection.ts
|
|
5136
5503
|
import WebSocket2 from "ws";
|
|
5137
5504
|
|
|
@@ -5361,6 +5728,8 @@ var StreamForwarder = class {
|
|
|
5361
5728
|
if (done) break;
|
|
5362
5729
|
const chunk = Buffer.from(value);
|
|
5363
5730
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5731
|
+
await stream.pausePromise;
|
|
5732
|
+
if (ac.signal.aborted) break;
|
|
5364
5733
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5365
5734
|
if (this.binaryFramesSupported) {
|
|
5366
5735
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5403,6 +5772,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5403
5772
|
}
|
|
5404
5773
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5405
5774
|
var BASE_RECONNECT_DELAY = 500;
|
|
5775
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5406
5776
|
function getReconnectDelay(attempt) {
|
|
5407
5777
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5408
5778
|
const jitter = Math.random() * 1e3;
|
|
@@ -5474,10 +5844,27 @@ function connectTunnel(options) {
|
|
|
5474
5844
|
ws.close();
|
|
5475
5845
|
reject(new Error("Connection timeout"));
|
|
5476
5846
|
}, 3e4);
|
|
5847
|
+
let relaySilenceTimeout;
|
|
5848
|
+
let relaySilenceTimedOut = false;
|
|
5849
|
+
const clearRelaySilenceTimeout = () => {
|
|
5850
|
+
if (relaySilenceTimeout === void 0) return;
|
|
5851
|
+
clearTimeout(relaySilenceTimeout);
|
|
5852
|
+
relaySilenceTimeout = void 0;
|
|
5853
|
+
};
|
|
5854
|
+
const armRelaySilenceTimeout = () => {
|
|
5855
|
+
clearRelaySilenceTimeout();
|
|
5856
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
5857
|
+
relaySilenceTimeout = void 0;
|
|
5858
|
+
relaySilenceTimedOut = true;
|
|
5859
|
+
ws.terminate();
|
|
5860
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
5861
|
+
relaySilenceTimeout.unref?.();
|
|
5862
|
+
};
|
|
5477
5863
|
let upgradeRejection = null;
|
|
5478
5864
|
let upgradeRejectionReason = null;
|
|
5479
5865
|
ws.on("unexpected-response", (_req, res) => {
|
|
5480
5866
|
clearTimeout(connectionTimeout);
|
|
5867
|
+
clearRelaySilenceTimeout();
|
|
5481
5868
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5482
5869
|
upgradeRejectionReason = reason;
|
|
5483
5870
|
const chunks = [];
|
|
@@ -5507,6 +5894,7 @@ function connectTunnel(options) {
|
|
|
5507
5894
|
onInfo?.("WebSocket connection established");
|
|
5508
5895
|
});
|
|
5509
5896
|
ws.on("message", (data, isBinary) => {
|
|
5897
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5510
5898
|
if (isBinary) {
|
|
5511
5899
|
try {
|
|
5512
5900
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5540,6 +5928,7 @@ function connectTunnel(options) {
|
|
|
5540
5928
|
switch (message.type) {
|
|
5541
5929
|
case "connected": {
|
|
5542
5930
|
clearTimeout(connectionTimeout);
|
|
5931
|
+
armRelaySilenceTimeout();
|
|
5543
5932
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5544
5933
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5545
5934
|
onConnected?.(connectedAgentId);
|
|
@@ -5564,6 +5953,7 @@ function connectTunnel(options) {
|
|
|
5564
5953
|
});
|
|
5565
5954
|
ws.on("error", (error2) => {
|
|
5566
5955
|
clearTimeout(connectionTimeout);
|
|
5956
|
+
clearRelaySilenceTimeout();
|
|
5567
5957
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5568
5958
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5569
5959
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5575,7 +5965,8 @@ function connectTunnel(options) {
|
|
|
5575
5965
|
);
|
|
5576
5966
|
});
|
|
5577
5967
|
ws.on("close", (code, reason) => {
|
|
5578
|
-
|
|
5968
|
+
clearRelaySilenceTimeout();
|
|
5969
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5579
5970
|
forwarder.abortAll();
|
|
5580
5971
|
onDisconnected?.(code, reasonStr);
|
|
5581
5972
|
});
|
|
@@ -5583,9 +5974,10 @@ function connectTunnel(options) {
|
|
|
5583
5974
|
}
|
|
5584
5975
|
|
|
5585
5976
|
// src/lib/tunnel/runner-connection.ts
|
|
5586
|
-
var RunnerConnection = class {
|
|
5977
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5587
5978
|
opts;
|
|
5588
5979
|
sleep;
|
|
5980
|
+
now;
|
|
5589
5981
|
connection = null;
|
|
5590
5982
|
resolvedAgentId;
|
|
5591
5983
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5594,10 +5986,17 @@ var RunnerConnection = class {
|
|
|
5594
5986
|
reconnectPromise = null;
|
|
5595
5987
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5596
5988
|
reconnectAttempt = 0;
|
|
5989
|
+
generation = 0;
|
|
5990
|
+
disconnectedAtMs = null;
|
|
5991
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
5992
|
+
"No pong received",
|
|
5993
|
+
"Orphaned socket"
|
|
5994
|
+
]);
|
|
5597
5995
|
constructor(opts) {
|
|
5598
5996
|
this.opts = opts;
|
|
5599
5997
|
this.resolvedAgentId = opts.agentId;
|
|
5600
5998
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
5999
|
+
this.now = opts.now ?? Date.now;
|
|
5601
6000
|
}
|
|
5602
6001
|
get agentId() {
|
|
5603
6002
|
return this.resolvedAgentId;
|
|
@@ -5633,14 +6032,24 @@ var RunnerConnection = class {
|
|
|
5633
6032
|
port: this.opts.port,
|
|
5634
6033
|
openCodePassword: this.opts.openCodePassword,
|
|
5635
6034
|
onConnected: (agentId) => {
|
|
6035
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5636
6036
|
this.reconnectAttempt = 0;
|
|
5637
6037
|
this.reconnecting = false;
|
|
5638
6038
|
this.resolvedAgentId = agentId;
|
|
5639
|
-
|
|
6039
|
+
this.generation++;
|
|
6040
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6041
|
+
this.disconnectedAtMs = null;
|
|
6042
|
+
events.onConnected(agentId, isReconnect, {
|
|
6043
|
+
generation: this.generation,
|
|
6044
|
+
attempt,
|
|
6045
|
+
downtimeMs
|
|
6046
|
+
});
|
|
5640
6047
|
},
|
|
5641
6048
|
onDisconnected: (code, reason) => {
|
|
5642
|
-
|
|
5643
|
-
|
|
6049
|
+
this.disconnectedAtMs = this.now();
|
|
6050
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6051
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6052
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5644
6053
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5645
6054
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5646
6055
|
});
|
|
@@ -5979,8 +6388,85 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5979
6388
|
};
|
|
5980
6389
|
}
|
|
5981
6390
|
|
|
6391
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6392
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6393
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6394
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6395
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6396
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6397
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6398
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6399
|
+
var BAND_RANK = {
|
|
6400
|
+
low: 1,
|
|
6401
|
+
very_low: 2,
|
|
6402
|
+
critical: 3
|
|
6403
|
+
};
|
|
6404
|
+
function bandFor(freeBytes) {
|
|
6405
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6406
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6407
|
+
return "low";
|
|
6408
|
+
}
|
|
6409
|
+
function formatBytes(bytes) {
|
|
6410
|
+
if (bytes === null) return "unavailable";
|
|
6411
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6412
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6413
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6414
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6415
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6416
|
+
}
|
|
6417
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6418
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6419
|
+
}
|
|
6420
|
+
function evaluateLowDiskWarning(input) {
|
|
6421
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6422
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6423
|
+
return { warning: null, state };
|
|
6424
|
+
}
|
|
6425
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6426
|
+
if (recovered) {
|
|
6427
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6428
|
+
}
|
|
6429
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6430
|
+
if (!low) return { warning: null, state };
|
|
6431
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6432
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6433
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6434
|
+
const worse = currentRank > previousRank;
|
|
6435
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6436
|
+
const nextState = {
|
|
6437
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6438
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6439
|
+
};
|
|
6440
|
+
return {
|
|
6441
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6442
|
+
state: nextState
|
|
6443
|
+
};
|
|
6444
|
+
}
|
|
6445
|
+
|
|
6446
|
+
// src/lib/opencode/session-store-health.ts
|
|
6447
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6448
|
+
function createSessionStoreHealth(options = {}) {
|
|
6449
|
+
const now = options.now ?? (() => Date.now());
|
|
6450
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6451
|
+
let lastUnwritableAt = null;
|
|
6452
|
+
let lastWritableAt = null;
|
|
6453
|
+
return {
|
|
6454
|
+
observeUnwritable: () => {
|
|
6455
|
+
lastUnwritableAt = now();
|
|
6456
|
+
},
|
|
6457
|
+
observeWritable: () => {
|
|
6458
|
+
lastWritableAt = now();
|
|
6459
|
+
},
|
|
6460
|
+
isUnwritable: () => {
|
|
6461
|
+
if (lastUnwritableAt === null) return false;
|
|
6462
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6463
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6464
|
+
}
|
|
6465
|
+
};
|
|
6466
|
+
}
|
|
6467
|
+
|
|
5982
6468
|
// src/lib/channels/driver.ts
|
|
5983
|
-
import { homedir as
|
|
6469
|
+
import { homedir as homedir6 } from "node:os";
|
|
5984
6470
|
|
|
5985
6471
|
// src/lib/runner-file-sync.ts
|
|
5986
6472
|
import { join as join9 } from "node:path";
|
|
@@ -6437,6 +6923,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6437
6923
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6438
6924
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6439
6925
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6926
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6440
6927
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6441
6928
|
var HEARTBEAT_MS = 6e4;
|
|
6442
6929
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6498,6 +6985,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6498
6985
|
maxActiveSessions;
|
|
6499
6986
|
watcherStallMs;
|
|
6500
6987
|
wedgeWarningIntervalMs;
|
|
6988
|
+
sessionStoreHealth;
|
|
6501
6989
|
openCodeClient;
|
|
6502
6990
|
/** Cache of conversationId → opencode sessionId. */
|
|
6503
6991
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6622,10 +7110,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6622
7110
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6623
7111
|
*/
|
|
6624
7112
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7113
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7114
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6625
7115
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6626
7116
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6627
7117
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6628
7118
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7119
|
+
/**
|
|
7120
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7121
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7122
|
+
* identical-text turn, so it never produces a correlation.
|
|
7123
|
+
*/
|
|
7124
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6629
7125
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6630
7126
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6631
7127
|
/**
|
|
@@ -6711,21 +7207,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6711
7207
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6712
7208
|
*/
|
|
6713
7209
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6714
|
-
/**
|
|
6715
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6716
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6717
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6718
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6719
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6720
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6721
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6722
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6723
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6724
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6725
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6726
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6727
|
-
*/
|
|
6728
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6729
7210
|
/**
|
|
6730
7211
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6731
7212
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6743,8 +7224,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6743
7224
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6744
7225
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6745
7226
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6746
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6747
|
-
*
|
|
7227
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7228
|
+
* the terminal-report fence and is never sent again).
|
|
6748
7229
|
*/
|
|
6749
7230
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6750
7231
|
/**
|
|
@@ -6863,10 +7344,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6863
7344
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6864
7345
|
this.now = config.now ?? (() => Date.now());
|
|
6865
7346
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6866
|
-
this.homeDir = config.homeDir ??
|
|
7347
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6867
7348
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6868
7349
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6869
7350
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7351
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6870
7352
|
}
|
|
6871
7353
|
get isV2() {
|
|
6872
7354
|
return this.openCodeClient.version === "v2";
|
|
@@ -6882,7 +7364,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6882
7364
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6883
7365
|
}
|
|
6884
7366
|
async listSessions() {
|
|
6885
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7367
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7368
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7369
|
+
});
|
|
6886
7370
|
}
|
|
6887
7371
|
async sessionExists(sessionId) {
|
|
6888
7372
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7059,6 +7543,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7059
7543
|
}
|
|
7060
7544
|
return ids;
|
|
7061
7545
|
}
|
|
7546
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7547
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7548
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7549
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7550
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7551
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7552
|
+
}
|
|
7553
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7554
|
+
}
|
|
7555
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7556
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7557
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7558
|
+
this.awaitingReadopt.delete(messageId);
|
|
7559
|
+
this.sessions.delete(conversationId);
|
|
7560
|
+
this.supersede(conversationId, sessionId);
|
|
7561
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7562
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7563
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7564
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7565
|
+
this.log({
|
|
7566
|
+
level: "error",
|
|
7567
|
+
message: errorMessage3,
|
|
7568
|
+
conversation_id: conversationId,
|
|
7569
|
+
message_id: messageId
|
|
7570
|
+
});
|
|
7571
|
+
const reportError = await this.reportUntrackableAck(
|
|
7572
|
+
conversationId,
|
|
7573
|
+
messageId,
|
|
7574
|
+
null,
|
|
7575
|
+
errorMessage3
|
|
7576
|
+
);
|
|
7577
|
+
if (reportError !== null) {
|
|
7578
|
+
this.log({
|
|
7579
|
+
level: "warn",
|
|
7580
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7581
|
+
conversation_id: conversationId,
|
|
7582
|
+
message_id: messageId
|
|
7583
|
+
});
|
|
7584
|
+
}
|
|
7585
|
+
if (source === "readopt") {
|
|
7586
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7587
|
+
}
|
|
7588
|
+
}
|
|
7062
7589
|
/**
|
|
7063
7590
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7064
7591
|
*
|
|
@@ -7238,6 +7765,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7238
7765
|
}
|
|
7239
7766
|
break;
|
|
7240
7767
|
}
|
|
7768
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7769
|
+
if (correlation) {
|
|
7770
|
+
correlation.attempts += 1;
|
|
7771
|
+
let correlatedId = null;
|
|
7772
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7773
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7774
|
+
conv,
|
|
7775
|
+
message,
|
|
7776
|
+
correlation.sessionId
|
|
7777
|
+
);
|
|
7778
|
+
if (polled.ok)
|
|
7779
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7780
|
+
} else if (correlation.knownUserIds === null) {
|
|
7781
|
+
this.log({
|
|
7782
|
+
level: "debug",
|
|
7783
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7784
|
+
conversation_id: conv.id,
|
|
7785
|
+
message_id: message.id
|
|
7786
|
+
});
|
|
7787
|
+
}
|
|
7788
|
+
if (correlatedId !== null) {
|
|
7789
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7790
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7791
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7792
|
+
this.dispatched.add(message.id);
|
|
7793
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7794
|
+
dispatched += 1;
|
|
7795
|
+
this.log({
|
|
7796
|
+
level: "info",
|
|
7797
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7798
|
+
conversation_id: conv.id,
|
|
7799
|
+
message_id: message.id
|
|
7800
|
+
});
|
|
7801
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7802
|
+
opencode_message_id: correlatedId
|
|
7803
|
+
});
|
|
7804
|
+
continue;
|
|
7805
|
+
}
|
|
7806
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7807
|
+
await this.handoffUntrackableCorrelation(
|
|
7808
|
+
conv.id,
|
|
7809
|
+
message.id,
|
|
7810
|
+
correlation.sessionId,
|
|
7811
|
+
correlation.attempts,
|
|
7812
|
+
"fresh"
|
|
7813
|
+
);
|
|
7814
|
+
break;
|
|
7815
|
+
}
|
|
7816
|
+
break;
|
|
7817
|
+
}
|
|
7241
7818
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7242
7819
|
if (effectiveOpencodeMessageId) {
|
|
7243
7820
|
const outcome = await this.resolveRedrive(
|
|
@@ -7259,6 +7836,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7259
7836
|
model: message.opencode_model ?? void 0
|
|
7260
7837
|
};
|
|
7261
7838
|
let opencodeMessageId;
|
|
7839
|
+
let preDispatchUserIds = null;
|
|
7262
7840
|
try {
|
|
7263
7841
|
this.log({
|
|
7264
7842
|
level: "info",
|
|
@@ -7288,7 +7866,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7288
7866
|
message.content,
|
|
7289
7867
|
options,
|
|
7290
7868
|
sendAttachments,
|
|
7291
|
-
this.openCodeClient
|
|
7869
|
+
this.openCodeClient,
|
|
7870
|
+
{
|
|
7871
|
+
onKnownUserIds: (ids) => {
|
|
7872
|
+
preDispatchUserIds = ids;
|
|
7873
|
+
},
|
|
7874
|
+
onKnownUserIdsError: (error2) => {
|
|
7875
|
+
this.log({
|
|
7876
|
+
level: "warn",
|
|
7877
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
7878
|
+
conversation_id: conv.id,
|
|
7879
|
+
message_id: message.id
|
|
7880
|
+
});
|
|
7881
|
+
}
|
|
7882
|
+
}
|
|
7292
7883
|
)
|
|
7293
7884
|
);
|
|
7294
7885
|
} catch (err) {
|
|
@@ -7296,6 +7887,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7296
7887
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7297
7888
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7298
7889
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
7890
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7299
7891
|
this.log({
|
|
7300
7892
|
level: "error",
|
|
7301
7893
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7371,39 +7963,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7371
7963
|
if (this.isV2) {
|
|
7372
7964
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7373
7965
|
}
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7379
|
-
|
|
7380
|
-
message_id: message.id
|
|
7381
|
-
});
|
|
7382
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7383
|
-
continue;
|
|
7384
|
-
}
|
|
7385
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7386
|
-
this.sessions.delete(conv.id);
|
|
7387
|
-
this.supersede(conv.id, sessionId);
|
|
7388
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
7966
|
+
this.awaitingCorrelation.set(message.id, {
|
|
7967
|
+
sessionId,
|
|
7968
|
+
content: message.content,
|
|
7969
|
+
attempts: 0,
|
|
7970
|
+
knownUserIds: preDispatchUserIds
|
|
7971
|
+
});
|
|
7389
7972
|
this.log({
|
|
7390
|
-
level: "
|
|
7391
|
-
message:
|
|
7973
|
+
level: "warn",
|
|
7974
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7392
7975
|
conversation_id: conv.id,
|
|
7393
7976
|
message_id: message.id
|
|
7394
7977
|
});
|
|
7395
|
-
|
|
7396
|
-
this.log({
|
|
7397
|
-
level: "warn",
|
|
7398
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7399
|
-
conversation_id: conv.id,
|
|
7400
|
-
message_id: message.id
|
|
7401
|
-
});
|
|
7402
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7403
|
-
});
|
|
7978
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7404
7979
|
break;
|
|
7405
7980
|
}
|
|
7406
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7407
7981
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7408
7982
|
this.releasedOpencodeIds.delete(message.id);
|
|
7409
7983
|
this.dispatched.add(message.id);
|
|
@@ -8008,23 +8582,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8008
8582
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8009
8583
|
return 1;
|
|
8010
8584
|
}
|
|
8011
|
-
/**
|
|
8012
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8013
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8014
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8015
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8016
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8017
|
-
* count, since a new session is a genuinely different attempt.
|
|
8018
|
-
*/
|
|
8019
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8020
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8021
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8022
|
-
existing.count += 1;
|
|
8023
|
-
return existing.count;
|
|
8024
|
-
}
|
|
8025
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8026
|
-
return 1;
|
|
8027
|
-
}
|
|
8028
8585
|
/** Record one queued give-up toward the never-started bound. */
|
|
8029
8586
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8030
8587
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8336,8 +8893,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8336
8893
|
b2LastDescendantCheckMs: 0,
|
|
8337
8894
|
b2RootOngoingHeldLogged: false,
|
|
8338
8895
|
b2AbandonedSignalled: false,
|
|
8896
|
+
b2AbandonedWithoutAnswer: false,
|
|
8339
8897
|
ambiguousPinnedSinceMs: 0,
|
|
8340
8898
|
ambiguousResolved: false,
|
|
8899
|
+
sessionStoreFailurePending: false,
|
|
8341
8900
|
neverStartedResolved: false
|
|
8342
8901
|
});
|
|
8343
8902
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8454,8 +9013,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8454
9013
|
b2LastDescendantCheckMs: 0,
|
|
8455
9014
|
b2RootOngoingHeldLogged: false,
|
|
8456
9015
|
b2AbandonedSignalled: false,
|
|
9016
|
+
b2AbandonedWithoutAnswer: false,
|
|
8457
9017
|
ambiguousPinnedSinceMs: 0,
|
|
8458
9018
|
ambiguousResolved: false,
|
|
9019
|
+
sessionStoreFailurePending: false,
|
|
8459
9020
|
neverStartedResolved: false
|
|
8460
9021
|
});
|
|
8461
9022
|
}
|
|
@@ -8869,6 +9430,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8869
9430
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
9431
|
this.neverStartedAttempts.delete(id);
|
|
8871
9432
|
}
|
|
9433
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9434
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9435
|
+
return;
|
|
9436
|
+
}
|
|
8872
9437
|
if (inFlight.neverStartedResolved) {
|
|
8873
9438
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8874
9439
|
inFlight.neverStartedResolved = false;
|
|
@@ -9028,10 +9593,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9028
9593
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9029
9594
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9030
9595
|
inFlight.b2AbandonedSignalled = false;
|
|
9596
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9031
9597
|
}
|
|
9032
9598
|
} else {
|
|
9033
9599
|
if (inFlight.b2AbandonedSignalled) {
|
|
9034
|
-
|
|
9600
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9601
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9602
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9603
|
+
sessionId,
|
|
9604
|
+
watcher,
|
|
9605
|
+
inFlight,
|
|
9606
|
+
messages,
|
|
9607
|
+
reply?.info?.finish ?? reply?.finish
|
|
9608
|
+
);
|
|
9609
|
+
} else {
|
|
9610
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9611
|
+
}
|
|
9035
9612
|
return;
|
|
9036
9613
|
}
|
|
9037
9614
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9048,17 +9625,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9048
9625
|
descendantOngoing,
|
|
9049
9626
|
rootOngoing
|
|
9050
9627
|
})) {
|
|
9628
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9629
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9630
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9631
|
+
watched_for_ms: pinnedForMs,
|
|
9632
|
+
run_state: "failed",
|
|
9633
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9634
|
+
});
|
|
9635
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9636
|
+
return;
|
|
9637
|
+
}
|
|
9051
9638
|
inFlight.b2AbandonedSignalled = true;
|
|
9639
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9640
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9641
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9642
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9643
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9644
|
+
watched_for_ms: pinnedForMs,
|
|
9645
|
+
run_state: "failed",
|
|
9646
|
+
finish,
|
|
9647
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9648
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9649
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9650
|
+
});
|
|
9651
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9652
|
+
sessionId,
|
|
9653
|
+
watcher,
|
|
9654
|
+
inFlight,
|
|
9655
|
+
messages,
|
|
9656
|
+
finish
|
|
9657
|
+
);
|
|
9658
|
+
return;
|
|
9659
|
+
}
|
|
9052
9660
|
this.log({
|
|
9053
9661
|
level: "warn",
|
|
9054
9662
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9055
9663
|
conversation_id: conv.id,
|
|
9056
9664
|
message_id: id
|
|
9057
9665
|
});
|
|
9058
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9059
9666
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9060
9667
|
watched_for_ms: pinnedForMs,
|
|
9061
|
-
finish
|
|
9668
|
+
finish,
|
|
9062
9669
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9063
9670
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9064
9671
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9106,6 +9713,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9106
9713
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9107
9714
|
sessionOngoing: ongoing
|
|
9108
9715
|
})) {
|
|
9716
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9717
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9718
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9719
|
+
watched_for_ms: pinnedForMs,
|
|
9720
|
+
run_state: "failed",
|
|
9721
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9722
|
+
});
|
|
9723
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9724
|
+
return;
|
|
9725
|
+
}
|
|
9109
9726
|
inFlight.ambiguousResolved = true;
|
|
9110
9727
|
this.log({
|
|
9111
9728
|
level: "warn",
|
|
@@ -9330,6 +9947,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9330
9947
|
}
|
|
9331
9948
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9332
9949
|
}
|
|
9950
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
9951
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
9952
|
+
const conv = watcher.conv;
|
|
9953
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
9954
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
9955
|
+
this.log({
|
|
9956
|
+
level: "warn",
|
|
9957
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
9958
|
+
conversation_id: conv.id,
|
|
9959
|
+
message_id: inFlight.evidentMessageId
|
|
9960
|
+
});
|
|
9961
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
9962
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
9963
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
9964
|
+
messages,
|
|
9965
|
+
inFlight.opencodeMessageId,
|
|
9966
|
+
inFlight.evidentMessageId
|
|
9967
|
+
);
|
|
9968
|
+
try {
|
|
9969
|
+
await this.markFailed(
|
|
9970
|
+
conv.id,
|
|
9971
|
+
inFlight.evidentMessageId,
|
|
9972
|
+
sessionId,
|
|
9973
|
+
error2,
|
|
9974
|
+
usage,
|
|
9975
|
+
void 0,
|
|
9976
|
+
usageAgentName,
|
|
9977
|
+
subagentInvocations
|
|
9978
|
+
);
|
|
9979
|
+
} catch (err) {
|
|
9980
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
9981
|
+
if (err instanceof ChannelTerminalError) {
|
|
9982
|
+
this.log({
|
|
9983
|
+
level: "warn",
|
|
9984
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9985
|
+
conversation_id: conv.id,
|
|
9986
|
+
message_id: inFlight.evidentMessageId
|
|
9987
|
+
});
|
|
9988
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9989
|
+
return;
|
|
9990
|
+
}
|
|
9991
|
+
if (this.now() >= inFlight.deadline) {
|
|
9992
|
+
this.log({
|
|
9993
|
+
level: "warn",
|
|
9994
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
9995
|
+
conversation_id: conv.id,
|
|
9996
|
+
message_id: inFlight.evidentMessageId
|
|
9997
|
+
});
|
|
9998
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9999
|
+
return;
|
|
10000
|
+
}
|
|
10001
|
+
this.log({
|
|
10002
|
+
level: "warn",
|
|
10003
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10004
|
+
conversation_id: conv.id,
|
|
10005
|
+
message_id: inFlight.evidentMessageId
|
|
10006
|
+
});
|
|
10007
|
+
return;
|
|
10008
|
+
}
|
|
10009
|
+
inFlight.done = true;
|
|
10010
|
+
await this.reportSubagentAuthFailures(
|
|
10011
|
+
watcher.conv.id,
|
|
10012
|
+
inFlight.opencodeMessageId,
|
|
10013
|
+
inFlight.evidentMessageId,
|
|
10014
|
+
messages
|
|
10015
|
+
);
|
|
10016
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10017
|
+
}
|
|
9333
10018
|
/**
|
|
9334
10019
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9335
10020
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9385,6 +10070,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9385
10070
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9386
10071
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9387
10072
|
}
|
|
10073
|
+
/**
|
|
10074
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10075
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10076
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10077
|
+
*/
|
|
10078
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10079
|
+
const conv = watcher.conv;
|
|
10080
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10081
|
+
this.log({
|
|
10082
|
+
level: "error",
|
|
10083
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10084
|
+
conversation_id: conv.id,
|
|
10085
|
+
message_id: inFlight.evidentMessageId
|
|
10086
|
+
});
|
|
10087
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10088
|
+
try {
|
|
10089
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10090
|
+
} catch (err) {
|
|
10091
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10092
|
+
if (err instanceof ChannelTerminalError) {
|
|
10093
|
+
this.log({
|
|
10094
|
+
level: "warn",
|
|
10095
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10096
|
+
conversation_id: conv.id,
|
|
10097
|
+
message_id: inFlight.evidentMessageId
|
|
10098
|
+
});
|
|
10099
|
+
this.recordReleasedOpencodeId(
|
|
10100
|
+
inFlight.evidentMessageId,
|
|
10101
|
+
sessionId,
|
|
10102
|
+
inFlight.opencodeMessageId
|
|
10103
|
+
);
|
|
10104
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10105
|
+
return;
|
|
10106
|
+
}
|
|
10107
|
+
if (this.now() >= inFlight.deadline) {
|
|
10108
|
+
this.log({
|
|
10109
|
+
level: "warn",
|
|
10110
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10111
|
+
conversation_id: conv.id,
|
|
10112
|
+
message_id: inFlight.evidentMessageId
|
|
10113
|
+
});
|
|
10114
|
+
this.recordReleasedOpencodeId(
|
|
10115
|
+
inFlight.evidentMessageId,
|
|
10116
|
+
sessionId,
|
|
10117
|
+
inFlight.opencodeMessageId
|
|
10118
|
+
);
|
|
10119
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10120
|
+
return;
|
|
10121
|
+
}
|
|
10122
|
+
this.log({
|
|
10123
|
+
level: "warn",
|
|
10124
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10125
|
+
conversation_id: conv.id,
|
|
10126
|
+
message_id: inFlight.evidentMessageId
|
|
10127
|
+
});
|
|
10128
|
+
return;
|
|
10129
|
+
}
|
|
10130
|
+
inFlight.done = true;
|
|
10131
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10132
|
+
}
|
|
9388
10133
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9389
10134
|
/**
|
|
9390
10135
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9423,11 +10168,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9423
10168
|
}
|
|
9424
10169
|
}
|
|
9425
10170
|
}
|
|
9426
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10171
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9427
10172
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10173
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10174
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10175
|
+
this.awaitingCorrelation.delete(id);
|
|
10176
|
+
this.log({
|
|
10177
|
+
level: "debug",
|
|
10178
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10179
|
+
message_id: id
|
|
10180
|
+
});
|
|
10181
|
+
}
|
|
10182
|
+
}
|
|
9428
10183
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9429
10184
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9430
10185
|
this.untrackableAck.delete(id);
|
|
10186
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9431
10187
|
this.untrackableAckSignalled.delete(id);
|
|
9432
10188
|
this.untrackableAckWarned.delete(id);
|
|
9433
10189
|
this.log({
|
|
@@ -9467,7 +10223,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9467
10223
|
bySession.set(row.opencode_session_id, list);
|
|
9468
10224
|
}
|
|
9469
10225
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9470
|
-
let messages;
|
|
10226
|
+
let messages = null;
|
|
9471
10227
|
try {
|
|
9472
10228
|
if (this.isV2) {
|
|
9473
10229
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9476,9 +10232,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9476
10232
|
level: "warn",
|
|
9477
10233
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9478
10234
|
});
|
|
9479
|
-
|
|
10235
|
+
} else {
|
|
10236
|
+
messages = snapshot;
|
|
9480
10237
|
}
|
|
9481
|
-
messages = snapshot;
|
|
9482
10238
|
} else {
|
|
9483
10239
|
const polled = await pollSessionMessagesForRedrive(
|
|
9484
10240
|
this.port,
|
|
@@ -9491,15 +10247,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9491
10247
|
level: "warn",
|
|
9492
10248
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9493
10249
|
});
|
|
9494
|
-
|
|
10250
|
+
} else {
|
|
10251
|
+
messages = polled.messages;
|
|
9495
10252
|
}
|
|
9496
|
-
messages = polled.messages;
|
|
9497
10253
|
}
|
|
9498
10254
|
} catch (err) {
|
|
9499
10255
|
this.log({
|
|
9500
10256
|
level: "warn",
|
|
9501
10257
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9502
10258
|
});
|
|
10259
|
+
}
|
|
10260
|
+
if (messages === null) {
|
|
10261
|
+
for (const row of sessionRows) {
|
|
10262
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10263
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10264
|
+
}
|
|
10265
|
+
}
|
|
9503
10266
|
continue;
|
|
9504
10267
|
}
|
|
9505
10268
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9509,6 +10272,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9509
10272
|
}
|
|
9510
10273
|
}
|
|
9511
10274
|
}
|
|
10275
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10276
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10277
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10278
|
+
if (!correlation) return false;
|
|
10279
|
+
correlation.attempts += 1;
|
|
10280
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10281
|
+
if (correlatedId !== null) {
|
|
10282
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10283
|
+
const conv = this.convForRow(sessionId, row);
|
|
10284
|
+
const message = this.queuedMessageForRow(row);
|
|
10285
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10286
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10287
|
+
this.dispatched.add(row.id);
|
|
10288
|
+
this.readopted.add(row.id);
|
|
10289
|
+
this.ensureWatcherRunning(sessionId);
|
|
10290
|
+
this.log({
|
|
10291
|
+
level: "info",
|
|
10292
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10293
|
+
conversation_id: row.conversation_id,
|
|
10294
|
+
message_id: row.id
|
|
10295
|
+
});
|
|
10296
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10297
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10298
|
+
opencode_message_id: correlatedId
|
|
10299
|
+
});
|
|
10300
|
+
return true;
|
|
10301
|
+
}
|
|
10302
|
+
if (correlation.knownUserIds === null) {
|
|
10303
|
+
this.log({
|
|
10304
|
+
level: "debug",
|
|
10305
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10306
|
+
conversation_id: row.conversation_id,
|
|
10307
|
+
message_id: row.id
|
|
10308
|
+
});
|
|
10309
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10310
|
+
this.log({
|
|
10311
|
+
level: "warn",
|
|
10312
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10313
|
+
conversation_id: row.conversation_id,
|
|
10314
|
+
message_id: row.id
|
|
10315
|
+
});
|
|
10316
|
+
}
|
|
10317
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10318
|
+
await this.handoffUntrackableCorrelation(
|
|
10319
|
+
row.conversation_id,
|
|
10320
|
+
row.id,
|
|
10321
|
+
correlation.sessionId,
|
|
10322
|
+
correlation.attempts,
|
|
10323
|
+
"readopt"
|
|
10324
|
+
);
|
|
10325
|
+
}
|
|
10326
|
+
return true;
|
|
10327
|
+
}
|
|
9512
10328
|
/**
|
|
9513
10329
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9514
10330
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9516,8 +10332,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9516
10332
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9517
10333
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9518
10334
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9519
|
-
* before the restart)
|
|
9520
|
-
*
|
|
10335
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10336
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10337
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10338
|
+
* `forceReadoptRun`):
|
|
9521
10339
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9522
10340
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9523
10341
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9526,7 +10344,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9526
10344
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9527
10345
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9528
10346
|
* tracking the stored id so the reply correlates by it;
|
|
9529
|
-
* - `unknown`/null id →
|
|
10347
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10348
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9530
10349
|
*
|
|
9531
10350
|
* Only `ChannelAuthError` propagates.
|
|
9532
10351
|
*/
|
|
@@ -9540,6 +10359,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9540
10359
|
});
|
|
9541
10360
|
return;
|
|
9542
10361
|
}
|
|
10362
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9543
10363
|
const ocId = row.opencode_message_id;
|
|
9544
10364
|
const state = messageRunState(messages, ocId ?? "");
|
|
9545
10365
|
if (state === "done") {
|
|
@@ -9606,12 +10426,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9606
10426
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9607
10427
|
this.dontRedispatch.delete(row.id);
|
|
9608
10428
|
this.untrackableAck.delete(row.id);
|
|
10429
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9609
10430
|
this.untrackableAckSignalled.delete(row.id);
|
|
9610
10431
|
this.untrackableAckWarned.delete(row.id);
|
|
9611
10432
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9612
10433
|
return;
|
|
9613
10434
|
}
|
|
9614
|
-
if (this.
|
|
10435
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10436
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10437
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10438
|
+
this.untrackableAckWarned.add(row.id);
|
|
10439
|
+
this.log({
|
|
10440
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10441
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10442
|
+
conversation_id: row.conversation_id,
|
|
10443
|
+
message_id: row.id
|
|
10444
|
+
});
|
|
10445
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10446
|
+
const reportError = await this.reportUntrackableAck(
|
|
10447
|
+
row.conversation_id,
|
|
10448
|
+
row.id,
|
|
10449
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10450
|
+
errorMessage3
|
|
10451
|
+
);
|
|
10452
|
+
if (reportError !== null) {
|
|
10453
|
+
this.log({
|
|
10454
|
+
level: "warn",
|
|
10455
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10456
|
+
conversation_id: row.conversation_id,
|
|
10457
|
+
message_id: row.id
|
|
10458
|
+
});
|
|
10459
|
+
}
|
|
10460
|
+
} else {
|
|
10461
|
+
this.log({
|
|
10462
|
+
level: "debug",
|
|
10463
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10464
|
+
conversation_id: row.conversation_id,
|
|
10465
|
+
message_id: row.id
|
|
10466
|
+
});
|
|
10467
|
+
}
|
|
10468
|
+
return;
|
|
10469
|
+
}
|
|
10470
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9615
10471
|
this.log({
|
|
9616
10472
|
level: "debug",
|
|
9617
10473
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9789,6 +10645,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9789
10645
|
}
|
|
9790
10646
|
this.dontRedispatch.delete(row.id);
|
|
9791
10647
|
this.untrackableAck.delete(row.id);
|
|
10648
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9792
10649
|
this.untrackableAckSignalled.delete(row.id);
|
|
9793
10650
|
this.untrackableAckWarned.delete(row.id);
|
|
9794
10651
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9808,8 +10665,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9808
10665
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9809
10666
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9810
10667
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9811
|
-
* re-entry)
|
|
9812
|
-
*
|
|
10668
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10669
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10670
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9813
10671
|
*
|
|
9814
10672
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9815
10673
|
* `processed_at` (Invariant 1).
|
|
@@ -9872,6 +10730,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9872
10730
|
);
|
|
9873
10731
|
}
|
|
9874
10732
|
let ocId;
|
|
10733
|
+
let preDispatchUserIds = null;
|
|
9875
10734
|
try {
|
|
9876
10735
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9877
10736
|
sessionId,
|
|
@@ -9881,7 +10740,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9881
10740
|
row.content,
|
|
9882
10741
|
options,
|
|
9883
10742
|
sendAttachments,
|
|
9884
|
-
this.openCodeClient
|
|
10743
|
+
this.openCodeClient,
|
|
10744
|
+
{
|
|
10745
|
+
onKnownUserIds: (ids) => {
|
|
10746
|
+
preDispatchUserIds = ids;
|
|
10747
|
+
},
|
|
10748
|
+
onKnownUserIdsError: (error2) => {
|
|
10749
|
+
this.log({
|
|
10750
|
+
level: "warn",
|
|
10751
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10752
|
+
conversation_id: row.conversation_id,
|
|
10753
|
+
message_id: row.id
|
|
10754
|
+
});
|
|
10755
|
+
}
|
|
10756
|
+
}
|
|
9885
10757
|
)
|
|
9886
10758
|
);
|
|
9887
10759
|
} catch (err) {
|
|
@@ -9914,6 +10786,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9914
10786
|
}
|
|
9915
10787
|
} else {
|
|
9916
10788
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10789
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9917
10790
|
}
|
|
9918
10791
|
this.log({
|
|
9919
10792
|
level: "error",
|
|
@@ -9962,39 +10835,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9962
10835
|
}
|
|
9963
10836
|
if (ocId === null) {
|
|
9964
10837
|
this.awaitingReadopt.delete(row.id);
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
this.log({
|
|
9972
|
-
level: "error",
|
|
9973
|
-
message: errorMessage3,
|
|
9974
|
-
conversation_id: row.conversation_id,
|
|
9975
|
-
message_id: row.id
|
|
9976
|
-
});
|
|
9977
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9978
|
-
this.log({
|
|
9979
|
-
level: "warn",
|
|
9980
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9981
|
-
conversation_id: row.conversation_id,
|
|
9982
|
-
message_id: row.id
|
|
9983
|
-
});
|
|
9984
|
-
});
|
|
9985
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9986
|
-
return;
|
|
9987
|
-
}
|
|
10838
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10839
|
+
sessionId,
|
|
10840
|
+
content: row.content,
|
|
10841
|
+
attempts: 0,
|
|
10842
|
+
knownUserIds: preDispatchUserIds
|
|
10843
|
+
});
|
|
9988
10844
|
this.log({
|
|
9989
10845
|
level: "warn",
|
|
9990
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10846
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9991
10847
|
conversation_id: row.conversation_id,
|
|
9992
10848
|
message_id: row.id
|
|
9993
10849
|
});
|
|
9994
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9995
10850
|
return;
|
|
9996
10851
|
}
|
|
9997
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9998
10852
|
const processedAtMs = this.processedAtMs(row);
|
|
9999
10853
|
this.registerReadopted(
|
|
10000
10854
|
readoptConv,
|
|
@@ -12116,6 +12970,7 @@ function createCredentialSync({
|
|
|
12116
12970
|
|
|
12117
12971
|
// src/commands/run.ts
|
|
12118
12972
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
12973
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12119
12974
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12120
12975
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12121
12976
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12271,7 +13126,13 @@ function logActivity(state, entry) {
|
|
|
12271
13126
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12272
13127
|
if (!meetsThreshold(state, level)) return;
|
|
12273
13128
|
forwardRunnerActivity(
|
|
12274
|
-
{
|
|
13129
|
+
{
|
|
13130
|
+
level,
|
|
13131
|
+
message: entry.message,
|
|
13132
|
+
error: entry.error,
|
|
13133
|
+
metadata: entry.metadata,
|
|
13134
|
+
...entry.source ? { source: entry.source } : {}
|
|
13135
|
+
},
|
|
12275
13136
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12276
13137
|
);
|
|
12277
13138
|
const fullEntry = {
|
|
@@ -12293,7 +13154,7 @@ function logActivity(state, entry) {
|
|
|
12293
13154
|
}
|
|
12294
13155
|
function reportSessionDbRecovery(state) {
|
|
12295
13156
|
try {
|
|
12296
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13157
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12297
13158
|
for (const record of report.records) {
|
|
12298
13159
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12299
13160
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12328,7 +13189,9 @@ function displayStatus(state) {
|
|
|
12328
13189
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12329
13190
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12330
13191
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12331
|
-
const detail = last ? chalk7.dim(
|
|
13192
|
+
const detail = last ? chalk7.dim(
|
|
13193
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13194
|
+
) : "";
|
|
12332
13195
|
const agent = state.agentName ?? state.agentId;
|
|
12333
13196
|
console.log(
|
|
12334
13197
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12347,7 +13210,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12347
13210
|
logActivity(state, {
|
|
12348
13211
|
type: "info",
|
|
12349
13212
|
level: "warn",
|
|
12350
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13213
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12351
13214
|
});
|
|
12352
13215
|
}
|
|
12353
13216
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12372,7 +13235,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12372
13235
|
}
|
|
12373
13236
|
const result = await importOpenCodeV2Credentials({
|
|
12374
13237
|
request: client.request,
|
|
12375
|
-
homeDir:
|
|
13238
|
+
homeDir: homedir7(),
|
|
12376
13239
|
directory,
|
|
12377
13240
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12378
13241
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12580,9 +13443,9 @@ async function driveChannels(state, driver) {
|
|
|
12580
13443
|
}
|
|
12581
13444
|
}
|
|
12582
13445
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12583
|
-
var
|
|
13446
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12584
13447
|
function shouldWarnForReclaimSkip(reason) {
|
|
12585
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13448
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12586
13449
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12587
13450
|
if (!version2) return false;
|
|
12588
13451
|
const major = Number(version2[1]);
|
|
@@ -12591,7 +13454,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12591
13454
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12592
13455
|
}
|
|
12593
13456
|
function sessionDbPath() {
|
|
12594
|
-
return join11(
|
|
13457
|
+
return join11(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
13458
|
+
}
|
|
13459
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
13460
|
+
logActivity(state, {
|
|
13461
|
+
...entry,
|
|
13462
|
+
source: "cli.session-cleanup",
|
|
13463
|
+
metadata: { ...entry.metadata, phase }
|
|
13464
|
+
});
|
|
12595
13465
|
}
|
|
12596
13466
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12597
13467
|
const record = {
|
|
@@ -12632,69 +13502,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12632
13502
|
async function runSweep(state, driver, config) {
|
|
12633
13503
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12634
13504
|
try {
|
|
12635
|
-
const
|
|
12636
|
-
|
|
13505
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13506
|
+
const sessions = enumeration.sessions;
|
|
13507
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13508
|
+
if (enumerationReason !== null) {
|
|
12637
13509
|
logActivity(state, {
|
|
12638
13510
|
type: "info",
|
|
12639
|
-
|
|
13511
|
+
level: "warn",
|
|
13512
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12640
13513
|
});
|
|
12641
|
-
return;
|
|
12642
13514
|
}
|
|
12643
|
-
const toDelete = selectSessionsToDelete(
|
|
12644
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12645
|
-
{
|
|
12646
|
-
maxAgeMs: config.maxAgeMs,
|
|
12647
|
-
maxCount: config.maxCount,
|
|
12648
|
-
nowMs: Date.now(),
|
|
12649
|
-
protectedIds: driver.protectedSessionIds()
|
|
12650
|
-
}
|
|
12651
|
-
);
|
|
12652
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12653
13515
|
let deleted = 0;
|
|
12654
13516
|
let failed = 0;
|
|
13517
|
+
let failed4xx = 0;
|
|
13518
|
+
let failed5xx = 0;
|
|
13519
|
+
let failedOtherStatus = 0;
|
|
13520
|
+
let failedThrew = 0;
|
|
12655
13521
|
let skippedNewlyActive = 0;
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
12660
|
-
|
|
12661
|
-
|
|
12662
|
-
|
|
12663
|
-
|
|
13522
|
+
let remaining = 0;
|
|
13523
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13524
|
+
if (sessions !== null) {
|
|
13525
|
+
const toDelete = selectSessionsToDelete(
|
|
13526
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13527
|
+
{
|
|
13528
|
+
maxAgeMs: config.maxAgeMs,
|
|
13529
|
+
maxCount: config.maxCount,
|
|
13530
|
+
nowMs: Date.now(),
|
|
13531
|
+
protectedIds: protectedNow
|
|
13532
|
+
}
|
|
13533
|
+
);
|
|
13534
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13535
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13536
|
+
protectedNow = driver.protectedSessionIds();
|
|
13537
|
+
for (const id of deleteCandidates) {
|
|
13538
|
+
if (protectedNow.has(id)) {
|
|
13539
|
+
skippedNewlyActive++;
|
|
13540
|
+
logActivity(state, {
|
|
13541
|
+
type: "info",
|
|
13542
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13543
|
+
});
|
|
13544
|
+
continue;
|
|
13545
|
+
}
|
|
13546
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13547
|
+
if (outcome.ok) {
|
|
13548
|
+
deleted++;
|
|
13549
|
+
} else if (outcome.status === "threw") {
|
|
13550
|
+
failed++;
|
|
13551
|
+
failedThrew++;
|
|
13552
|
+
} else {
|
|
13553
|
+
failed++;
|
|
13554
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13555
|
+
if (statusClass === 4) failed4xx++;
|
|
13556
|
+
else if (statusClass === 5) failed5xx++;
|
|
13557
|
+
else failedOtherStatus++;
|
|
13558
|
+
}
|
|
12664
13559
|
}
|
|
12665
|
-
|
|
12666
|
-
|
|
13560
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13561
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13562
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13563
|
+
logActivity(state, {
|
|
13564
|
+
type: "info",
|
|
13565
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13566
|
+
});
|
|
12667
13567
|
}
|
|
12668
|
-
const
|
|
12669
|
-
|
|
12670
|
-
|
|
12671
|
-
|
|
12672
|
-
|
|
12673
|
-
|
|
13568
|
+
const sessionMetadata = {
|
|
13569
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13570
|
+
sessions_deleted: deleted,
|
|
13571
|
+
sessions_failed: failed,
|
|
13572
|
+
sessions_failed_4xx: failed4xx,
|
|
13573
|
+
sessions_failed_5xx: failed5xx,
|
|
13574
|
+
sessions_failed_other: failedOtherStatus,
|
|
13575
|
+
sessions_failed_threw: failedThrew,
|
|
13576
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13577
|
+
sessions_remaining: remaining,
|
|
13578
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13579
|
+
};
|
|
12674
13580
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12675
13581
|
dbPath: sessionDbPath(),
|
|
12676
|
-
|
|
13582
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13583
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13584
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12677
13585
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12678
13586
|
});
|
|
12679
13587
|
if (reclaimResult.ok) {
|
|
13588
|
+
state.sessionStoreHealth.observeWritable();
|
|
12680
13589
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12681
13590
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12682
13591
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12683
|
-
|
|
13592
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12684
13593
|
type: "info",
|
|
12685
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${
|
|
13594
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
13595
|
+
metadata: {
|
|
13596
|
+
outcome: "reclaimed",
|
|
13597
|
+
mode: reclaimResult.mode,
|
|
13598
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
13599
|
+
after_bytes: reclaimResult.afterBytes,
|
|
13600
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
13601
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
13602
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13603
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13604
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13605
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13606
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13607
|
+
...sessionMetadata
|
|
13608
|
+
}
|
|
12686
13609
|
});
|
|
12687
13610
|
} else {
|
|
12688
|
-
|
|
13611
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13612
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13613
|
+
}
|
|
13614
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12689
13615
|
type: "info",
|
|
12690
|
-
|
|
13616
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13617
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
13618
|
+
metadata: {
|
|
13619
|
+
outcome: "reclaim-skipped",
|
|
13620
|
+
reason: reclaimResult.skipped,
|
|
13621
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
13622
|
+
...sessionMetadata
|
|
13623
|
+
}
|
|
12691
13624
|
});
|
|
12692
13625
|
}
|
|
12693
13626
|
} catch (error2) {
|
|
12694
13627
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12695
|
-
|
|
13628
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12696
13629
|
type: "error",
|
|
12697
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
13630
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
13631
|
+
metadata: {
|
|
13632
|
+
outcome: "sweep-failed",
|
|
13633
|
+
reason: "sweep-error"
|
|
13634
|
+
}
|
|
12698
13635
|
});
|
|
12699
13636
|
}
|
|
12700
13637
|
}
|
|
@@ -12710,14 +13647,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12710
13647
|
for (const warning2 of config.warnings) {
|
|
12711
13648
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12712
13649
|
}
|
|
12713
|
-
const dbBytes = statSessionDbBytes(
|
|
13650
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12714
13651
|
void (async () => {
|
|
12715
13652
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12716
13653
|
if (reclaimAvailability !== null) {
|
|
12717
|
-
|
|
13654
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12718
13655
|
type: "info",
|
|
12719
13656
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12720
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
13657
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
13658
|
+
metadata: {
|
|
13659
|
+
outcome: "preflight-skipped",
|
|
13660
|
+
reason: reclaimAvailability.reason,
|
|
13661
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
13662
|
+
}
|
|
12721
13663
|
});
|
|
12722
13664
|
}
|
|
12723
13665
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12734,9 +13676,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12734
13676
|
);
|
|
12735
13677
|
});
|
|
12736
13678
|
if (!config.enabled) return;
|
|
12737
|
-
|
|
13679
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12738
13680
|
type: "info",
|
|
12739
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
13681
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
13682
|
+
metadata: {
|
|
13683
|
+
outcome: "armed",
|
|
13684
|
+
interval_ms: config.intervalMs,
|
|
13685
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
13686
|
+
max_count: config.maxCount ?? null
|
|
13687
|
+
}
|
|
12740
13688
|
});
|
|
12741
13689
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12742
13690
|
const firstSweep = setTimeout(
|
|
@@ -12915,7 +13863,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12915
13863
|
});
|
|
12916
13864
|
return;
|
|
12917
13865
|
}
|
|
12918
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
13866
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12919
13867
|
state.stopResourceUsageSampling = stop;
|
|
12920
13868
|
let consecutiveFailures = 0;
|
|
12921
13869
|
const tick = async () => {
|
|
@@ -12928,6 +13876,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12928
13876
|
message: `Resource usage collection: ${warning2}`
|
|
12929
13877
|
});
|
|
12930
13878
|
}
|
|
13879
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13880
|
+
...usage,
|
|
13881
|
+
nowMs: Date.now(),
|
|
13882
|
+
state: state.lowDiskWarningState
|
|
13883
|
+
});
|
|
13884
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13885
|
+
if (lowDiskDecision.warning !== null) {
|
|
13886
|
+
logActivity(state, {
|
|
13887
|
+
type: "info",
|
|
13888
|
+
level: "warn",
|
|
13889
|
+
message: lowDiskDecision.warning
|
|
13890
|
+
});
|
|
13891
|
+
}
|
|
12931
13892
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12932
13893
|
if (result.ok) {
|
|
12933
13894
|
if (consecutiveFailures > 0) {
|
|
@@ -13140,7 +14101,7 @@ async function run(options) {
|
|
|
13140
14101
|
let fileSyncDirectories;
|
|
13141
14102
|
try {
|
|
13142
14103
|
logLevel = resolveLogLevel(options);
|
|
13143
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14104
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13144
14105
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13145
14106
|
throw new Error(
|
|
13146
14107
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13174,6 +14135,7 @@ async function run(options) {
|
|
|
13174
14135
|
opencodeClient: null,
|
|
13175
14136
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13176
14137
|
sessionDbProvenanceAnomaly: false,
|
|
14138
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13177
14139
|
opencodeProcess: null,
|
|
13178
14140
|
stopOpenCodeLogTail: null,
|
|
13179
14141
|
litestreamProcess: null,
|
|
@@ -13191,10 +14153,12 @@ async function run(options) {
|
|
|
13191
14153
|
openaiUsageRearm: null,
|
|
13192
14154
|
resourceUsageTimer: null,
|
|
13193
14155
|
stopResourceUsageSampling: null,
|
|
14156
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13194
14157
|
credentialSync: null,
|
|
13195
14158
|
authHeader: ""
|
|
13196
14159
|
};
|
|
13197
14160
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14161
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13198
14162
|
if (options.credentialSyncMarker) {
|
|
13199
14163
|
state.credentialSync = createCredentialSync({
|
|
13200
14164
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13380,13 +14344,21 @@ async function run(options) {
|
|
|
13380
14344
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13381
14345
|
}
|
|
13382
14346
|
state.credentialSync?.arm();
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14347
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14348
|
+
() => ({
|
|
13386
14349
|
agentId: state.agentId,
|
|
13387
14350
|
authHeader: state.authHeader
|
|
13388
|
-
})
|
|
13389
|
-
|
|
14351
|
+
}),
|
|
14352
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14353
|
+
);
|
|
14354
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14355
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14356
|
+
openCodeActivityForwarder
|
|
14357
|
+
);
|
|
14358
|
+
state.stopOpenCodeLogTail = () => {
|
|
14359
|
+
openCodeActivityForwarder.flush();
|
|
14360
|
+
openCodeLogTail.stop();
|
|
14361
|
+
};
|
|
13390
14362
|
let sessionDbVerifyFatal = false;
|
|
13391
14363
|
if (!options.restoreSessionDb) {
|
|
13392
14364
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
@@ -13503,7 +14475,7 @@ async function run(options) {
|
|
|
13503
14475
|
const provenance = checkSessionDbProvenance({
|
|
13504
14476
|
dbPath: sessionDbPath(),
|
|
13505
14477
|
currentVersion: state.opencodeVersion,
|
|
13506
|
-
homeDir:
|
|
14478
|
+
homeDir: homedir7(),
|
|
13507
14479
|
env: process.env
|
|
13508
14480
|
});
|
|
13509
14481
|
if (provenance.anomaly) {
|
|
@@ -13670,10 +14642,11 @@ async function run(options) {
|
|
|
13670
14642
|
getAuthHeader: () => state.authHeader,
|
|
13671
14643
|
conversationFilter: state.conversationFilter,
|
|
13672
14644
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14645
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13673
14646
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13674
14647
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13675
14648
|
fileSyncDirectories,
|
|
13676
|
-
homeDir:
|
|
14649
|
+
homeDir: homedir7(),
|
|
13677
14650
|
maxActiveSessions,
|
|
13678
14651
|
log: (entry) => (
|
|
13679
14652
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13695,12 +14668,13 @@ async function run(options) {
|
|
|
13695
14668
|
openCodePassword: state.opencodePassword,
|
|
13696
14669
|
isRunning: () => state.running,
|
|
13697
14670
|
events: {
|
|
13698
|
-
onConnected: (agentId, isReconnect) => {
|
|
14671
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13699
14672
|
state.connected = true;
|
|
13700
14673
|
state.agentId = agentId;
|
|
14674
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13701
14675
|
logActivity(state, {
|
|
13702
14676
|
type: "info",
|
|
13703
|
-
message:
|
|
14677
|
+
message: connectionMessage
|
|
13704
14678
|
});
|
|
13705
14679
|
if (options.tunnelReadyFile) {
|
|
13706
14680
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13721,7 +14695,12 @@ async function run(options) {
|
|
|
13721
14695
|
version: state.opencodeVersion,
|
|
13722
14696
|
major: state.opencodeApiVersion,
|
|
13723
14697
|
connected: state.opencodeConnected
|
|
13724
|
-
})
|
|
14698
|
+
}),
|
|
14699
|
+
connection_generation: transition.generation,
|
|
14700
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14701
|
+
reconnect_attempt: transition.attempt,
|
|
14702
|
+
downtime_ms: transition.downtimeMs
|
|
14703
|
+
} : {}
|
|
13725
14704
|
});
|
|
13726
14705
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13727
14706
|
if (state.interactive) displayStatus(state);
|
|
@@ -13743,13 +14722,18 @@ async function run(options) {
|
|
|
13743
14722
|
if (state.interactive) displayStatus(state);
|
|
13744
14723
|
});
|
|
13745
14724
|
},
|
|
13746
|
-
onDisconnected: (code, reason) => {
|
|
14725
|
+
onDisconnected: (code, reason, transition) => {
|
|
13747
14726
|
state.connected = false;
|
|
14727
|
+
const safeReason = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13748
14728
|
logActivity(state, {
|
|
13749
14729
|
type: "info",
|
|
13750
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14730
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason})`
|
|
14731
|
+
});
|
|
14732
|
+
emitAgentDisconnected(state.agentId, {
|
|
14733
|
+
code,
|
|
14734
|
+
reason: safeReason,
|
|
14735
|
+
connection_generation: transition.generation
|
|
13751
14736
|
});
|
|
13752
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13753
14737
|
if (state.interactive) displayStatus(state);
|
|
13754
14738
|
},
|
|
13755
14739
|
onError: (error2) => {
|
|
@@ -13849,7 +14833,7 @@ async function run(options) {
|
|
|
13849
14833
|
},
|
|
13850
14834
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13851
14835
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13852
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14836
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13853
14837
|
firstDelayMs: firstReportDelayMs,
|
|
13854
14838
|
nextDelayMs: usageReportDelayMs,
|
|
13855
14839
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13892,7 +14876,7 @@ async function run(options) {
|
|
|
13892
14876
|
}
|
|
13893
14877
|
|
|
13894
14878
|
// src/index.ts
|
|
13895
|
-
var
|
|
14879
|
+
var version = getCliVersion();
|
|
13896
14880
|
var program = new Command();
|
|
13897
14881
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13898
14882
|
"--endpoint <url>",
|