@evident-ai/cli 3.5.1 → 3.5.2-dev.03d3dab
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1328 -404
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -1063,13 +1062,83 @@ async function claudeUsage() {
|
|
|
1063
1062
|
|
|
1064
1063
|
// src/commands/run.ts
|
|
1065
1064
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1065
|
+
import { homedir as homedir7 } from "node:os";
|
|
1067
1066
|
import { isAbsolute as isAbsolute3, join as join11, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1067
|
import chalk7 from "chalk";
|
|
1069
1068
|
|
|
1070
1069
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1070
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1071
|
|
|
1072
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1073
|
+
function roleOf(m) {
|
|
1074
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1075
|
+
if (typeof m.role === "string") return m.role;
|
|
1076
|
+
const infoRole = m.info?.role;
|
|
1077
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1078
|
+
}
|
|
1079
|
+
function idOf(m) {
|
|
1080
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1081
|
+
if (typeof m.id === "string") return m.id;
|
|
1082
|
+
const infoId = m.info?.id;
|
|
1083
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1084
|
+
}
|
|
1085
|
+
function parentIdOf(m) {
|
|
1086
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1087
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1088
|
+
const infoParent = m.info?.parentID;
|
|
1089
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1090
|
+
}
|
|
1091
|
+
function textPartsOf(m) {
|
|
1092
|
+
if (!m || typeof m !== "object") return [];
|
|
1093
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1094
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1095
|
+
}
|
|
1096
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1097
|
+
if (!messages || messages.length === 0) return [];
|
|
1098
|
+
const byParent = messages.filter(
|
|
1099
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1100
|
+
);
|
|
1101
|
+
if (byParent.length > 0) return byParent;
|
|
1102
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1103
|
+
if (userIndex === -1) return [];
|
|
1104
|
+
const run2 = [];
|
|
1105
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1106
|
+
const role = roleOf(messages[i]);
|
|
1107
|
+
if (role === "user") break;
|
|
1108
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1109
|
+
}
|
|
1110
|
+
return run2;
|
|
1111
|
+
}
|
|
1112
|
+
function lastText(assistantRun) {
|
|
1113
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1114
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1115
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1116
|
+
}
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1120
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1121
|
+
if (lastText(run2) !== null) return run2;
|
|
1122
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1123
|
+
if (userIndex === -1) return run2;
|
|
1124
|
+
let continuation;
|
|
1125
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1126
|
+
if (roleOf(messages[i]) === "user") {
|
|
1127
|
+
continuation = messages[i];
|
|
1128
|
+
break;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const continuationId = idOf(continuation);
|
|
1132
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1133
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1134
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1135
|
+
}
|
|
1136
|
+
return run2;
|
|
1137
|
+
}
|
|
1138
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1139
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1073
1142
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1143
|
var TelemetryEventTypes = {
|
|
1075
1144
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1227,7 @@ import ora4 from "ora";
|
|
|
1158
1227
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1228
|
|
|
1160
1229
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.
|
|
1230
|
+
var CLI_VERSION = (true ? "3.5.2-dev.03d3dab" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1231
|
function getCliVersion() {
|
|
1163
1232
|
return CLI_VERSION;
|
|
1164
1233
|
}
|
|
@@ -1172,6 +1241,23 @@ var authProvider = null;
|
|
|
1172
1241
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1242
|
authProvider = provider;
|
|
1174
1243
|
}
|
|
1244
|
+
var diagnosticsReporter = null;
|
|
1245
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1246
|
+
diagnosticsReporter = reporter;
|
|
1247
|
+
}
|
|
1248
|
+
function reportTelemetryDiagnostic(message) {
|
|
1249
|
+
if (!diagnosticsReporter) {
|
|
1250
|
+
console.error(message);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
try {
|
|
1254
|
+
diagnosticsReporter(message);
|
|
1255
|
+
} catch (error2) {
|
|
1256
|
+
console.error(
|
|
1257
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1175
1261
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1262
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1263
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1323,9 @@ async function flushEvents() {
|
|
|
1237
1323
|
signal: controller.signal
|
|
1238
1324
|
});
|
|
1239
1325
|
if (!response.ok) {
|
|
1240
|
-
|
|
1326
|
+
reportTelemetryDiagnostic(
|
|
1327
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1328
|
+
);
|
|
1241
1329
|
}
|
|
1242
1330
|
} finally {
|
|
1243
1331
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1335,9 @@ async function flushEvents() {
|
|
|
1247
1335
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1336
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1337
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1338
|
+
reportTelemetryDiagnostic(
|
|
1339
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1340
|
+
);
|
|
1251
1341
|
lastFlushFailureLoggedAt = now;
|
|
1252
1342
|
suppressedFlushFailureCount = 0;
|
|
1253
1343
|
} else {
|
|
@@ -1299,8 +1389,227 @@ var EventTypes = {
|
|
|
1299
1389
|
};
|
|
1300
1390
|
|
|
1301
1391
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
|
|
1392
|
+
import { homedir as homedir2 } from "node:os";
|
|
1393
|
+
|
|
1394
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1395
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1396
|
+
import { dirname as dirname2 } from "node:path";
|
|
1397
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1398
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1399
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1400
|
+
function errorMessage(error2) {
|
|
1401
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1402
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1403
|
+
}
|
|
1404
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1405
|
+
const messages = [];
|
|
1406
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1407
|
+
let current = error2;
|
|
1408
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1409
|
+
visited.add(current);
|
|
1410
|
+
if (typeof current === "string") {
|
|
1411
|
+
messages.push(current);
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
if (current instanceof Error) {
|
|
1415
|
+
messages.push(current.name, current.message);
|
|
1416
|
+
current = current.cause;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
if (typeof current === "object") {
|
|
1420
|
+
const record = current;
|
|
1421
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1422
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1423
|
+
current = record.cause;
|
|
1424
|
+
continue;
|
|
1425
|
+
}
|
|
1426
|
+
break;
|
|
1427
|
+
}
|
|
1428
|
+
return messages.some(
|
|
1429
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1430
|
+
message
|
|
1431
|
+
)
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1435
|
+
try {
|
|
1436
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1437
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1438
|
+
if (availableBytes < requiredBytes) {
|
|
1439
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1440
|
+
}
|
|
1441
|
+
return null;
|
|
1442
|
+
} catch (err) {
|
|
1443
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
function readLogicalBytes(db) {
|
|
1447
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1448
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1449
|
+
return pageCount * pageSize;
|
|
1450
|
+
}
|
|
1451
|
+
function readFreelistCount(db) {
|
|
1452
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1453
|
+
}
|
|
1454
|
+
function readCheckpointResult(db) {
|
|
1455
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1456
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1457
|
+
}
|
|
1458
|
+
async function probeReclaimAvailability(input) {
|
|
1459
|
+
const { dbPath, requiredBytes } = input;
|
|
1460
|
+
let sqlite;
|
|
1461
|
+
try {
|
|
1462
|
+
sqlite = await import("node:sqlite");
|
|
1463
|
+
} catch (err) {
|
|
1464
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1465
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1466
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1467
|
+
}
|
|
1468
|
+
let autoVacuum = null;
|
|
1469
|
+
try {
|
|
1470
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1471
|
+
try {
|
|
1472
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1473
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1474
|
+
} finally {
|
|
1475
|
+
db.close();
|
|
1476
|
+
}
|
|
1477
|
+
} catch (err) {
|
|
1478
|
+
console.warn(
|
|
1479
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1480
|
+
);
|
|
1481
|
+
}
|
|
1482
|
+
if (autoVacuum !== 0) return null;
|
|
1483
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1484
|
+
}
|
|
1485
|
+
async function reclaimSessionDbSpace(input) {
|
|
1486
|
+
const {
|
|
1487
|
+
dbPath,
|
|
1488
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1489
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1490
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1491
|
+
now = () => performance.now(),
|
|
1492
|
+
allowFullVacuum = true
|
|
1493
|
+
} = input;
|
|
1494
|
+
let sqlite;
|
|
1495
|
+
try {
|
|
1496
|
+
sqlite = await import("node:sqlite");
|
|
1497
|
+
} catch (err) {
|
|
1498
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1499
|
+
console.warn(
|
|
1500
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1501
|
+
);
|
|
1502
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1503
|
+
}
|
|
1504
|
+
const { DatabaseSync } = sqlite;
|
|
1505
|
+
let db;
|
|
1506
|
+
try {
|
|
1507
|
+
db = new DatabaseSync(dbPath);
|
|
1508
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1509
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1510
|
+
if (autoVacuum === 0) {
|
|
1511
|
+
if (!allowFullVacuum) {
|
|
1512
|
+
console.warn(
|
|
1513
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1514
|
+
);
|
|
1515
|
+
return {
|
|
1516
|
+
ok: false,
|
|
1517
|
+
skipped: "full-vacuum-blocked",
|
|
1518
|
+
detail: "a session turn is live"
|
|
1519
|
+
};
|
|
1520
|
+
}
|
|
1521
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1522
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1523
|
+
if (skipReason !== null) {
|
|
1524
|
+
console.warn(
|
|
1525
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1526
|
+
);
|
|
1527
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1528
|
+
}
|
|
1529
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1530
|
+
const freelistBefore = readFreelistCount(db);
|
|
1531
|
+
const startedAt = now();
|
|
1532
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1533
|
+
db.exec("VACUUM");
|
|
1534
|
+
const afterBytes = readLogicalBytes(db);
|
|
1535
|
+
const freelistAfter = readFreelistCount(db);
|
|
1536
|
+
const checkpoint = readCheckpointResult(db);
|
|
1537
|
+
return {
|
|
1538
|
+
ok: true,
|
|
1539
|
+
mode: "convert",
|
|
1540
|
+
beforeBytes,
|
|
1541
|
+
afterBytes,
|
|
1542
|
+
freelistBefore,
|
|
1543
|
+
freelistAfter,
|
|
1544
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1545
|
+
stoppedBy: "freelist-drained",
|
|
1546
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1547
|
+
checkpoint
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
if (autoVacuum === 2) {
|
|
1551
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1552
|
+
const freelistBefore = readFreelistCount(db);
|
|
1553
|
+
const startedAt = now();
|
|
1554
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1555
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1556
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1557
|
+
let pagesAttempted = 0;
|
|
1558
|
+
let freelistAfter = freelistBefore;
|
|
1559
|
+
let stoppedBy = "freelist-drained";
|
|
1560
|
+
while (freelistAfter > 0) {
|
|
1561
|
+
if (pagesAttempted >= pageLimit) {
|
|
1562
|
+
stoppedBy = "page-ceiling";
|
|
1563
|
+
break;
|
|
1564
|
+
}
|
|
1565
|
+
if (now() - startedAt >= budget) {
|
|
1566
|
+
stoppedBy = "time-budget";
|
|
1567
|
+
break;
|
|
1568
|
+
}
|
|
1569
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1570
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1571
|
+
pagesAttempted += pages;
|
|
1572
|
+
freelistAfter = readFreelistCount(db);
|
|
1573
|
+
}
|
|
1574
|
+
const afterBytes = readLogicalBytes(db);
|
|
1575
|
+
const checkpoint = readCheckpointResult(db);
|
|
1576
|
+
return {
|
|
1577
|
+
ok: true,
|
|
1578
|
+
mode: "incremental",
|
|
1579
|
+
beforeBytes,
|
|
1580
|
+
afterBytes,
|
|
1581
|
+
freelistBefore,
|
|
1582
|
+
freelistAfter,
|
|
1583
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1584
|
+
stoppedBy,
|
|
1585
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1586
|
+
checkpoint
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
console.warn(
|
|
1590
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1591
|
+
);
|
|
1592
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1595
|
+
return {
|
|
1596
|
+
ok: false,
|
|
1597
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1598
|
+
detail: errorMessage(err)
|
|
1599
|
+
};
|
|
1600
|
+
} finally {
|
|
1601
|
+
db?.close();
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1606
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1607
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1608
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1609
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
1610
|
+
};
|
|
1303
1611
|
var SEVERITY_BY_LEVEL = {
|
|
1612
|
+
info: "info",
|
|
1304
1613
|
warn: "warning",
|
|
1305
1614
|
error: "error"
|
|
1306
1615
|
};
|
|
@@ -1310,26 +1619,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
1619
|
if (!levelMatch) return null;
|
|
1311
1620
|
const level = levelMatch[1].toUpperCase();
|
|
1312
1621
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
1622
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1623
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1624
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1625
|
+
const parsed = {
|
|
1626
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1627
|
+
sessionID: sessionMatch?.[1],
|
|
1628
|
+
messageID: messageIDMatch?.[1]
|
|
1629
|
+
};
|
|
1630
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1631
|
+
if (message !== void 0) parsed.message = message;
|
|
1632
|
+
return parsed;
|
|
1315
1633
|
}
|
|
1316
1634
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
1635
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1636
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1318
1637
|
let buffer = Buffer.alloc(0);
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1638
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1639
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1640
|
+
const metadata = {
|
|
1641
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1642
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1643
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1644
|
+
};
|
|
1322
1645
|
forwardRunnerActivity(
|
|
1323
1646
|
{
|
|
1324
1647
|
level: parsed.level,
|
|
1325
1648
|
error: line,
|
|
1326
|
-
metadata:
|
|
1649
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1327
1650
|
source: "opencode"
|
|
1328
1651
|
},
|
|
1329
1652
|
getContext()
|
|
1330
1653
|
);
|
|
1331
1654
|
};
|
|
1332
|
-
|
|
1655
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1656
|
+
for (const [key, state] of repeatStates) {
|
|
1657
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1658
|
+
continue;
|
|
1659
|
+
}
|
|
1660
|
+
if (state.repeatsSuppressed > 0) {
|
|
1661
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1662
|
+
}
|
|
1663
|
+
repeatStates.delete(key);
|
|
1664
|
+
}
|
|
1665
|
+
};
|
|
1666
|
+
const flushLine = (line) => {
|
|
1667
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1668
|
+
try {
|
|
1669
|
+
onSessionStoreWriteFailure();
|
|
1670
|
+
} catch (err) {
|
|
1671
|
+
reportTelemetryDiagnostic(
|
|
1672
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1677
|
+
if (!parsed) return;
|
|
1678
|
+
const now = Date.now();
|
|
1679
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1680
|
+
expireRepeatStates(now, key);
|
|
1681
|
+
if (key === void 0) {
|
|
1682
|
+
forwardLine(line, parsed);
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
const previous = repeatStates.get(key);
|
|
1686
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1687
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1688
|
+
repeatStates.set(key, {
|
|
1689
|
+
line,
|
|
1690
|
+
parsed,
|
|
1691
|
+
lastForwardedAt: now,
|
|
1692
|
+
repeatsSuppressed: 0
|
|
1693
|
+
});
|
|
1694
|
+
return;
|
|
1695
|
+
}
|
|
1696
|
+
previous.line = line;
|
|
1697
|
+
previous.parsed = parsed;
|
|
1698
|
+
previous.repeatsSuppressed++;
|
|
1699
|
+
};
|
|
1700
|
+
const forward = ((chunk) => {
|
|
1333
1701
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
1702
|
let newlineIndex;
|
|
1335
1703
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,14 +1708,44 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
1708
|
flushLine(buffer.toString("utf-8"));
|
|
1341
1709
|
buffer = Buffer.alloc(0);
|
|
1342
1710
|
}
|
|
1711
|
+
});
|
|
1712
|
+
forward.flush = () => {
|
|
1713
|
+
for (const state of repeatStates.values()) {
|
|
1714
|
+
if (state.repeatsSuppressed > 0) {
|
|
1715
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
repeatStates.clear();
|
|
1343
1719
|
};
|
|
1720
|
+
return forward;
|
|
1344
1721
|
}
|
|
1345
1722
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
1723
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
1724
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
1725
|
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1726
|
+
function redactHomeDirectory(message) {
|
|
1727
|
+
const home = homedir2();
|
|
1728
|
+
if (!home || home === "/") {
|
|
1729
|
+
return message;
|
|
1730
|
+
}
|
|
1731
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1732
|
+
let result = "";
|
|
1733
|
+
let searchFrom = 0;
|
|
1734
|
+
let homeIndex;
|
|
1735
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1736
|
+
const homeEnd = homeIndex + home.length;
|
|
1737
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1738
|
+
const nextCharacter = message[homeEnd];
|
|
1739
|
+
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1740
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1741
|
+
result += atPathBoundary ? "~" : home;
|
|
1742
|
+
searchFrom = homeEnd;
|
|
1743
|
+
}
|
|
1744
|
+
return result + message.slice(searchFrom);
|
|
1745
|
+
}
|
|
1746
|
+
function redactDiagnosticText(message) {
|
|
1747
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1748
|
+
return redactHomeDirectory(redacted);
|
|
1351
1749
|
}
|
|
1352
1750
|
function truncate(message) {
|
|
1353
1751
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1357,14 +1755,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1357
1755
|
if (!metadata) return {};
|
|
1358
1756
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
1757
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
1758
|
+
reportTelemetryDiagnostic(
|
|
1759
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
1760
|
);
|
|
1363
1761
|
}
|
|
1364
1762
|
const sanitised = [];
|
|
1365
1763
|
for (const [key, value] of entries) {
|
|
1366
1764
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
1765
|
+
sanitised.push([
|
|
1766
|
+
key,
|
|
1767
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1768
|
+
]);
|
|
1368
1769
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
1770
|
sanitised.push([key, value]);
|
|
1370
1771
|
}
|
|
@@ -1382,8 +1783,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
1783
|
}
|
|
1383
1784
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
1785
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
1786
|
+
reportTelemetryDiagnostic(
|
|
1787
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
1788
|
);
|
|
1388
1789
|
}
|
|
1389
1790
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +1794,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
1794
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
1795
|
window.windowDroppedCount++;
|
|
1395
1796
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
1797
|
+
reportTelemetryDiagnostic(
|
|
1798
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
1799
|
);
|
|
1399
1800
|
}
|
|
1400
1801
|
return false;
|
|
@@ -1404,12 +1805,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
1805
|
}
|
|
1405
1806
|
function forwardRunnerActivity(entry, context) {
|
|
1406
1807
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
1808
|
const source = entry.source ?? "cli.run";
|
|
1809
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
1810
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
1811
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
1812
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
1813
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
1814
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
1815
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
1816
|
message,
|
|
@@ -1417,8 +1818,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
1818
|
agentId: context.agentId
|
|
1418
1819
|
});
|
|
1419
1820
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
1821
|
+
reportTelemetryDiagnostic(
|
|
1822
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
1823
|
);
|
|
1423
1824
|
}
|
|
1424
1825
|
}
|
|
@@ -1708,8 +2109,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1708
2109
|
|
|
1709
2110
|
// src/lib/http-timeout.ts
|
|
1710
2111
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2112
|
+
var RequestTimeoutError = class extends Error {
|
|
2113
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2114
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2115
|
+
this.operation = operation;
|
|
2116
|
+
this.timeoutMs = timeoutMs;
|
|
2117
|
+
this.elapsedMs = elapsedMs;
|
|
2118
|
+
this.name = "TimeoutError";
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
function describeRequestOperation(input, init) {
|
|
2122
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2123
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2124
|
+
let path;
|
|
2125
|
+
try {
|
|
2126
|
+
path = new URL(rawUrl).pathname;
|
|
2127
|
+
} catch (error2) {
|
|
2128
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2129
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2130
|
+
}
|
|
2131
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2132
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2133
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2134
|
+
return `${method} ${maskedPath}`;
|
|
2135
|
+
}
|
|
1711
2136
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2137
|
+
return (async (input, init) => {
|
|
2138
|
+
const startedAt = Date.now();
|
|
2139
|
+
try {
|
|
2140
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2141
|
+
} catch (error2) {
|
|
2142
|
+
if (error2?.name === "TimeoutError") {
|
|
2143
|
+
throw new RequestTimeoutError(
|
|
2144
|
+
describeRequestOperation(input, init),
|
|
2145
|
+
timeoutMs,
|
|
2146
|
+
Date.now() - startedAt
|
|
2147
|
+
);
|
|
2148
|
+
}
|
|
2149
|
+
throw error2;
|
|
2150
|
+
}
|
|
2151
|
+
});
|
|
1713
2152
|
}
|
|
1714
2153
|
|
|
1715
2154
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2214,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2214
|
return null;
|
|
1776
2215
|
}
|
|
1777
2216
|
}
|
|
1778
|
-
function
|
|
2217
|
+
function roleOf2(m) {
|
|
1779
2218
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2219
|
if (typeof m.role === "string") return m.role;
|
|
1781
2220
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2228,13 @@ function createdOf(m) {
|
|
|
1789
2228
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2229
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2230
|
}
|
|
1792
|
-
function
|
|
2231
|
+
function idOf2(m) {
|
|
1793
2232
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2233
|
if (typeof m.id === "string") return m.id;
|
|
1795
2234
|
const infoId = m.info?.id;
|
|
1796
2235
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2236
|
}
|
|
1798
|
-
function
|
|
2237
|
+
function parentIdOf2(m) {
|
|
1799
2238
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2239
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2240
|
const infoParent = m.info?.parentID;
|
|
@@ -1860,9 +2299,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2299
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2300
|
if (!messages || messages.length === 0) return false;
|
|
1862
2301
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2302
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2303
|
return completedOf(last) == null;
|
|
1865
2304
|
}
|
|
2305
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2306
|
function sessionLastActivityMs(session) {
|
|
1867
2307
|
const candidates = [
|
|
1868
2308
|
session.time?.updated,
|
|
@@ -1877,30 +2317,60 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2317
|
}
|
|
1878
2318
|
return null;
|
|
1879
2319
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2320
|
+
function sessionListPath(options) {
|
|
2321
|
+
const params = new URLSearchParams();
|
|
2322
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2323
|
+
const query = params.toString();
|
|
2324
|
+
return query ? `/session?${query}` : "/session";
|
|
2325
|
+
}
|
|
2326
|
+
async function listSessionPage(port, client, options) {
|
|
2327
|
+
const path = sessionListPath(options);
|
|
1882
2328
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2329
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2330
|
if (!res.ok) return null;
|
|
1885
2331
|
const body = await res.json();
|
|
1886
2332
|
return Array.isArray(body) ? body : null;
|
|
1887
2333
|
} catch (error2) {
|
|
1888
2334
|
console.error(
|
|
1889
|
-
`[
|
|
2335
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2336
|
);
|
|
1891
2337
|
return null;
|
|
1892
2338
|
}
|
|
1893
2339
|
}
|
|
2340
|
+
async function listSessions(port, client, options) {
|
|
2341
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2342
|
+
return listSessionPage(port, client, options);
|
|
2343
|
+
}
|
|
2344
|
+
async function listAllSessions(port, client) {
|
|
2345
|
+
if (client?.version === "v2") {
|
|
2346
|
+
const sessions2 = await listV2Sessions(client);
|
|
2347
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2348
|
+
}
|
|
2349
|
+
const sessions = await listSessionPage(port, client, {
|
|
2350
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2351
|
+
});
|
|
2352
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2353
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2354
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2355
|
+
}
|
|
2356
|
+
return { sessions };
|
|
2357
|
+
}
|
|
1894
2358
|
async function deleteSession(port, id, client) {
|
|
1895
2359
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2360
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2361
|
+
const res = await requestWithClient(
|
|
2362
|
+
port,
|
|
2363
|
+
client,
|
|
2364
|
+
`/session/${id}`,
|
|
2365
|
+
{ method: "DELETE" },
|
|
2366
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2367
|
+
);
|
|
2368
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1899
2369
|
} catch (error2) {
|
|
1900
2370
|
console.error(
|
|
1901
2371
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2372
|
);
|
|
1903
|
-
return false;
|
|
2373
|
+
return { ok: false, status: "threw" };
|
|
1904
2374
|
}
|
|
1905
2375
|
}
|
|
1906
2376
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +2608,33 @@ function messageText(m) {
|
|
|
2138
2608
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
2609
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
2610
|
}
|
|
2141
|
-
|
|
2611
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2612
|
+
if (!messages || messages.length === 0) return null;
|
|
2613
|
+
let best = null;
|
|
2614
|
+
for (const message of messages) {
|
|
2615
|
+
if (roleOf2(message) !== "user") continue;
|
|
2616
|
+
const id = idOf2(message);
|
|
2617
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2618
|
+
if (messageText(message) !== content) continue;
|
|
2619
|
+
const created = createdOf(message) ?? 0;
|
|
2620
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2621
|
+
}
|
|
2622
|
+
return best?.id ?? null;
|
|
2623
|
+
}
|
|
2624
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, onKnownUserIds) {
|
|
2142
2625
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
2626
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
2627
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
2628
|
);
|
|
2629
|
+
if (onKnownUserIds) {
|
|
2630
|
+
try {
|
|
2631
|
+
onKnownUserIds(before === null ? null : knownUserIds);
|
|
2632
|
+
} catch (error2) {
|
|
2633
|
+
console.error(
|
|
2634
|
+
`[sendPromptAsync] pre-dispatch snapshot callback failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2635
|
+
);
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2146
2638
|
const parts = [{ type: "text", text: content }];
|
|
2147
2639
|
let pendingOutcomes = null;
|
|
2148
2640
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,22 +2673,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
2673
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
2674
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
2675
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
2676
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2677
|
+
if (dispatchedUserMessageId !== null) {
|
|
2678
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2679
|
+
return dispatchedUserMessageId;
|
|
2200
2680
|
}
|
|
2201
2681
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
2682
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2207,13 +2687,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
2687
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
2688
|
if (!messages || messages.length === 0) return null;
|
|
2209
2689
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
2690
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
2691
|
);
|
|
2212
2692
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
2693
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
2694
|
if (userIndex === -1) return null;
|
|
2215
2695
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
2696
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
2697
|
}
|
|
2218
2698
|
return null;
|
|
2219
2699
|
}
|
|
@@ -2223,7 +2703,7 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2223
2703
|
let lastNonErrored = null;
|
|
2224
2704
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
2705
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2706
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2227
2707
|
if (lastCorrelated === null) lastCorrelated = m;
|
|
2228
2708
|
if (errorOf(m) == null) {
|
|
2229
2709
|
lastNonErrored = m;
|
|
@@ -2231,12 +2711,12 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2231
2711
|
}
|
|
2232
2712
|
}
|
|
2233
2713
|
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2234
|
-
const userIndex = messages.findIndex((m) =>
|
|
2714
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
2715
|
if (userIndex === -1) return null;
|
|
2236
2716
|
let last = null;
|
|
2237
2717
|
let lastOk = null;
|
|
2238
2718
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
2719
|
+
const role = roleOf2(messages[i]);
|
|
2240
2720
|
if (role === "user") break;
|
|
2241
2721
|
if (role === "assistant") {
|
|
2242
2722
|
last = messages[i];
|
|
@@ -2248,16 +2728,16 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2248
2728
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
2729
|
if (!messages || messages.length === 0) return [];
|
|
2250
2730
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
2731
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
2732
|
);
|
|
2253
2733
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
2734
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
2735
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
2736
|
if (userIndex === -1) return [];
|
|
2257
2737
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
2738
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
2739
|
+
if (roleOf2(message) === "user") break;
|
|
2740
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
2741
|
}
|
|
2262
2742
|
}
|
|
2263
2743
|
const refs = [];
|
|
@@ -2297,7 +2777,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
2777
|
if (!messages || messages.length === 0) return [];
|
|
2298
2778
|
const calls = [];
|
|
2299
2779
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
2780
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
2781
|
for (const part of message.parts ?? []) {
|
|
2302
2782
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
2783
|
continue;
|
|
@@ -2327,7 +2807,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
2807
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
2808
|
const unattributed = [];
|
|
2329
2809
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
2810
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
2811
|
const created = finiteNumber(createdOf(message));
|
|
2332
2812
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
2813
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2404,7 +2884,7 @@ function sumAssistantUsage(messages) {
|
|
|
2404
2884
|
function messageUsage(messages, userMessageId) {
|
|
2405
2885
|
if (!messages || messages.length === 0) return null;
|
|
2406
2886
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
2887
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
2888
|
);
|
|
2409
2889
|
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
2890
|
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
@@ -2420,7 +2900,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
2900
|
}
|
|
2421
2901
|
function messageRunState(messages, userMessageId) {
|
|
2422
2902
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
2903
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
2904
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
2905
|
if (!hasUser) {
|
|
2426
2906
|
if (!reply) return "unknown";
|
|
@@ -2512,7 +2992,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
2992
|
if (!messages || messages.length === 0) return null;
|
|
2513
2993
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
2994
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
2995
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
2996
|
const created = createdOf(message);
|
|
2517
2997
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
2998
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3024,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3024
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3025
|
if (!messages || messages.length === 0) return false;
|
|
2546
3026
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3027
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3028
|
);
|
|
2549
3029
|
}
|
|
2550
3030
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3031
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3032
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3033
|
if (userIndex === -1) return false;
|
|
2554
3034
|
let hasLaterUser = false;
|
|
2555
3035
|
let hasStartedLaterUser = false;
|
|
2556
3036
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3037
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3038
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3039
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3040
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3041
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3042
|
return false;
|
|
2563
3043
|
}
|
|
2564
3044
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3045
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3046
|
)) {
|
|
2567
3047
|
hasStartedLaterUser = true;
|
|
2568
3048
|
}
|
|
@@ -3033,13 +3513,17 @@ async function listV2Sessions(client) {
|
|
|
3033
3513
|
}
|
|
3034
3514
|
async function deleteV2Session(client, sessionId) {
|
|
3035
3515
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
3516
|
+
const response = await client.request(
|
|
3517
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3518
|
+
{ method: "DELETE" },
|
|
3519
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3520
|
+
);
|
|
3521
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3038
3522
|
} catch (error2) {
|
|
3039
3523
|
console.error(
|
|
3040
3524
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
3525
|
);
|
|
3042
|
-
return false;
|
|
3526
|
+
return { ok: false, status: "threw" };
|
|
3043
3527
|
}
|
|
3044
3528
|
}
|
|
3045
3529
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3214,7 +3698,7 @@ import { join as join4 } from "node:path";
|
|
|
3214
3698
|
|
|
3215
3699
|
// src/lib/openai-usage.ts
|
|
3216
3700
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3701
|
+
import { homedir as homedir3 } from "node:os";
|
|
3218
3702
|
import { join as join3 } from "node:path";
|
|
3219
3703
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
3704
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
@@ -3229,7 +3713,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
3713
|
}
|
|
3230
3714
|
function readOpenCodeChatGptCredentials() {
|
|
3231
3715
|
try {
|
|
3232
|
-
const raw = readFileSync3(join3(
|
|
3716
|
+
const raw = readFileSync3(join3(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
3717
|
let parsed;
|
|
3234
3718
|
try {
|
|
3235
3719
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +3854,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
3854
|
].some((name) => headers.has(name));
|
|
3371
3855
|
}
|
|
3372
3856
|
async function getOpenAiUsage(port, client) {
|
|
3857
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
3858
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
3859
|
if (!credentials2) {
|
|
3375
3860
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3861
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
3862
|
"no_credentials"
|
|
3378
3863
|
);
|
|
3379
3864
|
}
|
|
3380
3865
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
3866
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3867
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
3868
|
"credentials_expired"
|
|
3384
3869
|
);
|
|
3385
3870
|
}
|
|
@@ -3660,9 +4145,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4145
|
|
|
3661
4146
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4147
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4148
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4149
|
+
import { homedir as homedir4 } from "node:os";
|
|
4150
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3666
4151
|
|
|
3667
4152
|
// src/lib/runner-synchroniser.ts
|
|
3668
4153
|
import { spawn } from "node:child_process";
|
|
@@ -3787,7 +4272,7 @@ function clearMarker(options) {
|
|
|
3787
4272
|
function markNoReplicate(options, message) {
|
|
3788
4273
|
if (options.noReplicateMarker) {
|
|
3789
4274
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4275
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3791
4276
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4277
|
} catch (error2) {
|
|
3793
4278
|
options.log(
|
|
@@ -3912,7 +4397,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4397
|
return null;
|
|
3913
4398
|
}
|
|
3914
4399
|
try {
|
|
3915
|
-
if (
|
|
4400
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4401
|
} catch (error2) {
|
|
3917
4402
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4403
|
options.log(
|
|
@@ -3942,7 +4427,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
4427
|
return null;
|
|
3943
4428
|
}
|
|
3944
4429
|
try {
|
|
3945
|
-
mkdirSync(
|
|
4430
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3946
4431
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
4432
|
} catch (error2) {
|
|
3948
4433
|
options.log(
|
|
@@ -4017,14 +4502,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
4502
|
case 0:
|
|
4018
4503
|
return;
|
|
4019
4504
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4505
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
4506
|
options.log(
|
|
4022
4507
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
4508
|
"warn"
|
|
4024
4509
|
);
|
|
4025
4510
|
return;
|
|
4026
4511
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4512
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
4513
|
discardSessionDbDebris(options);
|
|
4029
4514
|
markNoReplicate(
|
|
4030
4515
|
options,
|
|
@@ -4103,7 +4588,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4103
4588
|
}
|
|
4104
4589
|
function fileExists(path) {
|
|
4105
4590
|
try {
|
|
4106
|
-
|
|
4591
|
+
statSync3(path);
|
|
4107
4592
|
return true;
|
|
4108
4593
|
} catch (error2) {
|
|
4109
4594
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +4598,7 @@ function fileExists(path) {
|
|
|
4113
4598
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
4599
|
const env = options.env ?? process.env;
|
|
4115
4600
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4601
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
4602
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
4603
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
4604
|
env
|
|
@@ -4192,7 +4677,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4192
4677
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
4678
|
import { createRequire } from "node:module";
|
|
4194
4679
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
4680
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4196
4681
|
var require2 = createRequire(import.meta.url);
|
|
4197
4682
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
4683
|
let db;
|
|
@@ -4258,7 +4743,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
4743
|
}
|
|
4259
4744
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
4745
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
4746
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4262
4747
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
4748
|
`, "utf8");
|
|
4264
4749
|
} catch (error2) {
|
|
@@ -4493,12 +4978,15 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4493
4978
|
}
|
|
4494
4979
|
} catch {
|
|
4495
4980
|
try {
|
|
4496
|
-
const psOutput = execSync(
|
|
4981
|
+
const psOutput = execSync("ps aux", {
|
|
4497
4982
|
encoding: "utf-8",
|
|
4498
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
4983
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
4984
|
+
// The whole table reaches Node; the 1 MiB default would surface a spurious failure.
|
|
4985
|
+
maxBuffer: 16 * 1024 * 1024
|
|
4499
4986
|
}).trim();
|
|
4500
4987
|
if (psOutput) {
|
|
4501
4988
|
for (const line of psOutput.split("\n")) {
|
|
4989
|
+
if (!psPattern.test(line)) continue;
|
|
4502
4990
|
const parts = line.trim().split(/\s+/);
|
|
4503
4991
|
if (parts.length >= 2) {
|
|
4504
4992
|
const pid = parseInt(parts[1], 10);
|
|
@@ -4507,8 +4995,9 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4507
4995
|
}
|
|
4508
4996
|
}
|
|
4509
4997
|
} catch (err) {
|
|
4998
|
+
const reason = (typeof err === "object" && err !== null && "stderr" in err && typeof err.stderr === "string" ? err.stderr : "").split("\n").map((line) => line.trim()).find(Boolean) ?? (err instanceof Error ? err.message : String(err)).split("\n").map((line) => line.trim()).find(Boolean) ?? "no error detail available";
|
|
4510
4999
|
console.warn(
|
|
4511
|
-
`
|
|
5000
|
+
`Could not list running processes, so an already-running OpenCode may not be detected: ${reason}`
|
|
4512
5001
|
);
|
|
4513
5002
|
}
|
|
4514
5003
|
}
|
|
@@ -4540,7 +5029,7 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4540
5029
|
return instances;
|
|
4541
5030
|
}
|
|
4542
5031
|
function findOpenCodeProcesses() {
|
|
4543
|
-
return findProcessesByPattern("opencode serve|opencode-serve",
|
|
5032
|
+
return findProcessesByPattern("opencode serve|opencode-serve", /opencode (serve|--port)/);
|
|
4544
5033
|
}
|
|
4545
5034
|
async function scanPortsForOpenCode() {
|
|
4546
5035
|
const instances = [];
|
|
@@ -4835,7 +5324,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4835
5324
|
toDelete.push(s.id);
|
|
4836
5325
|
}
|
|
4837
5326
|
}
|
|
4838
|
-
|
|
5327
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5328
|
+
return toDelete.sort(
|
|
5329
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5330
|
+
);
|
|
4839
5331
|
}
|
|
4840
5332
|
var DEFAULT_INTERVAL = "1h";
|
|
4841
5333
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4895,13 +5387,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4895
5387
|
}
|
|
4896
5388
|
|
|
4897
5389
|
// src/lib/opencode/session-db-size.ts
|
|
4898
|
-
import { statSync as
|
|
5390
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4899
5391
|
import { join as join6 } from "node:path";
|
|
4900
5392
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4901
5393
|
function statSessionDbBytes(homeDir) {
|
|
4902
5394
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4903
5395
|
try {
|
|
4904
|
-
return
|
|
5396
|
+
return statSync4(dbPath).size;
|
|
4905
5397
|
} catch (err) {
|
|
4906
5398
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4907
5399
|
if (!isMissingFile) {
|
|
@@ -4927,12 +5419,12 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4927
5419
|
}
|
|
4928
5420
|
|
|
4929
5421
|
// src/lib/opencode/log-tail.ts
|
|
4930
|
-
import { statSync as
|
|
4931
|
-
import { homedir as
|
|
5422
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5423
|
+
import { homedir as homedir5 } from "node:os";
|
|
4932
5424
|
import { join as join7 } from "node:path";
|
|
4933
5425
|
import { open as open2, stat } from "node:fs/promises";
|
|
4934
5426
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4935
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
5427
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
4936
5428
|
const dataDir = env.XDG_DATA_HOME || join7(homeDir, ".local", "share");
|
|
4937
5429
|
return join7(dataDir, "opencode", "log", "opencode.log");
|
|
4938
5430
|
}
|
|
@@ -4950,7 +5442,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4950
5442
|
let inode = null;
|
|
4951
5443
|
let baselineReady = true;
|
|
4952
5444
|
try {
|
|
4953
|
-
const initial =
|
|
5445
|
+
const initial = statSync5(logPath);
|
|
4954
5446
|
offset = initial.size;
|
|
4955
5447
|
inode = initial.ino;
|
|
4956
5448
|
} catch (error2) {
|
|
@@ -5009,125 +5501,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5009
5501
|
};
|
|
5010
5502
|
}
|
|
5011
5503
|
|
|
5012
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5013
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5014
|
-
import { dirname as dirname4 } from "node:path";
|
|
5015
|
-
function errorMessage(error2) {
|
|
5016
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5017
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5018
|
-
}
|
|
5019
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5020
|
-
try {
|
|
5021
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5022
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5023
|
-
if (availableBytes < requiredBytes) {
|
|
5024
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5025
|
-
}
|
|
5026
|
-
return null;
|
|
5027
|
-
} catch (err) {
|
|
5028
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5029
|
-
}
|
|
5030
|
-
}
|
|
5031
|
-
function readLogicalBytes(db) {
|
|
5032
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5033
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5034
|
-
return pageCount * pageSize;
|
|
5035
|
-
}
|
|
5036
|
-
function readCheckpointResult(db) {
|
|
5037
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5038
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5039
|
-
}
|
|
5040
|
-
async function probeReclaimAvailability(input) {
|
|
5041
|
-
const { dbPath, requiredBytes } = input;
|
|
5042
|
-
let sqlite;
|
|
5043
|
-
try {
|
|
5044
|
-
sqlite = await import("node:sqlite");
|
|
5045
|
-
} catch (err) {
|
|
5046
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5047
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5048
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5049
|
-
}
|
|
5050
|
-
let autoVacuum = null;
|
|
5051
|
-
try {
|
|
5052
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5053
|
-
try {
|
|
5054
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5055
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5056
|
-
} finally {
|
|
5057
|
-
db.close();
|
|
5058
|
-
}
|
|
5059
|
-
} catch (err) {
|
|
5060
|
-
console.warn(
|
|
5061
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5062
|
-
);
|
|
5063
|
-
}
|
|
5064
|
-
if (autoVacuum !== 0) return null;
|
|
5065
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5066
|
-
}
|
|
5067
|
-
async function reclaimSessionDbSpace(input) {
|
|
5068
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5069
|
-
let sqlite;
|
|
5070
|
-
try {
|
|
5071
|
-
sqlite = await import("node:sqlite");
|
|
5072
|
-
} catch (err) {
|
|
5073
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5074
|
-
console.warn(
|
|
5075
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5076
|
-
);
|
|
5077
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5078
|
-
}
|
|
5079
|
-
const { DatabaseSync } = sqlite;
|
|
5080
|
-
let db;
|
|
5081
|
-
try {
|
|
5082
|
-
db = new DatabaseSync(dbPath);
|
|
5083
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5084
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5085
|
-
if (autoVacuum === 0) {
|
|
5086
|
-
if (!allowFullVacuum) {
|
|
5087
|
-
console.warn(
|
|
5088
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5089
|
-
);
|
|
5090
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5091
|
-
}
|
|
5092
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5093
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5094
|
-
if (skipReason !== null) {
|
|
5095
|
-
console.warn(
|
|
5096
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5097
|
-
);
|
|
5098
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5099
|
-
}
|
|
5100
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5101
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5102
|
-
db.exec("VACUUM");
|
|
5103
|
-
const afterBytes = readLogicalBytes(db);
|
|
5104
|
-
const checkpoint = readCheckpointResult(db);
|
|
5105
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5106
|
-
}
|
|
5107
|
-
if (autoVacuum === 2) {
|
|
5108
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5109
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5110
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5111
|
-
const afterBytes = readLogicalBytes(db);
|
|
5112
|
-
const checkpoint = readCheckpointResult(db);
|
|
5113
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5114
|
-
}
|
|
5115
|
-
console.warn(
|
|
5116
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5117
|
-
);
|
|
5118
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5119
|
-
} catch (err) {
|
|
5120
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5121
|
-
return {
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "reclaim-error",
|
|
5124
|
-
detail: errorMessage(err)
|
|
5125
|
-
};
|
|
5126
|
-
} finally {
|
|
5127
|
-
db?.close();
|
|
5128
|
-
}
|
|
5129
|
-
}
|
|
5130
|
-
|
|
5131
5504
|
// src/lib/tunnel/connection.ts
|
|
5132
5505
|
import WebSocket2 from "ws";
|
|
5133
5506
|
|
|
@@ -5357,6 +5730,8 @@ var StreamForwarder = class {
|
|
|
5357
5730
|
if (done) break;
|
|
5358
5731
|
const chunk = Buffer.from(value);
|
|
5359
5732
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5733
|
+
await stream.pausePromise;
|
|
5734
|
+
if (ac.signal.aborted) break;
|
|
5360
5735
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5361
5736
|
if (this.binaryFramesSupported) {
|
|
5362
5737
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5399,6 +5774,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5399
5774
|
}
|
|
5400
5775
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5401
5776
|
var BASE_RECONNECT_DELAY = 500;
|
|
5777
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5402
5778
|
function getReconnectDelay(attempt) {
|
|
5403
5779
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5404
5780
|
const jitter = Math.random() * 1e3;
|
|
@@ -5470,10 +5846,27 @@ function connectTunnel(options) {
|
|
|
5470
5846
|
ws.close();
|
|
5471
5847
|
reject(new Error("Connection timeout"));
|
|
5472
5848
|
}, 3e4);
|
|
5849
|
+
let relaySilenceTimeout;
|
|
5850
|
+
let relaySilenceTimedOut = false;
|
|
5851
|
+
const clearRelaySilenceTimeout = () => {
|
|
5852
|
+
if (relaySilenceTimeout === void 0) return;
|
|
5853
|
+
clearTimeout(relaySilenceTimeout);
|
|
5854
|
+
relaySilenceTimeout = void 0;
|
|
5855
|
+
};
|
|
5856
|
+
const armRelaySilenceTimeout = () => {
|
|
5857
|
+
clearRelaySilenceTimeout();
|
|
5858
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
5859
|
+
relaySilenceTimeout = void 0;
|
|
5860
|
+
relaySilenceTimedOut = true;
|
|
5861
|
+
ws.terminate();
|
|
5862
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
5863
|
+
relaySilenceTimeout.unref?.();
|
|
5864
|
+
};
|
|
5473
5865
|
let upgradeRejection = null;
|
|
5474
5866
|
let upgradeRejectionReason = null;
|
|
5475
5867
|
ws.on("unexpected-response", (_req, res) => {
|
|
5476
5868
|
clearTimeout(connectionTimeout);
|
|
5869
|
+
clearRelaySilenceTimeout();
|
|
5477
5870
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5478
5871
|
upgradeRejectionReason = reason;
|
|
5479
5872
|
const chunks = [];
|
|
@@ -5503,6 +5896,7 @@ function connectTunnel(options) {
|
|
|
5503
5896
|
onInfo?.("WebSocket connection established");
|
|
5504
5897
|
});
|
|
5505
5898
|
ws.on("message", (data, isBinary) => {
|
|
5899
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5506
5900
|
if (isBinary) {
|
|
5507
5901
|
try {
|
|
5508
5902
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5536,6 +5930,7 @@ function connectTunnel(options) {
|
|
|
5536
5930
|
switch (message.type) {
|
|
5537
5931
|
case "connected": {
|
|
5538
5932
|
clearTimeout(connectionTimeout);
|
|
5933
|
+
armRelaySilenceTimeout();
|
|
5539
5934
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5540
5935
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5541
5936
|
onConnected?.(connectedAgentId);
|
|
@@ -5560,6 +5955,7 @@ function connectTunnel(options) {
|
|
|
5560
5955
|
});
|
|
5561
5956
|
ws.on("error", (error2) => {
|
|
5562
5957
|
clearTimeout(connectionTimeout);
|
|
5958
|
+
clearRelaySilenceTimeout();
|
|
5563
5959
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5564
5960
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5565
5961
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5571,7 +5967,8 @@ function connectTunnel(options) {
|
|
|
5571
5967
|
);
|
|
5572
5968
|
});
|
|
5573
5969
|
ws.on("close", (code, reason) => {
|
|
5574
|
-
|
|
5970
|
+
clearRelaySilenceTimeout();
|
|
5971
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5575
5972
|
forwarder.abortAll();
|
|
5576
5973
|
onDisconnected?.(code, reasonStr);
|
|
5577
5974
|
});
|
|
@@ -5579,9 +5976,10 @@ function connectTunnel(options) {
|
|
|
5579
5976
|
}
|
|
5580
5977
|
|
|
5581
5978
|
// src/lib/tunnel/runner-connection.ts
|
|
5582
|
-
var RunnerConnection = class {
|
|
5979
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5583
5980
|
opts;
|
|
5584
5981
|
sleep;
|
|
5982
|
+
now;
|
|
5585
5983
|
connection = null;
|
|
5586
5984
|
resolvedAgentId;
|
|
5587
5985
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5590,10 +5988,17 @@ var RunnerConnection = class {
|
|
|
5590
5988
|
reconnectPromise = null;
|
|
5591
5989
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5592
5990
|
reconnectAttempt = 0;
|
|
5991
|
+
generation = 0;
|
|
5992
|
+
disconnectedAtMs = null;
|
|
5993
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
5994
|
+
"No pong received",
|
|
5995
|
+
"Orphaned socket"
|
|
5996
|
+
]);
|
|
5593
5997
|
constructor(opts) {
|
|
5594
5998
|
this.opts = opts;
|
|
5595
5999
|
this.resolvedAgentId = opts.agentId;
|
|
5596
6000
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6001
|
+
this.now = opts.now ?? Date.now;
|
|
5597
6002
|
}
|
|
5598
6003
|
get agentId() {
|
|
5599
6004
|
return this.resolvedAgentId;
|
|
@@ -5629,14 +6034,24 @@ var RunnerConnection = class {
|
|
|
5629
6034
|
port: this.opts.port,
|
|
5630
6035
|
openCodePassword: this.opts.openCodePassword,
|
|
5631
6036
|
onConnected: (agentId) => {
|
|
6037
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5632
6038
|
this.reconnectAttempt = 0;
|
|
5633
6039
|
this.reconnecting = false;
|
|
5634
6040
|
this.resolvedAgentId = agentId;
|
|
5635
|
-
|
|
6041
|
+
this.generation++;
|
|
6042
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6043
|
+
this.disconnectedAtMs = null;
|
|
6044
|
+
events.onConnected(agentId, isReconnect, {
|
|
6045
|
+
generation: this.generation,
|
|
6046
|
+
attempt,
|
|
6047
|
+
downtimeMs
|
|
6048
|
+
});
|
|
5636
6049
|
},
|
|
5637
6050
|
onDisconnected: (code, reason) => {
|
|
5638
|
-
|
|
5639
|
-
|
|
6051
|
+
this.disconnectedAtMs = this.now();
|
|
6052
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6053
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6054
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5640
6055
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5641
6056
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5642
6057
|
});
|
|
@@ -5975,8 +6390,85 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5975
6390
|
};
|
|
5976
6391
|
}
|
|
5977
6392
|
|
|
6393
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6394
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6395
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6396
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6397
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6398
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6399
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6400
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6401
|
+
var BAND_RANK = {
|
|
6402
|
+
low: 1,
|
|
6403
|
+
very_low: 2,
|
|
6404
|
+
critical: 3
|
|
6405
|
+
};
|
|
6406
|
+
function bandFor(freeBytes) {
|
|
6407
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6408
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6409
|
+
return "low";
|
|
6410
|
+
}
|
|
6411
|
+
function formatBytes(bytes) {
|
|
6412
|
+
if (bytes === null) return "unavailable";
|
|
6413
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6414
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6415
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6416
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6417
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6418
|
+
}
|
|
6419
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6420
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6421
|
+
}
|
|
6422
|
+
function evaluateLowDiskWarning(input) {
|
|
6423
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6424
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6425
|
+
return { warning: null, state };
|
|
6426
|
+
}
|
|
6427
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6428
|
+
if (recovered) {
|
|
6429
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6430
|
+
}
|
|
6431
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6432
|
+
if (!low) return { warning: null, state };
|
|
6433
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6434
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6435
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6436
|
+
const worse = currentRank > previousRank;
|
|
6437
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6438
|
+
const nextState = {
|
|
6439
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6440
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6441
|
+
};
|
|
6442
|
+
return {
|
|
6443
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6444
|
+
state: nextState
|
|
6445
|
+
};
|
|
6446
|
+
}
|
|
6447
|
+
|
|
6448
|
+
// src/lib/opencode/session-store-health.ts
|
|
6449
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6450
|
+
function createSessionStoreHealth(options = {}) {
|
|
6451
|
+
const now = options.now ?? (() => Date.now());
|
|
6452
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6453
|
+
let lastUnwritableAt = null;
|
|
6454
|
+
let lastWritableAt = null;
|
|
6455
|
+
return {
|
|
6456
|
+
observeUnwritable: () => {
|
|
6457
|
+
lastUnwritableAt = now();
|
|
6458
|
+
},
|
|
6459
|
+
observeWritable: () => {
|
|
6460
|
+
lastWritableAt = now();
|
|
6461
|
+
},
|
|
6462
|
+
isUnwritable: () => {
|
|
6463
|
+
if (lastUnwritableAt === null) return false;
|
|
6464
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6465
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6466
|
+
}
|
|
6467
|
+
};
|
|
6468
|
+
}
|
|
6469
|
+
|
|
5978
6470
|
// src/lib/channels/driver.ts
|
|
5979
|
-
import { homedir as
|
|
6471
|
+
import { homedir as homedir6 } from "node:os";
|
|
5980
6472
|
|
|
5981
6473
|
// src/lib/runner-file-sync.ts
|
|
5982
6474
|
import { join as join9 } from "node:path";
|
|
@@ -6433,6 +6925,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6433
6925
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6434
6926
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6435
6927
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6928
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6436
6929
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6437
6930
|
var HEARTBEAT_MS = 6e4;
|
|
6438
6931
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6494,6 +6987,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6494
6987
|
maxActiveSessions;
|
|
6495
6988
|
watcherStallMs;
|
|
6496
6989
|
wedgeWarningIntervalMs;
|
|
6990
|
+
sessionStoreHealth;
|
|
6497
6991
|
openCodeClient;
|
|
6498
6992
|
/** Cache of conversationId → opencode sessionId. */
|
|
6499
6993
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6622,6 +7116,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6622
7116
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6623
7117
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6624
7118
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7119
|
+
/**
|
|
7120
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7121
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7122
|
+
* identical-text turn, so it never produces a correlation.
|
|
7123
|
+
*/
|
|
7124
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6625
7125
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6626
7126
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6627
7127
|
/**
|
|
@@ -6707,21 +7207,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6707
7207
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6708
7208
|
*/
|
|
6709
7209
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6710
|
-
/**
|
|
6711
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6712
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6713
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6714
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6715
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6716
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6717
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6718
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6719
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6720
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6721
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6722
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6723
|
-
*/
|
|
6724
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6725
7210
|
/**
|
|
6726
7211
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6727
7212
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6739,8 +7224,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6739
7224
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6740
7225
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6741
7226
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6742
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6743
|
-
*
|
|
7227
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7228
|
+
* the terminal-report fence and is never sent again).
|
|
6744
7229
|
*/
|
|
6745
7230
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6746
7231
|
/**
|
|
@@ -6859,10 +7344,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6859
7344
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6860
7345
|
this.now = config.now ?? (() => Date.now());
|
|
6861
7346
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6862
|
-
this.homeDir = config.homeDir ??
|
|
7347
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6863
7348
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6864
7349
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6865
7350
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7351
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6866
7352
|
}
|
|
6867
7353
|
get isV2() {
|
|
6868
7354
|
return this.openCodeClient.version === "v2";
|
|
@@ -6878,7 +7364,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6878
7364
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6879
7365
|
}
|
|
6880
7366
|
async listSessions() {
|
|
6881
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7367
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7368
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7369
|
+
});
|
|
6882
7370
|
}
|
|
6883
7371
|
async sessionExists(sessionId) {
|
|
6884
7372
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7055,6 +7543,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7055
7543
|
}
|
|
7056
7544
|
return ids;
|
|
7057
7545
|
}
|
|
7546
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7547
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7548
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7549
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7550
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7551
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7552
|
+
}
|
|
7553
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7554
|
+
}
|
|
7555
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7556
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7557
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7558
|
+
this.awaitingReadopt.delete(messageId);
|
|
7559
|
+
this.sessions.delete(conversationId);
|
|
7560
|
+
this.supersede(conversationId, sessionId);
|
|
7561
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7562
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7563
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7564
|
+
this.log({
|
|
7565
|
+
level: "error",
|
|
7566
|
+
message: errorMessage3,
|
|
7567
|
+
conversation_id: conversationId,
|
|
7568
|
+
message_id: messageId
|
|
7569
|
+
});
|
|
7570
|
+
const reportError = await this.reportUntrackableAck(
|
|
7571
|
+
conversationId,
|
|
7572
|
+
messageId,
|
|
7573
|
+
null,
|
|
7574
|
+
errorMessage3
|
|
7575
|
+
);
|
|
7576
|
+
if (reportError !== null) {
|
|
7577
|
+
this.log({
|
|
7578
|
+
level: "warn",
|
|
7579
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7580
|
+
conversation_id: conversationId,
|
|
7581
|
+
message_id: messageId
|
|
7582
|
+
});
|
|
7583
|
+
}
|
|
7584
|
+
if (source === "readopt") {
|
|
7585
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7586
|
+
}
|
|
7587
|
+
}
|
|
7058
7588
|
/**
|
|
7059
7589
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7060
7590
|
*
|
|
@@ -7234,6 +7764,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7234
7764
|
}
|
|
7235
7765
|
break;
|
|
7236
7766
|
}
|
|
7767
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7768
|
+
if (correlation) {
|
|
7769
|
+
correlation.attempts += 1;
|
|
7770
|
+
let correlatedId = null;
|
|
7771
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7772
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7773
|
+
conv,
|
|
7774
|
+
message,
|
|
7775
|
+
correlation.sessionId
|
|
7776
|
+
);
|
|
7777
|
+
if (polled.ok)
|
|
7778
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7779
|
+
} else if (correlation.knownUserIds === null) {
|
|
7780
|
+
this.log({
|
|
7781
|
+
level: "debug",
|
|
7782
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7783
|
+
conversation_id: conv.id,
|
|
7784
|
+
message_id: message.id
|
|
7785
|
+
});
|
|
7786
|
+
}
|
|
7787
|
+
if (correlatedId !== null) {
|
|
7788
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7789
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7790
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7791
|
+
this.dispatched.add(message.id);
|
|
7792
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7793
|
+
dispatched += 1;
|
|
7794
|
+
this.log({
|
|
7795
|
+
level: "info",
|
|
7796
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7797
|
+
conversation_id: conv.id,
|
|
7798
|
+
message_id: message.id
|
|
7799
|
+
});
|
|
7800
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7801
|
+
opencode_message_id: correlatedId
|
|
7802
|
+
});
|
|
7803
|
+
continue;
|
|
7804
|
+
}
|
|
7805
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7806
|
+
await this.handoffUntrackableCorrelation(
|
|
7807
|
+
conv.id,
|
|
7808
|
+
message.id,
|
|
7809
|
+
correlation.sessionId,
|
|
7810
|
+
correlation.attempts,
|
|
7811
|
+
"fresh"
|
|
7812
|
+
);
|
|
7813
|
+
break;
|
|
7814
|
+
}
|
|
7815
|
+
break;
|
|
7816
|
+
}
|
|
7237
7817
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7238
7818
|
if (effectiveOpencodeMessageId) {
|
|
7239
7819
|
const outcome = await this.resolveRedrive(
|
|
@@ -7255,6 +7835,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7255
7835
|
model: message.opencode_model ?? void 0
|
|
7256
7836
|
};
|
|
7257
7837
|
let opencodeMessageId;
|
|
7838
|
+
let preDispatchUserIds = null;
|
|
7258
7839
|
try {
|
|
7259
7840
|
this.log({
|
|
7260
7841
|
level: "info",
|
|
@@ -7284,7 +7865,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7284
7865
|
message.content,
|
|
7285
7866
|
options,
|
|
7286
7867
|
sendAttachments,
|
|
7287
|
-
this.openCodeClient
|
|
7868
|
+
this.openCodeClient,
|
|
7869
|
+
(ids) => {
|
|
7870
|
+
preDispatchUserIds = ids;
|
|
7871
|
+
}
|
|
7288
7872
|
)
|
|
7289
7873
|
);
|
|
7290
7874
|
} catch (err) {
|
|
@@ -7367,39 +7951,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7367
7951
|
if (this.isV2) {
|
|
7368
7952
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7369
7953
|
}
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
conversation_id: conv.id,
|
|
7376
|
-
message_id: message.id
|
|
7377
|
-
});
|
|
7378
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7379
|
-
continue;
|
|
7380
|
-
}
|
|
7381
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7382
|
-
this.sessions.delete(conv.id);
|
|
7383
|
-
this.supersede(conv.id, sessionId);
|
|
7384
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
7385
|
-
this.log({
|
|
7386
|
-
level: "error",
|
|
7387
|
-
message: errorMessage3,
|
|
7388
|
-
conversation_id: conv.id,
|
|
7389
|
-
message_id: message.id
|
|
7954
|
+
this.awaitingCorrelation.set(message.id, {
|
|
7955
|
+
sessionId,
|
|
7956
|
+
content: message.content,
|
|
7957
|
+
attempts: 0,
|
|
7958
|
+
knownUserIds: preDispatchUserIds
|
|
7390
7959
|
});
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
|
|
7394
|
-
|
|
7395
|
-
|
|
7396
|
-
message_id: message.id
|
|
7397
|
-
});
|
|
7398
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7960
|
+
this.log({
|
|
7961
|
+
level: "warn",
|
|
7962
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7963
|
+
conversation_id: conv.id,
|
|
7964
|
+
message_id: message.id
|
|
7399
7965
|
});
|
|
7966
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7400
7967
|
break;
|
|
7401
7968
|
}
|
|
7402
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7403
7969
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7404
7970
|
this.releasedOpencodeIds.delete(message.id);
|
|
7405
7971
|
this.dispatched.add(message.id);
|
|
@@ -8004,23 +8570,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8004
8570
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8005
8571
|
return 1;
|
|
8006
8572
|
}
|
|
8007
|
-
/**
|
|
8008
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8009
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8010
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8011
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8012
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8013
|
-
* count, since a new session is a genuinely different attempt.
|
|
8014
|
-
*/
|
|
8015
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8016
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8017
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8018
|
-
existing.count += 1;
|
|
8019
|
-
return existing.count;
|
|
8020
|
-
}
|
|
8021
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8022
|
-
return 1;
|
|
8023
|
-
}
|
|
8024
8573
|
/** Record one queued give-up toward the never-started bound. */
|
|
8025
8574
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8026
8575
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8332,8 +8881,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8332
8881
|
b2LastDescendantCheckMs: 0,
|
|
8333
8882
|
b2RootOngoingHeldLogged: false,
|
|
8334
8883
|
b2AbandonedSignalled: false,
|
|
8884
|
+
b2AbandonedWithoutAnswer: false,
|
|
8335
8885
|
ambiguousPinnedSinceMs: 0,
|
|
8336
8886
|
ambiguousResolved: false,
|
|
8887
|
+
sessionStoreFailurePending: false,
|
|
8337
8888
|
neverStartedResolved: false
|
|
8338
8889
|
});
|
|
8339
8890
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8450,8 +9001,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8450
9001
|
b2LastDescendantCheckMs: 0,
|
|
8451
9002
|
b2RootOngoingHeldLogged: false,
|
|
8452
9003
|
b2AbandonedSignalled: false,
|
|
9004
|
+
b2AbandonedWithoutAnswer: false,
|
|
8453
9005
|
ambiguousPinnedSinceMs: 0,
|
|
8454
9006
|
ambiguousResolved: false,
|
|
9007
|
+
sessionStoreFailurePending: false,
|
|
8455
9008
|
neverStartedResolved: false
|
|
8456
9009
|
});
|
|
8457
9010
|
}
|
|
@@ -8865,6 +9418,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8865
9418
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8866
9419
|
this.neverStartedAttempts.delete(id);
|
|
8867
9420
|
}
|
|
9421
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9422
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9423
|
+
return;
|
|
9424
|
+
}
|
|
8868
9425
|
if (inFlight.neverStartedResolved) {
|
|
8869
9426
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
9427
|
inFlight.neverStartedResolved = false;
|
|
@@ -9024,10 +9581,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9024
9581
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9025
9582
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9026
9583
|
inFlight.b2AbandonedSignalled = false;
|
|
9584
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9027
9585
|
}
|
|
9028
9586
|
} else {
|
|
9029
9587
|
if (inFlight.b2AbandonedSignalled) {
|
|
9030
|
-
|
|
9588
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9589
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9590
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9591
|
+
sessionId,
|
|
9592
|
+
watcher,
|
|
9593
|
+
inFlight,
|
|
9594
|
+
messages,
|
|
9595
|
+
reply?.info?.finish ?? reply?.finish
|
|
9596
|
+
);
|
|
9597
|
+
} else {
|
|
9598
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9599
|
+
}
|
|
9031
9600
|
return;
|
|
9032
9601
|
}
|
|
9033
9602
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9044,17 +9613,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9044
9613
|
descendantOngoing,
|
|
9045
9614
|
rootOngoing
|
|
9046
9615
|
})) {
|
|
9616
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9617
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9618
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9619
|
+
watched_for_ms: pinnedForMs,
|
|
9620
|
+
run_state: "failed",
|
|
9621
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9622
|
+
});
|
|
9623
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9624
|
+
return;
|
|
9625
|
+
}
|
|
9047
9626
|
inFlight.b2AbandonedSignalled = true;
|
|
9627
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9628
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9629
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9630
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9631
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9632
|
+
watched_for_ms: pinnedForMs,
|
|
9633
|
+
run_state: "failed",
|
|
9634
|
+
finish,
|
|
9635
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9636
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9637
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9638
|
+
});
|
|
9639
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9640
|
+
sessionId,
|
|
9641
|
+
watcher,
|
|
9642
|
+
inFlight,
|
|
9643
|
+
messages,
|
|
9644
|
+
finish
|
|
9645
|
+
);
|
|
9646
|
+
return;
|
|
9647
|
+
}
|
|
9048
9648
|
this.log({
|
|
9049
9649
|
level: "warn",
|
|
9050
9650
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9051
9651
|
conversation_id: conv.id,
|
|
9052
9652
|
message_id: id
|
|
9053
9653
|
});
|
|
9054
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9055
9654
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9056
9655
|
watched_for_ms: pinnedForMs,
|
|
9057
|
-
finish
|
|
9656
|
+
finish,
|
|
9058
9657
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9059
9658
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9060
9659
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9102,6 +9701,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9102
9701
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9103
9702
|
sessionOngoing: ongoing
|
|
9104
9703
|
})) {
|
|
9704
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9705
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9706
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9707
|
+
watched_for_ms: pinnedForMs,
|
|
9708
|
+
run_state: "failed",
|
|
9709
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9710
|
+
});
|
|
9711
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9712
|
+
return;
|
|
9713
|
+
}
|
|
9105
9714
|
inFlight.ambiguousResolved = true;
|
|
9106
9715
|
this.log({
|
|
9107
9716
|
level: "warn",
|
|
@@ -9326,6 +9935,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9326
9935
|
}
|
|
9327
9936
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9328
9937
|
}
|
|
9938
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
9939
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
9940
|
+
const conv = watcher.conv;
|
|
9941
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
9942
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
9943
|
+
this.log({
|
|
9944
|
+
level: "warn",
|
|
9945
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
9946
|
+
conversation_id: conv.id,
|
|
9947
|
+
message_id: inFlight.evidentMessageId
|
|
9948
|
+
});
|
|
9949
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
9950
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
9951
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
9952
|
+
messages,
|
|
9953
|
+
inFlight.opencodeMessageId,
|
|
9954
|
+
inFlight.evidentMessageId
|
|
9955
|
+
);
|
|
9956
|
+
try {
|
|
9957
|
+
await this.markFailed(
|
|
9958
|
+
conv.id,
|
|
9959
|
+
inFlight.evidentMessageId,
|
|
9960
|
+
sessionId,
|
|
9961
|
+
error2,
|
|
9962
|
+
usage,
|
|
9963
|
+
void 0,
|
|
9964
|
+
usageAgentName,
|
|
9965
|
+
subagentInvocations
|
|
9966
|
+
);
|
|
9967
|
+
} catch (err) {
|
|
9968
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
9969
|
+
if (err instanceof ChannelTerminalError) {
|
|
9970
|
+
this.log({
|
|
9971
|
+
level: "warn",
|
|
9972
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9973
|
+
conversation_id: conv.id,
|
|
9974
|
+
message_id: inFlight.evidentMessageId
|
|
9975
|
+
});
|
|
9976
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9977
|
+
return;
|
|
9978
|
+
}
|
|
9979
|
+
if (this.now() >= inFlight.deadline) {
|
|
9980
|
+
this.log({
|
|
9981
|
+
level: "warn",
|
|
9982
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
9983
|
+
conversation_id: conv.id,
|
|
9984
|
+
message_id: inFlight.evidentMessageId
|
|
9985
|
+
});
|
|
9986
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9987
|
+
return;
|
|
9988
|
+
}
|
|
9989
|
+
this.log({
|
|
9990
|
+
level: "warn",
|
|
9991
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
9992
|
+
conversation_id: conv.id,
|
|
9993
|
+
message_id: inFlight.evidentMessageId
|
|
9994
|
+
});
|
|
9995
|
+
return;
|
|
9996
|
+
}
|
|
9997
|
+
inFlight.done = true;
|
|
9998
|
+
await this.reportSubagentAuthFailures(
|
|
9999
|
+
watcher.conv.id,
|
|
10000
|
+
inFlight.opencodeMessageId,
|
|
10001
|
+
inFlight.evidentMessageId,
|
|
10002
|
+
messages
|
|
10003
|
+
);
|
|
10004
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10005
|
+
}
|
|
9329
10006
|
/**
|
|
9330
10007
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9331
10008
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9381,6 +10058,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9381
10058
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9382
10059
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9383
10060
|
}
|
|
10061
|
+
/**
|
|
10062
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10063
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10064
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10065
|
+
*/
|
|
10066
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10067
|
+
const conv = watcher.conv;
|
|
10068
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10069
|
+
this.log({
|
|
10070
|
+
level: "error",
|
|
10071
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10072
|
+
conversation_id: conv.id,
|
|
10073
|
+
message_id: inFlight.evidentMessageId
|
|
10074
|
+
});
|
|
10075
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10076
|
+
try {
|
|
10077
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10078
|
+
} catch (err) {
|
|
10079
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10080
|
+
if (err instanceof ChannelTerminalError) {
|
|
10081
|
+
this.log({
|
|
10082
|
+
level: "warn",
|
|
10083
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10084
|
+
conversation_id: conv.id,
|
|
10085
|
+
message_id: inFlight.evidentMessageId
|
|
10086
|
+
});
|
|
10087
|
+
this.recordReleasedOpencodeId(
|
|
10088
|
+
inFlight.evidentMessageId,
|
|
10089
|
+
sessionId,
|
|
10090
|
+
inFlight.opencodeMessageId
|
|
10091
|
+
);
|
|
10092
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10093
|
+
return;
|
|
10094
|
+
}
|
|
10095
|
+
if (this.now() >= inFlight.deadline) {
|
|
10096
|
+
this.log({
|
|
10097
|
+
level: "warn",
|
|
10098
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10099
|
+
conversation_id: conv.id,
|
|
10100
|
+
message_id: inFlight.evidentMessageId
|
|
10101
|
+
});
|
|
10102
|
+
this.recordReleasedOpencodeId(
|
|
10103
|
+
inFlight.evidentMessageId,
|
|
10104
|
+
sessionId,
|
|
10105
|
+
inFlight.opencodeMessageId
|
|
10106
|
+
);
|
|
10107
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10108
|
+
return;
|
|
10109
|
+
}
|
|
10110
|
+
this.log({
|
|
10111
|
+
level: "warn",
|
|
10112
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10113
|
+
conversation_id: conv.id,
|
|
10114
|
+
message_id: inFlight.evidentMessageId
|
|
10115
|
+
});
|
|
10116
|
+
return;
|
|
10117
|
+
}
|
|
10118
|
+
inFlight.done = true;
|
|
10119
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10120
|
+
}
|
|
9384
10121
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9385
10122
|
/**
|
|
9386
10123
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9419,8 +10156,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9419
10156
|
}
|
|
9420
10157
|
}
|
|
9421
10158
|
}
|
|
9422
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10159
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9423
10160
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10161
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10162
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10163
|
+
this.awaitingCorrelation.delete(id);
|
|
10164
|
+
this.log({
|
|
10165
|
+
level: "debug",
|
|
10166
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10167
|
+
message_id: id
|
|
10168
|
+
});
|
|
10169
|
+
}
|
|
10170
|
+
}
|
|
9424
10171
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9425
10172
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9426
10173
|
this.untrackableAck.delete(id);
|
|
@@ -9463,7 +10210,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9463
10210
|
bySession.set(row.opencode_session_id, list);
|
|
9464
10211
|
}
|
|
9465
10212
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9466
|
-
let messages;
|
|
10213
|
+
let messages = null;
|
|
9467
10214
|
try {
|
|
9468
10215
|
if (this.isV2) {
|
|
9469
10216
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9472,9 +10219,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9472
10219
|
level: "warn",
|
|
9473
10220
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9474
10221
|
});
|
|
9475
|
-
|
|
10222
|
+
} else {
|
|
10223
|
+
messages = snapshot;
|
|
9476
10224
|
}
|
|
9477
|
-
messages = snapshot;
|
|
9478
10225
|
} else {
|
|
9479
10226
|
const polled = await pollSessionMessagesForRedrive(
|
|
9480
10227
|
this.port,
|
|
@@ -9487,15 +10234,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9487
10234
|
level: "warn",
|
|
9488
10235
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9489
10236
|
});
|
|
9490
|
-
|
|
10237
|
+
} else {
|
|
10238
|
+
messages = polled.messages;
|
|
9491
10239
|
}
|
|
9492
|
-
messages = polled.messages;
|
|
9493
10240
|
}
|
|
9494
10241
|
} catch (err) {
|
|
9495
10242
|
this.log({
|
|
9496
10243
|
level: "warn",
|
|
9497
10244
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9498
10245
|
});
|
|
10246
|
+
}
|
|
10247
|
+
if (messages === null) {
|
|
10248
|
+
for (const row of sessionRows) {
|
|
10249
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10250
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10251
|
+
}
|
|
10252
|
+
}
|
|
9499
10253
|
continue;
|
|
9500
10254
|
}
|
|
9501
10255
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9505,6 +10259,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9505
10259
|
}
|
|
9506
10260
|
}
|
|
9507
10261
|
}
|
|
10262
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10263
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10264
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10265
|
+
if (!correlation) return false;
|
|
10266
|
+
correlation.attempts += 1;
|
|
10267
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10268
|
+
if (correlatedId !== null) {
|
|
10269
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10270
|
+
const conv = this.convForRow(sessionId, row);
|
|
10271
|
+
const message = this.queuedMessageForRow(row);
|
|
10272
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10273
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10274
|
+
this.dispatched.add(row.id);
|
|
10275
|
+
this.readopted.add(row.id);
|
|
10276
|
+
this.ensureWatcherRunning(sessionId);
|
|
10277
|
+
this.log({
|
|
10278
|
+
level: "info",
|
|
10279
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10280
|
+
conversation_id: row.conversation_id,
|
|
10281
|
+
message_id: row.id
|
|
10282
|
+
});
|
|
10283
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10284
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10285
|
+
opencode_message_id: correlatedId
|
|
10286
|
+
});
|
|
10287
|
+
return true;
|
|
10288
|
+
}
|
|
10289
|
+
if (correlation.knownUserIds === null) {
|
|
10290
|
+
this.log({
|
|
10291
|
+
level: "debug",
|
|
10292
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10293
|
+
conversation_id: row.conversation_id,
|
|
10294
|
+
message_id: row.id
|
|
10295
|
+
});
|
|
10296
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10297
|
+
this.log({
|
|
10298
|
+
level: "warn",
|
|
10299
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10300
|
+
conversation_id: row.conversation_id,
|
|
10301
|
+
message_id: row.id
|
|
10302
|
+
});
|
|
10303
|
+
}
|
|
10304
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10305
|
+
await this.handoffUntrackableCorrelation(
|
|
10306
|
+
row.conversation_id,
|
|
10307
|
+
row.id,
|
|
10308
|
+
correlation.sessionId,
|
|
10309
|
+
correlation.attempts,
|
|
10310
|
+
"readopt"
|
|
10311
|
+
);
|
|
10312
|
+
}
|
|
10313
|
+
return true;
|
|
10314
|
+
}
|
|
9508
10315
|
/**
|
|
9509
10316
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9510
10317
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9536,6 +10343,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9536
10343
|
});
|
|
9537
10344
|
return;
|
|
9538
10345
|
}
|
|
10346
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9539
10347
|
const ocId = row.opencode_message_id;
|
|
9540
10348
|
const state = messageRunState(messages, ocId ?? "");
|
|
9541
10349
|
if (state === "done") {
|
|
@@ -9804,8 +10612,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9804
10612
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9805
10613
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9806
10614
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9807
|
-
* re-entry)
|
|
9808
|
-
*
|
|
10615
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10616
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10617
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9809
10618
|
*
|
|
9810
10619
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9811
10620
|
* `processed_at` (Invariant 1).
|
|
@@ -9868,6 +10677,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9868
10677
|
);
|
|
9869
10678
|
}
|
|
9870
10679
|
let ocId;
|
|
10680
|
+
let preDispatchUserIds = null;
|
|
9871
10681
|
try {
|
|
9872
10682
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9873
10683
|
sessionId,
|
|
@@ -9877,7 +10687,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9877
10687
|
row.content,
|
|
9878
10688
|
options,
|
|
9879
10689
|
sendAttachments,
|
|
9880
|
-
this.openCodeClient
|
|
10690
|
+
this.openCodeClient,
|
|
10691
|
+
(ids) => {
|
|
10692
|
+
preDispatchUserIds = ids;
|
|
10693
|
+
}
|
|
9881
10694
|
)
|
|
9882
10695
|
);
|
|
9883
10696
|
} catch (err) {
|
|
@@ -9958,39 +10771,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9958
10771
|
}
|
|
9959
10772
|
if (ocId === null) {
|
|
9960
10773
|
this.awaitingReadopt.delete(row.id);
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
this.log({
|
|
9968
|
-
level: "error",
|
|
9969
|
-
message: errorMessage3,
|
|
9970
|
-
conversation_id: row.conversation_id,
|
|
9971
|
-
message_id: row.id
|
|
9972
|
-
});
|
|
9973
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9974
|
-
this.log({
|
|
9975
|
-
level: "warn",
|
|
9976
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9977
|
-
conversation_id: row.conversation_id,
|
|
9978
|
-
message_id: row.id
|
|
9979
|
-
});
|
|
9980
|
-
});
|
|
9981
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9982
|
-
return;
|
|
9983
|
-
}
|
|
10774
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10775
|
+
sessionId,
|
|
10776
|
+
content: row.content,
|
|
10777
|
+
attempts: 0,
|
|
10778
|
+
knownUserIds: preDispatchUserIds
|
|
10779
|
+
});
|
|
9984
10780
|
this.log({
|
|
9985
10781
|
level: "warn",
|
|
9986
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10782
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9987
10783
|
conversation_id: row.conversation_id,
|
|
9988
10784
|
message_id: row.id
|
|
9989
10785
|
});
|
|
9990
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9991
10786
|
return;
|
|
9992
10787
|
}
|
|
9993
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9994
10788
|
const processedAtMs = this.processedAtMs(row);
|
|
9995
10789
|
this.registerReadopted(
|
|
9996
10790
|
readoptConv,
|
|
@@ -12112,6 +12906,7 @@ function createCredentialSync({
|
|
|
12112
12906
|
|
|
12113
12907
|
// src/commands/run.ts
|
|
12114
12908
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
12909
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12115
12910
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12116
12911
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12117
12912
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12267,7 +13062,13 @@ function logActivity(state, entry) {
|
|
|
12267
13062
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12268
13063
|
if (!meetsThreshold(state, level)) return;
|
|
12269
13064
|
forwardRunnerActivity(
|
|
12270
|
-
{
|
|
13065
|
+
{
|
|
13066
|
+
level,
|
|
13067
|
+
message: entry.message,
|
|
13068
|
+
error: entry.error,
|
|
13069
|
+
metadata: entry.metadata,
|
|
13070
|
+
...entry.source ? { source: entry.source } : {}
|
|
13071
|
+
},
|
|
12271
13072
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12272
13073
|
);
|
|
12273
13074
|
const fullEntry = {
|
|
@@ -12289,7 +13090,7 @@ function logActivity(state, entry) {
|
|
|
12289
13090
|
}
|
|
12290
13091
|
function reportSessionDbRecovery(state) {
|
|
12291
13092
|
try {
|
|
12292
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13093
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12293
13094
|
for (const record of report.records) {
|
|
12294
13095
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12295
13096
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12324,7 +13125,9 @@ function displayStatus(state) {
|
|
|
12324
13125
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12325
13126
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12326
13127
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12327
|
-
const detail = last ? chalk7.dim(
|
|
13128
|
+
const detail = last ? chalk7.dim(
|
|
13129
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13130
|
+
) : "";
|
|
12328
13131
|
const agent = state.agentName ?? state.agentId;
|
|
12329
13132
|
console.log(
|
|
12330
13133
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12343,7 +13146,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12343
13146
|
logActivity(state, {
|
|
12344
13147
|
type: "info",
|
|
12345
13148
|
level: "warn",
|
|
12346
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13149
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12347
13150
|
});
|
|
12348
13151
|
}
|
|
12349
13152
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12368,7 +13171,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12368
13171
|
}
|
|
12369
13172
|
const result = await importOpenCodeV2Credentials({
|
|
12370
13173
|
request: client.request,
|
|
12371
|
-
homeDir:
|
|
13174
|
+
homeDir: homedir7(),
|
|
12372
13175
|
directory,
|
|
12373
13176
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12374
13177
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12576,9 +13379,9 @@ async function driveChannels(state, driver) {
|
|
|
12576
13379
|
}
|
|
12577
13380
|
}
|
|
12578
13381
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12579
|
-
var
|
|
13382
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12580
13383
|
function shouldWarnForReclaimSkip(reason) {
|
|
12581
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13384
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12582
13385
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12583
13386
|
if (!version2) return false;
|
|
12584
13387
|
const major = Number(version2[1]);
|
|
@@ -12587,7 +13390,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12587
13390
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12588
13391
|
}
|
|
12589
13392
|
function sessionDbPath() {
|
|
12590
|
-
return join11(
|
|
13393
|
+
return join11(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
13394
|
+
}
|
|
13395
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
13396
|
+
logActivity(state, {
|
|
13397
|
+
...entry,
|
|
13398
|
+
source: "cli.session-cleanup",
|
|
13399
|
+
metadata: { ...entry.metadata, phase }
|
|
13400
|
+
});
|
|
12591
13401
|
}
|
|
12592
13402
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12593
13403
|
const record = {
|
|
@@ -12628,69 +13438,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12628
13438
|
async function runSweep(state, driver, config) {
|
|
12629
13439
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12630
13440
|
try {
|
|
12631
|
-
const
|
|
12632
|
-
|
|
13441
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13442
|
+
const sessions = enumeration.sessions;
|
|
13443
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13444
|
+
if (enumerationReason !== null) {
|
|
12633
13445
|
logActivity(state, {
|
|
12634
13446
|
type: "info",
|
|
12635
|
-
|
|
13447
|
+
level: "warn",
|
|
13448
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12636
13449
|
});
|
|
12637
|
-
return;
|
|
12638
13450
|
}
|
|
12639
|
-
const toDelete = selectSessionsToDelete(
|
|
12640
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12641
|
-
{
|
|
12642
|
-
maxAgeMs: config.maxAgeMs,
|
|
12643
|
-
maxCount: config.maxCount,
|
|
12644
|
-
nowMs: Date.now(),
|
|
12645
|
-
protectedIds: driver.protectedSessionIds()
|
|
12646
|
-
}
|
|
12647
|
-
);
|
|
12648
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12649
13451
|
let deleted = 0;
|
|
12650
13452
|
let failed = 0;
|
|
13453
|
+
let failed4xx = 0;
|
|
13454
|
+
let failed5xx = 0;
|
|
13455
|
+
let failedOtherStatus = 0;
|
|
13456
|
+
let failedThrew = 0;
|
|
12651
13457
|
let skippedNewlyActive = 0;
|
|
12652
|
-
|
|
12653
|
-
|
|
12654
|
-
|
|
12655
|
-
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
13458
|
+
let remaining = 0;
|
|
13459
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13460
|
+
if (sessions !== null) {
|
|
13461
|
+
const toDelete = selectSessionsToDelete(
|
|
13462
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13463
|
+
{
|
|
13464
|
+
maxAgeMs: config.maxAgeMs,
|
|
13465
|
+
maxCount: config.maxCount,
|
|
13466
|
+
nowMs: Date.now(),
|
|
13467
|
+
protectedIds: protectedNow
|
|
13468
|
+
}
|
|
13469
|
+
);
|
|
13470
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13471
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13472
|
+
protectedNow = driver.protectedSessionIds();
|
|
13473
|
+
for (const id of deleteCandidates) {
|
|
13474
|
+
if (protectedNow.has(id)) {
|
|
13475
|
+
skippedNewlyActive++;
|
|
13476
|
+
logActivity(state, {
|
|
13477
|
+
type: "info",
|
|
13478
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13479
|
+
});
|
|
13480
|
+
continue;
|
|
13481
|
+
}
|
|
13482
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13483
|
+
if (outcome.ok) {
|
|
13484
|
+
deleted++;
|
|
13485
|
+
} else if (outcome.status === "threw") {
|
|
13486
|
+
failed++;
|
|
13487
|
+
failedThrew++;
|
|
13488
|
+
} else {
|
|
13489
|
+
failed++;
|
|
13490
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13491
|
+
if (statusClass === 4) failed4xx++;
|
|
13492
|
+
else if (statusClass === 5) failed5xx++;
|
|
13493
|
+
else failedOtherStatus++;
|
|
13494
|
+
}
|
|
12660
13495
|
}
|
|
12661
|
-
|
|
12662
|
-
|
|
13496
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13497
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13498
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13499
|
+
logActivity(state, {
|
|
13500
|
+
type: "info",
|
|
13501
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13502
|
+
});
|
|
12663
13503
|
}
|
|
12664
|
-
const
|
|
12665
|
-
|
|
12666
|
-
|
|
12667
|
-
|
|
12668
|
-
|
|
12669
|
-
|
|
13504
|
+
const sessionMetadata = {
|
|
13505
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13506
|
+
sessions_deleted: deleted,
|
|
13507
|
+
sessions_failed: failed,
|
|
13508
|
+
sessions_failed_4xx: failed4xx,
|
|
13509
|
+
sessions_failed_5xx: failed5xx,
|
|
13510
|
+
sessions_failed_other: failedOtherStatus,
|
|
13511
|
+
sessions_failed_threw: failedThrew,
|
|
13512
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13513
|
+
sessions_remaining: remaining,
|
|
13514
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13515
|
+
};
|
|
12670
13516
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12671
13517
|
dbPath: sessionDbPath(),
|
|
12672
|
-
|
|
13518
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13519
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13520
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12673
13521
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12674
13522
|
});
|
|
12675
13523
|
if (reclaimResult.ok) {
|
|
13524
|
+
state.sessionStoreHealth.observeWritable();
|
|
12676
13525
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12677
13526
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12678
13527
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12679
|
-
|
|
13528
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12680
13529
|
type: "info",
|
|
12681
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${
|
|
13530
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
13531
|
+
metadata: {
|
|
13532
|
+
outcome: "reclaimed",
|
|
13533
|
+
mode: reclaimResult.mode,
|
|
13534
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
13535
|
+
after_bytes: reclaimResult.afterBytes,
|
|
13536
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
13537
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
13538
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13539
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13540
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13541
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13542
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13543
|
+
...sessionMetadata
|
|
13544
|
+
}
|
|
12682
13545
|
});
|
|
12683
13546
|
} else {
|
|
12684
|
-
|
|
13547
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13548
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13549
|
+
}
|
|
13550
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12685
13551
|
type: "info",
|
|
12686
|
-
|
|
13552
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13553
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
13554
|
+
metadata: {
|
|
13555
|
+
outcome: "reclaim-skipped",
|
|
13556
|
+
reason: reclaimResult.skipped,
|
|
13557
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
13558
|
+
...sessionMetadata
|
|
13559
|
+
}
|
|
12687
13560
|
});
|
|
12688
13561
|
}
|
|
12689
13562
|
} catch (error2) {
|
|
12690
13563
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12691
|
-
|
|
13564
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12692
13565
|
type: "error",
|
|
12693
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
13566
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
13567
|
+
metadata: {
|
|
13568
|
+
outcome: "sweep-failed",
|
|
13569
|
+
reason: "sweep-error"
|
|
13570
|
+
}
|
|
12694
13571
|
});
|
|
12695
13572
|
}
|
|
12696
13573
|
}
|
|
@@ -12706,14 +13583,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12706
13583
|
for (const warning2 of config.warnings) {
|
|
12707
13584
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12708
13585
|
}
|
|
12709
|
-
const dbBytes = statSessionDbBytes(
|
|
13586
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12710
13587
|
void (async () => {
|
|
12711
13588
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12712
13589
|
if (reclaimAvailability !== null) {
|
|
12713
|
-
|
|
13590
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12714
13591
|
type: "info",
|
|
12715
13592
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12716
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
13593
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
13594
|
+
metadata: {
|
|
13595
|
+
outcome: "preflight-skipped",
|
|
13596
|
+
reason: reclaimAvailability.reason,
|
|
13597
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
13598
|
+
}
|
|
12717
13599
|
});
|
|
12718
13600
|
}
|
|
12719
13601
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12730,9 +13612,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12730
13612
|
);
|
|
12731
13613
|
});
|
|
12732
13614
|
if (!config.enabled) return;
|
|
12733
|
-
|
|
13615
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12734
13616
|
type: "info",
|
|
12735
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
13617
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
13618
|
+
metadata: {
|
|
13619
|
+
outcome: "armed",
|
|
13620
|
+
interval_ms: config.intervalMs,
|
|
13621
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
13622
|
+
max_count: config.maxCount ?? null
|
|
13623
|
+
}
|
|
12736
13624
|
});
|
|
12737
13625
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12738
13626
|
const firstSweep = setTimeout(
|
|
@@ -12911,7 +13799,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12911
13799
|
});
|
|
12912
13800
|
return;
|
|
12913
13801
|
}
|
|
12914
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
13802
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12915
13803
|
state.stopResourceUsageSampling = stop;
|
|
12916
13804
|
let consecutiveFailures = 0;
|
|
12917
13805
|
const tick = async () => {
|
|
@@ -12924,6 +13812,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12924
13812
|
message: `Resource usage collection: ${warning2}`
|
|
12925
13813
|
});
|
|
12926
13814
|
}
|
|
13815
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13816
|
+
...usage,
|
|
13817
|
+
nowMs: Date.now(),
|
|
13818
|
+
state: state.lowDiskWarningState
|
|
13819
|
+
});
|
|
13820
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13821
|
+
if (lowDiskDecision.warning !== null) {
|
|
13822
|
+
logActivity(state, {
|
|
13823
|
+
type: "info",
|
|
13824
|
+
level: "warn",
|
|
13825
|
+
message: lowDiskDecision.warning
|
|
13826
|
+
});
|
|
13827
|
+
}
|
|
12927
13828
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12928
13829
|
if (result.ok) {
|
|
12929
13830
|
if (consecutiveFailures > 0) {
|
|
@@ -13136,7 +14037,7 @@ async function run(options) {
|
|
|
13136
14037
|
let fileSyncDirectories;
|
|
13137
14038
|
try {
|
|
13138
14039
|
logLevel = resolveLogLevel(options);
|
|
13139
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14040
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13140
14041
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13141
14042
|
throw new Error(
|
|
13142
14043
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13170,6 +14071,7 @@ async function run(options) {
|
|
|
13170
14071
|
opencodeClient: null,
|
|
13171
14072
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13172
14073
|
sessionDbProvenanceAnomaly: false,
|
|
14074
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13173
14075
|
opencodeProcess: null,
|
|
13174
14076
|
stopOpenCodeLogTail: null,
|
|
13175
14077
|
litestreamProcess: null,
|
|
@@ -13187,10 +14089,12 @@ async function run(options) {
|
|
|
13187
14089
|
openaiUsageRearm: null,
|
|
13188
14090
|
resourceUsageTimer: null,
|
|
13189
14091
|
stopResourceUsageSampling: null,
|
|
14092
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13190
14093
|
credentialSync: null,
|
|
13191
14094
|
authHeader: ""
|
|
13192
14095
|
};
|
|
13193
14096
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14097
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13194
14098
|
if (options.credentialSyncMarker) {
|
|
13195
14099
|
state.credentialSync = createCredentialSync({
|
|
13196
14100
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13376,13 +14280,21 @@ async function run(options) {
|
|
|
13376
14280
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13377
14281
|
}
|
|
13378
14282
|
state.credentialSync?.arm();
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14283
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14284
|
+
() => ({
|
|
13382
14285
|
agentId: state.agentId,
|
|
13383
14286
|
authHeader: state.authHeader
|
|
13384
|
-
})
|
|
13385
|
-
|
|
14287
|
+
}),
|
|
14288
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14289
|
+
);
|
|
14290
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14291
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14292
|
+
openCodeActivityForwarder
|
|
14293
|
+
);
|
|
14294
|
+
state.stopOpenCodeLogTail = () => {
|
|
14295
|
+
openCodeActivityForwarder.flush();
|
|
14296
|
+
openCodeLogTail.stop();
|
|
14297
|
+
};
|
|
13386
14298
|
let sessionDbVerifyFatal = false;
|
|
13387
14299
|
if (!options.restoreSessionDb) {
|
|
13388
14300
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
@@ -13499,7 +14411,7 @@ async function run(options) {
|
|
|
13499
14411
|
const provenance = checkSessionDbProvenance({
|
|
13500
14412
|
dbPath: sessionDbPath(),
|
|
13501
14413
|
currentVersion: state.opencodeVersion,
|
|
13502
|
-
homeDir:
|
|
14414
|
+
homeDir: homedir7(),
|
|
13503
14415
|
env: process.env
|
|
13504
14416
|
});
|
|
13505
14417
|
if (provenance.anomaly) {
|
|
@@ -13666,10 +14578,11 @@ async function run(options) {
|
|
|
13666
14578
|
getAuthHeader: () => state.authHeader,
|
|
13667
14579
|
conversationFilter: state.conversationFilter,
|
|
13668
14580
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14581
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13669
14582
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13670
14583
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13671
14584
|
fileSyncDirectories,
|
|
13672
|
-
homeDir:
|
|
14585
|
+
homeDir: homedir7(),
|
|
13673
14586
|
maxActiveSessions,
|
|
13674
14587
|
log: (entry) => (
|
|
13675
14588
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13691,12 +14604,13 @@ async function run(options) {
|
|
|
13691
14604
|
openCodePassword: state.opencodePassword,
|
|
13692
14605
|
isRunning: () => state.running,
|
|
13693
14606
|
events: {
|
|
13694
|
-
onConnected: (agentId, isReconnect) => {
|
|
14607
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13695
14608
|
state.connected = true;
|
|
13696
14609
|
state.agentId = agentId;
|
|
14610
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13697
14611
|
logActivity(state, {
|
|
13698
14612
|
type: "info",
|
|
13699
|
-
message:
|
|
14613
|
+
message: connectionMessage
|
|
13700
14614
|
});
|
|
13701
14615
|
if (options.tunnelReadyFile) {
|
|
13702
14616
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13717,7 +14631,12 @@ async function run(options) {
|
|
|
13717
14631
|
version: state.opencodeVersion,
|
|
13718
14632
|
major: state.opencodeApiVersion,
|
|
13719
14633
|
connected: state.opencodeConnected
|
|
13720
|
-
})
|
|
14634
|
+
}),
|
|
14635
|
+
connection_generation: transition.generation,
|
|
14636
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14637
|
+
reconnect_attempt: transition.attempt,
|
|
14638
|
+
downtime_ms: transition.downtimeMs
|
|
14639
|
+
} : {}
|
|
13721
14640
|
});
|
|
13722
14641
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13723
14642
|
if (state.interactive) displayStatus(state);
|
|
@@ -13739,13 +14658,18 @@ async function run(options) {
|
|
|
13739
14658
|
if (state.interactive) displayStatus(state);
|
|
13740
14659
|
});
|
|
13741
14660
|
},
|
|
13742
|
-
onDisconnected: (code, reason) => {
|
|
14661
|
+
onDisconnected: (code, reason, transition) => {
|
|
13743
14662
|
state.connected = false;
|
|
14663
|
+
const safeReason = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13744
14664
|
logActivity(state, {
|
|
13745
14665
|
type: "info",
|
|
13746
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14666
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason})`
|
|
14667
|
+
});
|
|
14668
|
+
emitAgentDisconnected(state.agentId, {
|
|
14669
|
+
code,
|
|
14670
|
+
reason: safeReason,
|
|
14671
|
+
connection_generation: transition.generation
|
|
13747
14672
|
});
|
|
13748
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13749
14673
|
if (state.interactive) displayStatus(state);
|
|
13750
14674
|
},
|
|
13751
14675
|
onError: (error2) => {
|
|
@@ -13845,7 +14769,7 @@ async function run(options) {
|
|
|
13845
14769
|
},
|
|
13846
14770
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13847
14771
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13848
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14772
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13849
14773
|
firstDelayMs: firstReportDelayMs,
|
|
13850
14774
|
nextDelayMs: usageReportDelayMs,
|
|
13851
14775
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13888,7 +14812,7 @@ async function run(options) {
|
|
|
13888
14812
|
}
|
|
13889
14813
|
|
|
13890
14814
|
// src/index.ts
|
|
13891
|
-
var
|
|
14815
|
+
var version = getCliVersion();
|
|
13892
14816
|
var program = new Command();
|
|
13893
14817
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13894
14818
|
"--endpoint <url>",
|