@evident-ai/cli 3.5.1-dev.bf9e537 → 3.5.2-dev.03d3dab
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1365 -409
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -1063,13 +1062,83 @@ async function claudeUsage() {
|
|
|
1063
1062
|
|
|
1064
1063
|
// src/commands/run.ts
|
|
1065
1064
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1065
|
+
import { homedir as homedir7 } from "node:os";
|
|
1067
1066
|
import { isAbsolute as isAbsolute3, join as join11, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1067
|
import chalk7 from "chalk";
|
|
1069
1068
|
|
|
1070
1069
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1070
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1071
|
|
|
1072
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1073
|
+
function roleOf(m) {
|
|
1074
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1075
|
+
if (typeof m.role === "string") return m.role;
|
|
1076
|
+
const infoRole = m.info?.role;
|
|
1077
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1078
|
+
}
|
|
1079
|
+
function idOf(m) {
|
|
1080
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1081
|
+
if (typeof m.id === "string") return m.id;
|
|
1082
|
+
const infoId = m.info?.id;
|
|
1083
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1084
|
+
}
|
|
1085
|
+
function parentIdOf(m) {
|
|
1086
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1087
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1088
|
+
const infoParent = m.info?.parentID;
|
|
1089
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1090
|
+
}
|
|
1091
|
+
function textPartsOf(m) {
|
|
1092
|
+
if (!m || typeof m !== "object") return [];
|
|
1093
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1094
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1095
|
+
}
|
|
1096
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1097
|
+
if (!messages || messages.length === 0) return [];
|
|
1098
|
+
const byParent = messages.filter(
|
|
1099
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1100
|
+
);
|
|
1101
|
+
if (byParent.length > 0) return byParent;
|
|
1102
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1103
|
+
if (userIndex === -1) return [];
|
|
1104
|
+
const run2 = [];
|
|
1105
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1106
|
+
const role = roleOf(messages[i]);
|
|
1107
|
+
if (role === "user") break;
|
|
1108
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1109
|
+
}
|
|
1110
|
+
return run2;
|
|
1111
|
+
}
|
|
1112
|
+
function lastText(assistantRun) {
|
|
1113
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1114
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1115
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1116
|
+
}
|
|
1117
|
+
return null;
|
|
1118
|
+
}
|
|
1119
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1120
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1121
|
+
if (lastText(run2) !== null) return run2;
|
|
1122
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1123
|
+
if (userIndex === -1) return run2;
|
|
1124
|
+
let continuation;
|
|
1125
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1126
|
+
if (roleOf(messages[i]) === "user") {
|
|
1127
|
+
continuation = messages[i];
|
|
1128
|
+
break;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const continuationId = idOf(continuation);
|
|
1132
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1133
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1134
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1135
|
+
}
|
|
1136
|
+
return run2;
|
|
1137
|
+
}
|
|
1138
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1139
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1073
1142
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1143
|
var TelemetryEventTypes = {
|
|
1075
1144
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1227,7 @@ import ora4 from "ora";
|
|
|
1158
1227
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1228
|
|
|
1160
1229
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.
|
|
1230
|
+
var CLI_VERSION = (true ? "3.5.2-dev.03d3dab" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1231
|
function getCliVersion() {
|
|
1163
1232
|
return CLI_VERSION;
|
|
1164
1233
|
}
|
|
@@ -1172,6 +1241,23 @@ var authProvider = null;
|
|
|
1172
1241
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1242
|
authProvider = provider;
|
|
1174
1243
|
}
|
|
1244
|
+
var diagnosticsReporter = null;
|
|
1245
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1246
|
+
diagnosticsReporter = reporter;
|
|
1247
|
+
}
|
|
1248
|
+
function reportTelemetryDiagnostic(message) {
|
|
1249
|
+
if (!diagnosticsReporter) {
|
|
1250
|
+
console.error(message);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
try {
|
|
1254
|
+
diagnosticsReporter(message);
|
|
1255
|
+
} catch (error2) {
|
|
1256
|
+
console.error(
|
|
1257
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1175
1261
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1262
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1263
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1323,9 @@ async function flushEvents() {
|
|
|
1237
1323
|
signal: controller.signal
|
|
1238
1324
|
});
|
|
1239
1325
|
if (!response.ok) {
|
|
1240
|
-
|
|
1326
|
+
reportTelemetryDiagnostic(
|
|
1327
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1328
|
+
);
|
|
1241
1329
|
}
|
|
1242
1330
|
} finally {
|
|
1243
1331
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1335,9 @@ async function flushEvents() {
|
|
|
1247
1335
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1336
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1337
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1338
|
+
reportTelemetryDiagnostic(
|
|
1339
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1340
|
+
);
|
|
1251
1341
|
lastFlushFailureLoggedAt = now;
|
|
1252
1342
|
suppressedFlushFailureCount = 0;
|
|
1253
1343
|
} else {
|
|
@@ -1299,8 +1389,227 @@ var EventTypes = {
|
|
|
1299
1389
|
};
|
|
1300
1390
|
|
|
1301
1391
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
|
|
1392
|
+
import { homedir as homedir2 } from "node:os";
|
|
1393
|
+
|
|
1394
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1395
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1396
|
+
import { dirname as dirname2 } from "node:path";
|
|
1397
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1398
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1399
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1400
|
+
function errorMessage(error2) {
|
|
1401
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1402
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1403
|
+
}
|
|
1404
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1405
|
+
const messages = [];
|
|
1406
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1407
|
+
let current = error2;
|
|
1408
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1409
|
+
visited.add(current);
|
|
1410
|
+
if (typeof current === "string") {
|
|
1411
|
+
messages.push(current);
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
if (current instanceof Error) {
|
|
1415
|
+
messages.push(current.name, current.message);
|
|
1416
|
+
current = current.cause;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
if (typeof current === "object") {
|
|
1420
|
+
const record = current;
|
|
1421
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1422
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1423
|
+
current = record.cause;
|
|
1424
|
+
continue;
|
|
1425
|
+
}
|
|
1426
|
+
break;
|
|
1427
|
+
}
|
|
1428
|
+
return messages.some(
|
|
1429
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1430
|
+
message
|
|
1431
|
+
)
|
|
1432
|
+
);
|
|
1433
|
+
}
|
|
1434
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1435
|
+
try {
|
|
1436
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1437
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1438
|
+
if (availableBytes < requiredBytes) {
|
|
1439
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1440
|
+
}
|
|
1441
|
+
return null;
|
|
1442
|
+
} catch (err) {
|
|
1443
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
function readLogicalBytes(db) {
|
|
1447
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1448
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1449
|
+
return pageCount * pageSize;
|
|
1450
|
+
}
|
|
1451
|
+
function readFreelistCount(db) {
|
|
1452
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1453
|
+
}
|
|
1454
|
+
function readCheckpointResult(db) {
|
|
1455
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1456
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1457
|
+
}
|
|
1458
|
+
async function probeReclaimAvailability(input) {
|
|
1459
|
+
const { dbPath, requiredBytes } = input;
|
|
1460
|
+
let sqlite;
|
|
1461
|
+
try {
|
|
1462
|
+
sqlite = await import("node:sqlite");
|
|
1463
|
+
} catch (err) {
|
|
1464
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1465
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1466
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1467
|
+
}
|
|
1468
|
+
let autoVacuum = null;
|
|
1469
|
+
try {
|
|
1470
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1471
|
+
try {
|
|
1472
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1473
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1474
|
+
} finally {
|
|
1475
|
+
db.close();
|
|
1476
|
+
}
|
|
1477
|
+
} catch (err) {
|
|
1478
|
+
console.warn(
|
|
1479
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1480
|
+
);
|
|
1481
|
+
}
|
|
1482
|
+
if (autoVacuum !== 0) return null;
|
|
1483
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1484
|
+
}
|
|
1485
|
+
async function reclaimSessionDbSpace(input) {
|
|
1486
|
+
const {
|
|
1487
|
+
dbPath,
|
|
1488
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1489
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1490
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1491
|
+
now = () => performance.now(),
|
|
1492
|
+
allowFullVacuum = true
|
|
1493
|
+
} = input;
|
|
1494
|
+
let sqlite;
|
|
1495
|
+
try {
|
|
1496
|
+
sqlite = await import("node:sqlite");
|
|
1497
|
+
} catch (err) {
|
|
1498
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1499
|
+
console.warn(
|
|
1500
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1501
|
+
);
|
|
1502
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1503
|
+
}
|
|
1504
|
+
const { DatabaseSync } = sqlite;
|
|
1505
|
+
let db;
|
|
1506
|
+
try {
|
|
1507
|
+
db = new DatabaseSync(dbPath);
|
|
1508
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1509
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1510
|
+
if (autoVacuum === 0) {
|
|
1511
|
+
if (!allowFullVacuum) {
|
|
1512
|
+
console.warn(
|
|
1513
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1514
|
+
);
|
|
1515
|
+
return {
|
|
1516
|
+
ok: false,
|
|
1517
|
+
skipped: "full-vacuum-blocked",
|
|
1518
|
+
detail: "a session turn is live"
|
|
1519
|
+
};
|
|
1520
|
+
}
|
|
1521
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1522
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1523
|
+
if (skipReason !== null) {
|
|
1524
|
+
console.warn(
|
|
1525
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1526
|
+
);
|
|
1527
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1528
|
+
}
|
|
1529
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1530
|
+
const freelistBefore = readFreelistCount(db);
|
|
1531
|
+
const startedAt = now();
|
|
1532
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1533
|
+
db.exec("VACUUM");
|
|
1534
|
+
const afterBytes = readLogicalBytes(db);
|
|
1535
|
+
const freelistAfter = readFreelistCount(db);
|
|
1536
|
+
const checkpoint = readCheckpointResult(db);
|
|
1537
|
+
return {
|
|
1538
|
+
ok: true,
|
|
1539
|
+
mode: "convert",
|
|
1540
|
+
beforeBytes,
|
|
1541
|
+
afterBytes,
|
|
1542
|
+
freelistBefore,
|
|
1543
|
+
freelistAfter,
|
|
1544
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1545
|
+
stoppedBy: "freelist-drained",
|
|
1546
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1547
|
+
checkpoint
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
if (autoVacuum === 2) {
|
|
1551
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1552
|
+
const freelistBefore = readFreelistCount(db);
|
|
1553
|
+
const startedAt = now();
|
|
1554
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1555
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1556
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1557
|
+
let pagesAttempted = 0;
|
|
1558
|
+
let freelistAfter = freelistBefore;
|
|
1559
|
+
let stoppedBy = "freelist-drained";
|
|
1560
|
+
while (freelistAfter > 0) {
|
|
1561
|
+
if (pagesAttempted >= pageLimit) {
|
|
1562
|
+
stoppedBy = "page-ceiling";
|
|
1563
|
+
break;
|
|
1564
|
+
}
|
|
1565
|
+
if (now() - startedAt >= budget) {
|
|
1566
|
+
stoppedBy = "time-budget";
|
|
1567
|
+
break;
|
|
1568
|
+
}
|
|
1569
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1570
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1571
|
+
pagesAttempted += pages;
|
|
1572
|
+
freelistAfter = readFreelistCount(db);
|
|
1573
|
+
}
|
|
1574
|
+
const afterBytes = readLogicalBytes(db);
|
|
1575
|
+
const checkpoint = readCheckpointResult(db);
|
|
1576
|
+
return {
|
|
1577
|
+
ok: true,
|
|
1578
|
+
mode: "incremental",
|
|
1579
|
+
beforeBytes,
|
|
1580
|
+
afterBytes,
|
|
1581
|
+
freelistBefore,
|
|
1582
|
+
freelistAfter,
|
|
1583
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1584
|
+
stoppedBy,
|
|
1585
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1586
|
+
checkpoint
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
console.warn(
|
|
1590
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1591
|
+
);
|
|
1592
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1595
|
+
return {
|
|
1596
|
+
ok: false,
|
|
1597
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1598
|
+
detail: errorMessage(err)
|
|
1599
|
+
};
|
|
1600
|
+
} finally {
|
|
1601
|
+
db?.close();
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1606
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1607
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1608
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1609
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
1610
|
+
};
|
|
1303
1611
|
var SEVERITY_BY_LEVEL = {
|
|
1612
|
+
info: "info",
|
|
1304
1613
|
warn: "warning",
|
|
1305
1614
|
error: "error"
|
|
1306
1615
|
};
|
|
@@ -1310,26 +1619,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
1619
|
if (!levelMatch) return null;
|
|
1311
1620
|
const level = levelMatch[1].toUpperCase();
|
|
1312
1621
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
1622
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1623
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1624
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1625
|
+
const parsed = {
|
|
1626
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1627
|
+
sessionID: sessionMatch?.[1],
|
|
1628
|
+
messageID: messageIDMatch?.[1]
|
|
1629
|
+
};
|
|
1630
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1631
|
+
if (message !== void 0) parsed.message = message;
|
|
1632
|
+
return parsed;
|
|
1315
1633
|
}
|
|
1316
1634
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
1635
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1636
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1318
1637
|
let buffer = Buffer.alloc(0);
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1638
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1639
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1640
|
+
const metadata = {
|
|
1641
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1642
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1643
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1644
|
+
};
|
|
1322
1645
|
forwardRunnerActivity(
|
|
1323
1646
|
{
|
|
1324
1647
|
level: parsed.level,
|
|
1325
1648
|
error: line,
|
|
1326
|
-
metadata:
|
|
1649
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1327
1650
|
source: "opencode"
|
|
1328
1651
|
},
|
|
1329
1652
|
getContext()
|
|
1330
1653
|
);
|
|
1331
1654
|
};
|
|
1332
|
-
|
|
1655
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1656
|
+
for (const [key, state] of repeatStates) {
|
|
1657
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1658
|
+
continue;
|
|
1659
|
+
}
|
|
1660
|
+
if (state.repeatsSuppressed > 0) {
|
|
1661
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1662
|
+
}
|
|
1663
|
+
repeatStates.delete(key);
|
|
1664
|
+
}
|
|
1665
|
+
};
|
|
1666
|
+
const flushLine = (line) => {
|
|
1667
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1668
|
+
try {
|
|
1669
|
+
onSessionStoreWriteFailure();
|
|
1670
|
+
} catch (err) {
|
|
1671
|
+
reportTelemetryDiagnostic(
|
|
1672
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1677
|
+
if (!parsed) return;
|
|
1678
|
+
const now = Date.now();
|
|
1679
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1680
|
+
expireRepeatStates(now, key);
|
|
1681
|
+
if (key === void 0) {
|
|
1682
|
+
forwardLine(line, parsed);
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
const previous = repeatStates.get(key);
|
|
1686
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1687
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1688
|
+
repeatStates.set(key, {
|
|
1689
|
+
line,
|
|
1690
|
+
parsed,
|
|
1691
|
+
lastForwardedAt: now,
|
|
1692
|
+
repeatsSuppressed: 0
|
|
1693
|
+
});
|
|
1694
|
+
return;
|
|
1695
|
+
}
|
|
1696
|
+
previous.line = line;
|
|
1697
|
+
previous.parsed = parsed;
|
|
1698
|
+
previous.repeatsSuppressed++;
|
|
1699
|
+
};
|
|
1700
|
+
const forward = ((chunk) => {
|
|
1333
1701
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
1702
|
let newlineIndex;
|
|
1335
1703
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,14 +1708,44 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
1708
|
flushLine(buffer.toString("utf-8"));
|
|
1341
1709
|
buffer = Buffer.alloc(0);
|
|
1342
1710
|
}
|
|
1711
|
+
});
|
|
1712
|
+
forward.flush = () => {
|
|
1713
|
+
for (const state of repeatStates.values()) {
|
|
1714
|
+
if (state.repeatsSuppressed > 0) {
|
|
1715
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
repeatStates.clear();
|
|
1343
1719
|
};
|
|
1720
|
+
return forward;
|
|
1344
1721
|
}
|
|
1345
1722
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
1723
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
1724
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
1725
|
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1726
|
+
function redactHomeDirectory(message) {
|
|
1727
|
+
const home = homedir2();
|
|
1728
|
+
if (!home || home === "/") {
|
|
1729
|
+
return message;
|
|
1730
|
+
}
|
|
1731
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1732
|
+
let result = "";
|
|
1733
|
+
let searchFrom = 0;
|
|
1734
|
+
let homeIndex;
|
|
1735
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1736
|
+
const homeEnd = homeIndex + home.length;
|
|
1737
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1738
|
+
const nextCharacter = message[homeEnd];
|
|
1739
|
+
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1740
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1741
|
+
result += atPathBoundary ? "~" : home;
|
|
1742
|
+
searchFrom = homeEnd;
|
|
1743
|
+
}
|
|
1744
|
+
return result + message.slice(searchFrom);
|
|
1745
|
+
}
|
|
1746
|
+
function redactDiagnosticText(message) {
|
|
1747
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1748
|
+
return redactHomeDirectory(redacted);
|
|
1351
1749
|
}
|
|
1352
1750
|
function truncate(message) {
|
|
1353
1751
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1357,14 +1755,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1357
1755
|
if (!metadata) return {};
|
|
1358
1756
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
1757
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
1758
|
+
reportTelemetryDiagnostic(
|
|
1759
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
1760
|
);
|
|
1363
1761
|
}
|
|
1364
1762
|
const sanitised = [];
|
|
1365
1763
|
for (const [key, value] of entries) {
|
|
1366
1764
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
1765
|
+
sanitised.push([
|
|
1766
|
+
key,
|
|
1767
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1768
|
+
]);
|
|
1368
1769
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
1770
|
sanitised.push([key, value]);
|
|
1370
1771
|
}
|
|
@@ -1382,8 +1783,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
1783
|
}
|
|
1383
1784
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
1785
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
1786
|
+
reportTelemetryDiagnostic(
|
|
1787
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
1788
|
);
|
|
1388
1789
|
}
|
|
1389
1790
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +1794,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
1794
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
1795
|
window.windowDroppedCount++;
|
|
1395
1796
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
1797
|
+
reportTelemetryDiagnostic(
|
|
1798
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
1799
|
);
|
|
1399
1800
|
}
|
|
1400
1801
|
return false;
|
|
@@ -1404,12 +1805,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
1805
|
}
|
|
1405
1806
|
function forwardRunnerActivity(entry, context) {
|
|
1406
1807
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
1808
|
const source = entry.source ?? "cli.run";
|
|
1809
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
1810
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
1811
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
1812
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
1813
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
1814
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
1815
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
1816
|
message,
|
|
@@ -1417,8 +1818,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
1818
|
agentId: context.agentId
|
|
1418
1819
|
});
|
|
1419
1820
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
1821
|
+
reportTelemetryDiagnostic(
|
|
1822
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
1823
|
);
|
|
1423
1824
|
}
|
|
1424
1825
|
}
|
|
@@ -1620,6 +2021,9 @@ async function checkOpenCodeHealth(port) {
|
|
|
1620
2021
|
signal: AbortSignal.timeout(2e3)
|
|
1621
2022
|
// 2 second timeout
|
|
1622
2023
|
});
|
|
2024
|
+
if (response.status === 401) {
|
|
2025
|
+
return { healthy: false, authFailed: true, error: "HTTP 401" };
|
|
2026
|
+
}
|
|
1623
2027
|
if (!response.ok) {
|
|
1624
2028
|
return { healthy: false, error: `HTTP ${response.status}` };
|
|
1625
2029
|
}
|
|
@@ -1684,7 +2088,7 @@ async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
|
1684
2088
|
const startTime = Date.now();
|
|
1685
2089
|
while (Date.now() - startTime < timeoutMs) {
|
|
1686
2090
|
const health = await checkOpenCodeHealth(port);
|
|
1687
|
-
if (health.healthy) {
|
|
2091
|
+
if (health.healthy || health.authFailed) {
|
|
1688
2092
|
return health;
|
|
1689
2093
|
}
|
|
1690
2094
|
await new Promise((resolve4) => setTimeout(resolve4, 1e3));
|
|
@@ -1705,8 +2109,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1705
2109
|
|
|
1706
2110
|
// src/lib/http-timeout.ts
|
|
1707
2111
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2112
|
+
var RequestTimeoutError = class extends Error {
|
|
2113
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2114
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2115
|
+
this.operation = operation;
|
|
2116
|
+
this.timeoutMs = timeoutMs;
|
|
2117
|
+
this.elapsedMs = elapsedMs;
|
|
2118
|
+
this.name = "TimeoutError";
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
function describeRequestOperation(input, init) {
|
|
2122
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2123
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2124
|
+
let path;
|
|
2125
|
+
try {
|
|
2126
|
+
path = new URL(rawUrl).pathname;
|
|
2127
|
+
} catch (error2) {
|
|
2128
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2129
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2130
|
+
}
|
|
2131
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2132
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2133
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2134
|
+
return `${method} ${maskedPath}`;
|
|
2135
|
+
}
|
|
1708
2136
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1709
|
-
return ((input, init) =>
|
|
2137
|
+
return (async (input, init) => {
|
|
2138
|
+
const startedAt = Date.now();
|
|
2139
|
+
try {
|
|
2140
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2141
|
+
} catch (error2) {
|
|
2142
|
+
if (error2?.name === "TimeoutError") {
|
|
2143
|
+
throw new RequestTimeoutError(
|
|
2144
|
+
describeRequestOperation(input, init),
|
|
2145
|
+
timeoutMs,
|
|
2146
|
+
Date.now() - startedAt
|
|
2147
|
+
);
|
|
2148
|
+
}
|
|
2149
|
+
throw error2;
|
|
2150
|
+
}
|
|
2151
|
+
});
|
|
1710
2152
|
}
|
|
1711
2153
|
|
|
1712
2154
|
// src/lib/opencode/client.ts
|
|
@@ -1772,7 +2214,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1772
2214
|
return null;
|
|
1773
2215
|
}
|
|
1774
2216
|
}
|
|
1775
|
-
function
|
|
2217
|
+
function roleOf2(m) {
|
|
1776
2218
|
if (!m || typeof m !== "object") return void 0;
|
|
1777
2219
|
if (typeof m.role === "string") return m.role;
|
|
1778
2220
|
const infoRole = m.info?.role;
|
|
@@ -1786,13 +2228,13 @@ function createdOf(m) {
|
|
|
1786
2228
|
if (!m || typeof m !== "object") return void 0;
|
|
1787
2229
|
return m.info?.time?.created ?? m.time?.created;
|
|
1788
2230
|
}
|
|
1789
|
-
function
|
|
2231
|
+
function idOf2(m) {
|
|
1790
2232
|
if (!m || typeof m !== "object") return void 0;
|
|
1791
2233
|
if (typeof m.id === "string") return m.id;
|
|
1792
2234
|
const infoId = m.info?.id;
|
|
1793
2235
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1794
2236
|
}
|
|
1795
|
-
function
|
|
2237
|
+
function parentIdOf2(m) {
|
|
1796
2238
|
if (!m || typeof m !== "object") return void 0;
|
|
1797
2239
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1798
2240
|
const infoParent = m.info?.parentID;
|
|
@@ -1857,9 +2299,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1857
2299
|
function isSessionActivelyGenerating(messages) {
|
|
1858
2300
|
if (!messages || messages.length === 0) return false;
|
|
1859
2301
|
const last = messages[messages.length - 1];
|
|
1860
|
-
if (
|
|
2302
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1861
2303
|
return completedOf(last) == null;
|
|
1862
2304
|
}
|
|
2305
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1863
2306
|
function sessionLastActivityMs(session) {
|
|
1864
2307
|
const candidates = [
|
|
1865
2308
|
session.time?.updated,
|
|
@@ -1874,30 +2317,60 @@ function sessionLastActivityMs(session) {
|
|
|
1874
2317
|
}
|
|
1875
2318
|
return null;
|
|
1876
2319
|
}
|
|
1877
|
-
|
|
1878
|
-
|
|
2320
|
+
function sessionListPath(options) {
|
|
2321
|
+
const params = new URLSearchParams();
|
|
2322
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2323
|
+
const query = params.toString();
|
|
2324
|
+
return query ? `/session?${query}` : "/session";
|
|
2325
|
+
}
|
|
2326
|
+
async function listSessionPage(port, client, options) {
|
|
2327
|
+
const path = sessionListPath(options);
|
|
1879
2328
|
try {
|
|
1880
|
-
const res = await requestWithClient(port, client,
|
|
2329
|
+
const res = await requestWithClient(port, client, path);
|
|
1881
2330
|
if (!res.ok) return null;
|
|
1882
2331
|
const body = await res.json();
|
|
1883
2332
|
return Array.isArray(body) ? body : null;
|
|
1884
2333
|
} catch (error2) {
|
|
1885
2334
|
console.error(
|
|
1886
|
-
`[
|
|
2335
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1887
2336
|
);
|
|
1888
2337
|
return null;
|
|
1889
2338
|
}
|
|
1890
2339
|
}
|
|
2340
|
+
async function listSessions(port, client, options) {
|
|
2341
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2342
|
+
return listSessionPage(port, client, options);
|
|
2343
|
+
}
|
|
2344
|
+
async function listAllSessions(port, client) {
|
|
2345
|
+
if (client?.version === "v2") {
|
|
2346
|
+
const sessions2 = await listV2Sessions(client);
|
|
2347
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2348
|
+
}
|
|
2349
|
+
const sessions = await listSessionPage(port, client, {
|
|
2350
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2351
|
+
});
|
|
2352
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2353
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2354
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2355
|
+
}
|
|
2356
|
+
return { sessions };
|
|
2357
|
+
}
|
|
1891
2358
|
async function deleteSession(port, id, client) {
|
|
1892
2359
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1893
2360
|
try {
|
|
1894
|
-
const res = await requestWithClient(
|
|
1895
|
-
|
|
2361
|
+
const res = await requestWithClient(
|
|
2362
|
+
port,
|
|
2363
|
+
client,
|
|
2364
|
+
`/session/${id}`,
|
|
2365
|
+
{ method: "DELETE" },
|
|
2366
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2367
|
+
);
|
|
2368
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1896
2369
|
} catch (error2) {
|
|
1897
2370
|
console.error(
|
|
1898
2371
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1899
2372
|
);
|
|
1900
|
-
return false;
|
|
2373
|
+
return { ok: false, status: "threw" };
|
|
1901
2374
|
}
|
|
1902
2375
|
}
|
|
1903
2376
|
async function sessionExists(port, id, client) {
|
|
@@ -2135,11 +2608,33 @@ function messageText(m) {
|
|
|
2135
2608
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2136
2609
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2137
2610
|
}
|
|
2138
|
-
|
|
2611
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2612
|
+
if (!messages || messages.length === 0) return null;
|
|
2613
|
+
let best = null;
|
|
2614
|
+
for (const message of messages) {
|
|
2615
|
+
if (roleOf2(message) !== "user") continue;
|
|
2616
|
+
const id = idOf2(message);
|
|
2617
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2618
|
+
if (messageText(message) !== content) continue;
|
|
2619
|
+
const created = createdOf(message) ?? 0;
|
|
2620
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2621
|
+
}
|
|
2622
|
+
return best?.id ?? null;
|
|
2623
|
+
}
|
|
2624
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, onKnownUserIds) {
|
|
2139
2625
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2140
2626
|
const knownUserIds = new Set(
|
|
2141
|
-
(before ?? []).filter((m) =>
|
|
2627
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2142
2628
|
);
|
|
2629
|
+
if (onKnownUserIds) {
|
|
2630
|
+
try {
|
|
2631
|
+
onKnownUserIds(before === null ? null : knownUserIds);
|
|
2632
|
+
} catch (error2) {
|
|
2633
|
+
console.error(
|
|
2634
|
+
`[sendPromptAsync] pre-dispatch snapshot callback failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
2635
|
+
);
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2143
2638
|
const parts = [{ type: "text", text: content }];
|
|
2144
2639
|
let pendingOutcomes = null;
|
|
2145
2640
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2178,22 +2673,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2178
2673
|
const READ_BACK_DELAY_MS = 150;
|
|
2179
2674
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2180
2675
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
const id = idOf(m);
|
|
2186
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2187
|
-
if (messageText(m) !== content) continue;
|
|
2188
|
-
const created = createdOf(m) ?? 0;
|
|
2189
|
-
if (best === null || created > best.created) {
|
|
2190
|
-
best = { id, created };
|
|
2191
|
-
}
|
|
2192
|
-
}
|
|
2193
|
-
if (best) {
|
|
2194
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2195
|
-
return best.id;
|
|
2196
|
-
}
|
|
2676
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2677
|
+
if (dispatchedUserMessageId !== null) {
|
|
2678
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2679
|
+
return dispatchedUserMessageId;
|
|
2197
2680
|
}
|
|
2198
2681
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2199
2682
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2204,13 +2687,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2204
2687
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2205
2688
|
if (!messages || messages.length === 0) return null;
|
|
2206
2689
|
const byParent = messages.find(
|
|
2207
|
-
(m) =>
|
|
2690
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2208
2691
|
);
|
|
2209
2692
|
if (byParent) return byParent;
|
|
2210
|
-
const userIndex = messages.findIndex((m) =>
|
|
2693
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2211
2694
|
if (userIndex === -1) return null;
|
|
2212
2695
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2213
|
-
if (
|
|
2696
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2214
2697
|
}
|
|
2215
2698
|
return null;
|
|
2216
2699
|
}
|
|
@@ -2220,7 +2703,7 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2220
2703
|
let lastNonErrored = null;
|
|
2221
2704
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2222
2705
|
const m = messages[i];
|
|
2223
|
-
if (
|
|
2706
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2224
2707
|
if (lastCorrelated === null) lastCorrelated = m;
|
|
2225
2708
|
if (errorOf(m) == null) {
|
|
2226
2709
|
lastNonErrored = m;
|
|
@@ -2228,12 +2711,12 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2228
2711
|
}
|
|
2229
2712
|
}
|
|
2230
2713
|
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2231
|
-
const userIndex = messages.findIndex((m) =>
|
|
2714
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2232
2715
|
if (userIndex === -1) return null;
|
|
2233
2716
|
let last = null;
|
|
2234
2717
|
let lastOk = null;
|
|
2235
2718
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2236
|
-
const role =
|
|
2719
|
+
const role = roleOf2(messages[i]);
|
|
2237
2720
|
if (role === "user") break;
|
|
2238
2721
|
if (role === "assistant") {
|
|
2239
2722
|
last = messages[i];
|
|
@@ -2245,16 +2728,16 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2245
2728
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2246
2729
|
if (!messages || messages.length === 0) return [];
|
|
2247
2730
|
const byParent = messages.filter(
|
|
2248
|
-
(message) =>
|
|
2731
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2249
2732
|
);
|
|
2250
2733
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2251
2734
|
if (assistants.length === 0) {
|
|
2252
|
-
const userIndex = messages.findIndex((message) =>
|
|
2735
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2253
2736
|
if (userIndex === -1) return [];
|
|
2254
2737
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2255
2738
|
const message = messages[i];
|
|
2256
|
-
if (
|
|
2257
|
-
if (
|
|
2739
|
+
if (roleOf2(message) === "user") break;
|
|
2740
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2258
2741
|
}
|
|
2259
2742
|
}
|
|
2260
2743
|
const refs = [];
|
|
@@ -2294,7 +2777,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2294
2777
|
if (!messages || messages.length === 0) return [];
|
|
2295
2778
|
const calls = [];
|
|
2296
2779
|
for (const message of messages) {
|
|
2297
|
-
if (
|
|
2780
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2298
2781
|
for (const part of message.parts ?? []) {
|
|
2299
2782
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2300
2783
|
continue;
|
|
@@ -2324,7 +2807,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2324
2807
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2325
2808
|
const unattributed = [];
|
|
2326
2809
|
for (const message of messages ?? []) {
|
|
2327
|
-
if (
|
|
2810
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2328
2811
|
const created = finiteNumber(createdOf(message));
|
|
2329
2812
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2330
2813
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2401,7 +2884,7 @@ function sumAssistantUsage(messages) {
|
|
|
2401
2884
|
function messageUsage(messages, userMessageId) {
|
|
2402
2885
|
if (!messages || messages.length === 0) return null;
|
|
2403
2886
|
const byParentAll = messages.filter(
|
|
2404
|
-
(m) =>
|
|
2887
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2405
2888
|
);
|
|
2406
2889
|
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2407
2890
|
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
@@ -2417,7 +2900,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2417
2900
|
}
|
|
2418
2901
|
function messageRunState(messages, userMessageId) {
|
|
2419
2902
|
if (!messages || messages.length === 0) return "unknown";
|
|
2420
|
-
const hasUser = messages.some((m) =>
|
|
2903
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2421
2904
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2422
2905
|
if (!hasUser) {
|
|
2423
2906
|
if (!reply) return "unknown";
|
|
@@ -2509,7 +2992,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2509
2992
|
if (!messages || messages.length === 0) return null;
|
|
2510
2993
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2511
2994
|
const message = messages[i];
|
|
2512
|
-
if (
|
|
2995
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2513
2996
|
const created = createdOf(message);
|
|
2514
2997
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2515
2998
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2541,25 +3024,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2541
3024
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2542
3025
|
if (!messages || messages.length === 0) return false;
|
|
2543
3026
|
return messages.some(
|
|
2544
|
-
(m) =>
|
|
3027
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2545
3028
|
);
|
|
2546
3029
|
}
|
|
2547
3030
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2548
3031
|
if (!messages || messages.length === 0) return false;
|
|
2549
|
-
const userIndex = messages.findIndex((message) =>
|
|
3032
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2550
3033
|
if (userIndex === -1) return false;
|
|
2551
3034
|
let hasLaterUser = false;
|
|
2552
3035
|
let hasStartedLaterUser = false;
|
|
2553
3036
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2554
3037
|
const message = messages[i];
|
|
2555
|
-
if (
|
|
3038
|
+
if (roleOf2(message) !== "user") continue;
|
|
2556
3039
|
hasLaterUser = true;
|
|
2557
|
-
const laterUserMessageId =
|
|
3040
|
+
const laterUserMessageId = idOf2(message);
|
|
2558
3041
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2559
3042
|
return false;
|
|
2560
3043
|
}
|
|
2561
3044
|
if (messages.some(
|
|
2562
|
-
(candidate) =>
|
|
3045
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2563
3046
|
)) {
|
|
2564
3047
|
hasStartedLaterUser = true;
|
|
2565
3048
|
}
|
|
@@ -3030,13 +3513,17 @@ async function listV2Sessions(client) {
|
|
|
3030
3513
|
}
|
|
3031
3514
|
async function deleteV2Session(client, sessionId) {
|
|
3032
3515
|
try {
|
|
3033
|
-
await client.request(
|
|
3034
|
-
|
|
3516
|
+
const response = await client.request(
|
|
3517
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3518
|
+
{ method: "DELETE" },
|
|
3519
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3520
|
+
);
|
|
3521
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3035
3522
|
} catch (error2) {
|
|
3036
3523
|
console.error(
|
|
3037
3524
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3038
3525
|
);
|
|
3039
|
-
return false;
|
|
3526
|
+
return { ok: false, status: "threw" };
|
|
3040
3527
|
}
|
|
3041
3528
|
}
|
|
3042
3529
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3211,7 +3698,7 @@ import { join as join4 } from "node:path";
|
|
|
3211
3698
|
|
|
3212
3699
|
// src/lib/openai-usage.ts
|
|
3213
3700
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3214
|
-
import { homedir as
|
|
3701
|
+
import { homedir as homedir3 } from "node:os";
|
|
3215
3702
|
import { join as join3 } from "node:path";
|
|
3216
3703
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3217
3704
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
@@ -3226,7 +3713,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3226
3713
|
}
|
|
3227
3714
|
function readOpenCodeChatGptCredentials() {
|
|
3228
3715
|
try {
|
|
3229
|
-
const raw = readFileSync3(join3(
|
|
3716
|
+
const raw = readFileSync3(join3(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3230
3717
|
let parsed;
|
|
3231
3718
|
try {
|
|
3232
3719
|
parsed = JSON.parse(raw);
|
|
@@ -3367,16 +3854,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3367
3854
|
].some((name) => headers.has(name));
|
|
3368
3855
|
}
|
|
3369
3856
|
async function getOpenAiUsage(port, client) {
|
|
3857
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3370
3858
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3371
3859
|
if (!credentials2) {
|
|
3372
3860
|
throw new OpenAiUsageError(
|
|
3373
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3861
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3374
3862
|
"no_credentials"
|
|
3375
3863
|
);
|
|
3376
3864
|
}
|
|
3377
3865
|
if (credentials2.expiresAt < Date.now()) {
|
|
3378
3866
|
throw new OpenAiUsageError(
|
|
3379
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3867
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3380
3868
|
"credentials_expired"
|
|
3381
3869
|
);
|
|
3382
3870
|
}
|
|
@@ -3657,9 +4145,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3657
4145
|
|
|
3658
4146
|
// src/lib/opencode/session-db-boot.ts
|
|
3659
4147
|
import { spawn as spawn2 } from "node:child_process";
|
|
3660
|
-
import { mkdirSync, statSync as
|
|
3661
|
-
import { homedir as
|
|
3662
|
-
import { dirname as
|
|
4148
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4149
|
+
import { homedir as homedir4 } from "node:os";
|
|
4150
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3663
4151
|
|
|
3664
4152
|
// src/lib/runner-synchroniser.ts
|
|
3665
4153
|
import { spawn } from "node:child_process";
|
|
@@ -3784,7 +4272,7 @@ function clearMarker(options) {
|
|
|
3784
4272
|
function markNoReplicate(options, message) {
|
|
3785
4273
|
if (options.noReplicateMarker) {
|
|
3786
4274
|
try {
|
|
3787
|
-
mkdirSync(
|
|
4275
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3788
4276
|
writeFileSync(options.noReplicateMarker, "");
|
|
3789
4277
|
} catch (error2) {
|
|
3790
4278
|
options.log(
|
|
@@ -3909,7 +4397,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3909
4397
|
return null;
|
|
3910
4398
|
}
|
|
3911
4399
|
try {
|
|
3912
|
-
if (
|
|
4400
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3913
4401
|
} catch (error2) {
|
|
3914
4402
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3915
4403
|
options.log(
|
|
@@ -3939,7 +4427,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3939
4427
|
return null;
|
|
3940
4428
|
}
|
|
3941
4429
|
try {
|
|
3942
|
-
mkdirSync(
|
|
4430
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3943
4431
|
writeFileSync(configPath, rendered.stdout);
|
|
3944
4432
|
} catch (error2) {
|
|
3945
4433
|
options.log(
|
|
@@ -4014,14 +4502,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4014
4502
|
case 0:
|
|
4015
4503
|
return;
|
|
4016
4504
|
case 31:
|
|
4017
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4505
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4018
4506
|
options.log(
|
|
4019
4507
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4020
4508
|
"warn"
|
|
4021
4509
|
);
|
|
4022
4510
|
return;
|
|
4023
4511
|
case 32:
|
|
4024
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4512
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4025
4513
|
discardSessionDbDebris(options);
|
|
4026
4514
|
markNoReplicate(
|
|
4027
4515
|
options,
|
|
@@ -4100,7 +4588,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4100
4588
|
}
|
|
4101
4589
|
function fileExists(path) {
|
|
4102
4590
|
try {
|
|
4103
|
-
|
|
4591
|
+
statSync3(path);
|
|
4104
4592
|
return true;
|
|
4105
4593
|
} catch (error2) {
|
|
4106
4594
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4110,7 +4598,7 @@ function fileExists(path) {
|
|
|
4110
4598
|
async function restoreAndVerifySessionDb(options) {
|
|
4111
4599
|
const env = options.env ?? process.env;
|
|
4112
4600
|
clearMarker(options);
|
|
4113
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4601
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4114
4602
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4115
4603
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4116
4604
|
env
|
|
@@ -4189,7 +4677,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4189
4677
|
// src/lib/opencode/session-db-provenance.ts
|
|
4190
4678
|
import { createRequire } from "node:module";
|
|
4191
4679
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4192
|
-
import { dirname as
|
|
4680
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4193
4681
|
var require2 = createRequire(import.meta.url);
|
|
4194
4682
|
function readSessionDbMigrationIds(dbPath) {
|
|
4195
4683
|
let db;
|
|
@@ -4255,7 +4743,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4255
4743
|
}
|
|
4256
4744
|
function saveSessionDbProvenanceState(path, state) {
|
|
4257
4745
|
try {
|
|
4258
|
-
mkdirSync2(
|
|
4746
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4259
4747
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4260
4748
|
`, "utf8");
|
|
4261
4749
|
} catch (error2) {
|
|
@@ -4311,6 +4799,9 @@ function isSessionDbProvenanceRecord(value) {
|
|
|
4311
4799
|
return typeof record.opencodeVersion === "string" && Array.isArray(record.migrationIds) && record.migrationIds.every((id) => typeof id === "string") && typeof record.updatedAt === "string";
|
|
4312
4800
|
}
|
|
4313
4801
|
|
|
4802
|
+
// src/lib/opencode/opencode2-package.ts
|
|
4803
|
+
var OPENCODE2_PACKAGE_SPEC = "@opencode-ai/cli@0.0.0-beta-17823";
|
|
4804
|
+
|
|
4314
4805
|
// src/lib/opencode/opencode-version-gate.ts
|
|
4315
4806
|
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
4316
4807
|
function isQueueValidatedVersion(version2) {
|
|
@@ -4487,12 +4978,15 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4487
4978
|
}
|
|
4488
4979
|
} catch {
|
|
4489
4980
|
try {
|
|
4490
|
-
const psOutput = execSync(
|
|
4981
|
+
const psOutput = execSync("ps aux", {
|
|
4491
4982
|
encoding: "utf-8",
|
|
4492
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
4983
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
4984
|
+
// The whole table reaches Node; the 1 MiB default would surface a spurious failure.
|
|
4985
|
+
maxBuffer: 16 * 1024 * 1024
|
|
4493
4986
|
}).trim();
|
|
4494
4987
|
if (psOutput) {
|
|
4495
4988
|
for (const line of psOutput.split("\n")) {
|
|
4989
|
+
if (!psPattern.test(line)) continue;
|
|
4496
4990
|
const parts = line.trim().split(/\s+/);
|
|
4497
4991
|
if (parts.length >= 2) {
|
|
4498
4992
|
const pid = parseInt(parts[1], 10);
|
|
@@ -4501,8 +4995,9 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4501
4995
|
}
|
|
4502
4996
|
}
|
|
4503
4997
|
} catch (err) {
|
|
4998
|
+
const reason = (typeof err === "object" && err !== null && "stderr" in err && typeof err.stderr === "string" ? err.stderr : "").split("\n").map((line) => line.trim()).find(Boolean) ?? (err instanceof Error ? err.message : String(err)).split("\n").map((line) => line.trim()).find(Boolean) ?? "no error detail available";
|
|
4504
4999
|
console.warn(
|
|
4505
|
-
`
|
|
5000
|
+
`Could not list running processes, so an already-running OpenCode may not be detected: ${reason}`
|
|
4506
5001
|
);
|
|
4507
5002
|
}
|
|
4508
5003
|
}
|
|
@@ -4534,7 +5029,7 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4534
5029
|
return instances;
|
|
4535
5030
|
}
|
|
4536
5031
|
function findOpenCodeProcesses() {
|
|
4537
|
-
return findProcessesByPattern("opencode serve|opencode-serve",
|
|
5032
|
+
return findProcessesByPattern("opencode serve|opencode-serve", /opencode (serve|--port)/);
|
|
4538
5033
|
}
|
|
4539
5034
|
async function scanPortsForOpenCode() {
|
|
4540
5035
|
const instances = [];
|
|
@@ -4601,7 +5096,8 @@ async function startOpenCode(port, options = {}) {
|
|
|
4601
5096
|
const child = spawn3(command, args, {
|
|
4602
5097
|
detached: true,
|
|
4603
5098
|
stdio: options.inheritStdio ? "inherit" : "ignore",
|
|
4604
|
-
cwd: process.cwd()
|
|
5099
|
+
cwd: process.cwd(),
|
|
5100
|
+
env: { ...process.env, OPENCODE_SERVER_PASSWORD: void 0 }
|
|
4605
5101
|
});
|
|
4606
5102
|
return child;
|
|
4607
5103
|
}
|
|
@@ -4617,7 +5113,7 @@ async function startOpenCode2(port, options = {}) {
|
|
|
4617
5113
|
args = [
|
|
4618
5114
|
"-y",
|
|
4619
5115
|
"-p",
|
|
4620
|
-
|
|
5116
|
+
OPENCODE2_PACKAGE_SPEC,
|
|
4621
5117
|
"--",
|
|
4622
5118
|
"opencode2",
|
|
4623
5119
|
"serve",
|
|
@@ -4694,7 +5190,7 @@ async function promptOpenCodeInstall(interactive) {
|
|
|
4694
5190
|
npm: "npm install -g opencode-ai",
|
|
4695
5191
|
curl: "curl -fsSL https://opencode.ai/install.sh | sh",
|
|
4696
5192
|
v2: {
|
|
4697
|
-
npm:
|
|
5193
|
+
npm: `npm install -g ${OPENCODE2_PACKAGE_SPEC}`,
|
|
4698
5194
|
curl: "curl -fsSL https://opencode.ai/v2/install | bash"
|
|
4699
5195
|
}
|
|
4700
5196
|
}
|
|
@@ -4828,7 +5324,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4828
5324
|
toDelete.push(s.id);
|
|
4829
5325
|
}
|
|
4830
5326
|
}
|
|
4831
|
-
|
|
5327
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5328
|
+
return toDelete.sort(
|
|
5329
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5330
|
+
);
|
|
4832
5331
|
}
|
|
4833
5332
|
var DEFAULT_INTERVAL = "1h";
|
|
4834
5333
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4888,13 +5387,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4888
5387
|
}
|
|
4889
5388
|
|
|
4890
5389
|
// src/lib/opencode/session-db-size.ts
|
|
4891
|
-
import { statSync as
|
|
5390
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4892
5391
|
import { join as join6 } from "node:path";
|
|
4893
5392
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4894
5393
|
function statSessionDbBytes(homeDir) {
|
|
4895
5394
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4896
5395
|
try {
|
|
4897
|
-
return
|
|
5396
|
+
return statSync4(dbPath).size;
|
|
4898
5397
|
} catch (err) {
|
|
4899
5398
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4900
5399
|
if (!isMissingFile) {
|
|
@@ -4920,12 +5419,12 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4920
5419
|
}
|
|
4921
5420
|
|
|
4922
5421
|
// src/lib/opencode/log-tail.ts
|
|
4923
|
-
import { statSync as
|
|
4924
|
-
import { homedir as
|
|
5422
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5423
|
+
import { homedir as homedir5 } from "node:os";
|
|
4925
5424
|
import { join as join7 } from "node:path";
|
|
4926
5425
|
import { open as open2, stat } from "node:fs/promises";
|
|
4927
5426
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4928
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
5427
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
4929
5428
|
const dataDir = env.XDG_DATA_HOME || join7(homeDir, ".local", "share");
|
|
4930
5429
|
return join7(dataDir, "opencode", "log", "opencode.log");
|
|
4931
5430
|
}
|
|
@@ -4943,7 +5442,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4943
5442
|
let inode = null;
|
|
4944
5443
|
let baselineReady = true;
|
|
4945
5444
|
try {
|
|
4946
|
-
const initial =
|
|
5445
|
+
const initial = statSync5(logPath);
|
|
4947
5446
|
offset = initial.size;
|
|
4948
5447
|
inode = initial.ino;
|
|
4949
5448
|
} catch (error2) {
|
|
@@ -5002,125 +5501,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5002
5501
|
};
|
|
5003
5502
|
}
|
|
5004
5503
|
|
|
5005
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5006
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5007
|
-
import { dirname as dirname4 } from "node:path";
|
|
5008
|
-
function errorMessage(error2) {
|
|
5009
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5010
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5011
|
-
}
|
|
5012
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5013
|
-
try {
|
|
5014
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5015
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5016
|
-
if (availableBytes < requiredBytes) {
|
|
5017
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5018
|
-
}
|
|
5019
|
-
return null;
|
|
5020
|
-
} catch (err) {
|
|
5021
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5022
|
-
}
|
|
5023
|
-
}
|
|
5024
|
-
function readLogicalBytes(db) {
|
|
5025
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5026
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5027
|
-
return pageCount * pageSize;
|
|
5028
|
-
}
|
|
5029
|
-
function readCheckpointResult(db) {
|
|
5030
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5031
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5032
|
-
}
|
|
5033
|
-
async function probeReclaimAvailability(input) {
|
|
5034
|
-
const { dbPath, requiredBytes } = input;
|
|
5035
|
-
let sqlite;
|
|
5036
|
-
try {
|
|
5037
|
-
sqlite = await import("node:sqlite");
|
|
5038
|
-
} catch (err) {
|
|
5039
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5040
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5041
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5042
|
-
}
|
|
5043
|
-
let autoVacuum = null;
|
|
5044
|
-
try {
|
|
5045
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5046
|
-
try {
|
|
5047
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5048
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5049
|
-
} finally {
|
|
5050
|
-
db.close();
|
|
5051
|
-
}
|
|
5052
|
-
} catch (err) {
|
|
5053
|
-
console.warn(
|
|
5054
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5055
|
-
);
|
|
5056
|
-
}
|
|
5057
|
-
if (autoVacuum !== 0) return null;
|
|
5058
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5059
|
-
}
|
|
5060
|
-
async function reclaimSessionDbSpace(input) {
|
|
5061
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5062
|
-
let sqlite;
|
|
5063
|
-
try {
|
|
5064
|
-
sqlite = await import("node:sqlite");
|
|
5065
|
-
} catch (err) {
|
|
5066
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5067
|
-
console.warn(
|
|
5068
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5069
|
-
);
|
|
5070
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5071
|
-
}
|
|
5072
|
-
const { DatabaseSync } = sqlite;
|
|
5073
|
-
let db;
|
|
5074
|
-
try {
|
|
5075
|
-
db = new DatabaseSync(dbPath);
|
|
5076
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5077
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5078
|
-
if (autoVacuum === 0) {
|
|
5079
|
-
if (!allowFullVacuum) {
|
|
5080
|
-
console.warn(
|
|
5081
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5082
|
-
);
|
|
5083
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5084
|
-
}
|
|
5085
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5086
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5087
|
-
if (skipReason !== null) {
|
|
5088
|
-
console.warn(
|
|
5089
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5090
|
-
);
|
|
5091
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5092
|
-
}
|
|
5093
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5094
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5095
|
-
db.exec("VACUUM");
|
|
5096
|
-
const afterBytes = readLogicalBytes(db);
|
|
5097
|
-
const checkpoint = readCheckpointResult(db);
|
|
5098
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5099
|
-
}
|
|
5100
|
-
if (autoVacuum === 2) {
|
|
5101
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5102
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5103
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5104
|
-
const afterBytes = readLogicalBytes(db);
|
|
5105
|
-
const checkpoint = readCheckpointResult(db);
|
|
5106
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5107
|
-
}
|
|
5108
|
-
console.warn(
|
|
5109
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5110
|
-
);
|
|
5111
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5112
|
-
} catch (err) {
|
|
5113
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5114
|
-
return {
|
|
5115
|
-
ok: false,
|
|
5116
|
-
skipped: "reclaim-error",
|
|
5117
|
-
detail: errorMessage(err)
|
|
5118
|
-
};
|
|
5119
|
-
} finally {
|
|
5120
|
-
db?.close();
|
|
5121
|
-
}
|
|
5122
|
-
}
|
|
5123
|
-
|
|
5124
5504
|
// src/lib/tunnel/connection.ts
|
|
5125
5505
|
import WebSocket2 from "ws";
|
|
5126
5506
|
|
|
@@ -5350,6 +5730,8 @@ var StreamForwarder = class {
|
|
|
5350
5730
|
if (done) break;
|
|
5351
5731
|
const chunk = Buffer.from(value);
|
|
5352
5732
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5733
|
+
await stream.pausePromise;
|
|
5734
|
+
if (ac.signal.aborted) break;
|
|
5353
5735
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5354
5736
|
if (this.binaryFramesSupported) {
|
|
5355
5737
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5392,6 +5774,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5392
5774
|
}
|
|
5393
5775
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5394
5776
|
var BASE_RECONNECT_DELAY = 500;
|
|
5777
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5395
5778
|
function getReconnectDelay(attempt) {
|
|
5396
5779
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5397
5780
|
const jitter = Math.random() * 1e3;
|
|
@@ -5463,10 +5846,27 @@ function connectTunnel(options) {
|
|
|
5463
5846
|
ws.close();
|
|
5464
5847
|
reject(new Error("Connection timeout"));
|
|
5465
5848
|
}, 3e4);
|
|
5849
|
+
let relaySilenceTimeout;
|
|
5850
|
+
let relaySilenceTimedOut = false;
|
|
5851
|
+
const clearRelaySilenceTimeout = () => {
|
|
5852
|
+
if (relaySilenceTimeout === void 0) return;
|
|
5853
|
+
clearTimeout(relaySilenceTimeout);
|
|
5854
|
+
relaySilenceTimeout = void 0;
|
|
5855
|
+
};
|
|
5856
|
+
const armRelaySilenceTimeout = () => {
|
|
5857
|
+
clearRelaySilenceTimeout();
|
|
5858
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
5859
|
+
relaySilenceTimeout = void 0;
|
|
5860
|
+
relaySilenceTimedOut = true;
|
|
5861
|
+
ws.terminate();
|
|
5862
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
5863
|
+
relaySilenceTimeout.unref?.();
|
|
5864
|
+
};
|
|
5466
5865
|
let upgradeRejection = null;
|
|
5467
5866
|
let upgradeRejectionReason = null;
|
|
5468
5867
|
ws.on("unexpected-response", (_req, res) => {
|
|
5469
5868
|
clearTimeout(connectionTimeout);
|
|
5869
|
+
clearRelaySilenceTimeout();
|
|
5470
5870
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5471
5871
|
upgradeRejectionReason = reason;
|
|
5472
5872
|
const chunks = [];
|
|
@@ -5496,6 +5896,7 @@ function connectTunnel(options) {
|
|
|
5496
5896
|
onInfo?.("WebSocket connection established");
|
|
5497
5897
|
});
|
|
5498
5898
|
ws.on("message", (data, isBinary) => {
|
|
5899
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5499
5900
|
if (isBinary) {
|
|
5500
5901
|
try {
|
|
5501
5902
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5529,6 +5930,7 @@ function connectTunnel(options) {
|
|
|
5529
5930
|
switch (message.type) {
|
|
5530
5931
|
case "connected": {
|
|
5531
5932
|
clearTimeout(connectionTimeout);
|
|
5933
|
+
armRelaySilenceTimeout();
|
|
5532
5934
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5533
5935
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5534
5936
|
onConnected?.(connectedAgentId);
|
|
@@ -5553,6 +5955,7 @@ function connectTunnel(options) {
|
|
|
5553
5955
|
});
|
|
5554
5956
|
ws.on("error", (error2) => {
|
|
5555
5957
|
clearTimeout(connectionTimeout);
|
|
5958
|
+
clearRelaySilenceTimeout();
|
|
5556
5959
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5557
5960
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5558
5961
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5564,7 +5967,8 @@ function connectTunnel(options) {
|
|
|
5564
5967
|
);
|
|
5565
5968
|
});
|
|
5566
5969
|
ws.on("close", (code, reason) => {
|
|
5567
|
-
|
|
5970
|
+
clearRelaySilenceTimeout();
|
|
5971
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5568
5972
|
forwarder.abortAll();
|
|
5569
5973
|
onDisconnected?.(code, reasonStr);
|
|
5570
5974
|
});
|
|
@@ -5572,9 +5976,10 @@ function connectTunnel(options) {
|
|
|
5572
5976
|
}
|
|
5573
5977
|
|
|
5574
5978
|
// src/lib/tunnel/runner-connection.ts
|
|
5575
|
-
var RunnerConnection = class {
|
|
5979
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5576
5980
|
opts;
|
|
5577
5981
|
sleep;
|
|
5982
|
+
now;
|
|
5578
5983
|
connection = null;
|
|
5579
5984
|
resolvedAgentId;
|
|
5580
5985
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5583,10 +5988,17 @@ var RunnerConnection = class {
|
|
|
5583
5988
|
reconnectPromise = null;
|
|
5584
5989
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5585
5990
|
reconnectAttempt = 0;
|
|
5991
|
+
generation = 0;
|
|
5992
|
+
disconnectedAtMs = null;
|
|
5993
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
5994
|
+
"No pong received",
|
|
5995
|
+
"Orphaned socket"
|
|
5996
|
+
]);
|
|
5586
5997
|
constructor(opts) {
|
|
5587
5998
|
this.opts = opts;
|
|
5588
5999
|
this.resolvedAgentId = opts.agentId;
|
|
5589
6000
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6001
|
+
this.now = opts.now ?? Date.now;
|
|
5590
6002
|
}
|
|
5591
6003
|
get agentId() {
|
|
5592
6004
|
return this.resolvedAgentId;
|
|
@@ -5622,14 +6034,24 @@ var RunnerConnection = class {
|
|
|
5622
6034
|
port: this.opts.port,
|
|
5623
6035
|
openCodePassword: this.opts.openCodePassword,
|
|
5624
6036
|
onConnected: (agentId) => {
|
|
6037
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5625
6038
|
this.reconnectAttempt = 0;
|
|
5626
6039
|
this.reconnecting = false;
|
|
5627
6040
|
this.resolvedAgentId = agentId;
|
|
5628
|
-
|
|
6041
|
+
this.generation++;
|
|
6042
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6043
|
+
this.disconnectedAtMs = null;
|
|
6044
|
+
events.onConnected(agentId, isReconnect, {
|
|
6045
|
+
generation: this.generation,
|
|
6046
|
+
attempt,
|
|
6047
|
+
downtimeMs
|
|
6048
|
+
});
|
|
5629
6049
|
},
|
|
5630
6050
|
onDisconnected: (code, reason) => {
|
|
5631
|
-
|
|
5632
|
-
|
|
6051
|
+
this.disconnectedAtMs = this.now();
|
|
6052
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6053
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6054
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5633
6055
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5634
6056
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5635
6057
|
});
|
|
@@ -5968,8 +6390,85 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5968
6390
|
};
|
|
5969
6391
|
}
|
|
5970
6392
|
|
|
6393
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6394
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6395
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6396
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6397
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6398
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6399
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6400
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6401
|
+
var BAND_RANK = {
|
|
6402
|
+
low: 1,
|
|
6403
|
+
very_low: 2,
|
|
6404
|
+
critical: 3
|
|
6405
|
+
};
|
|
6406
|
+
function bandFor(freeBytes) {
|
|
6407
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6408
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6409
|
+
return "low";
|
|
6410
|
+
}
|
|
6411
|
+
function formatBytes(bytes) {
|
|
6412
|
+
if (bytes === null) return "unavailable";
|
|
6413
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6414
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6415
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6416
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6417
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6418
|
+
}
|
|
6419
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6420
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6421
|
+
}
|
|
6422
|
+
function evaluateLowDiskWarning(input) {
|
|
6423
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6424
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6425
|
+
return { warning: null, state };
|
|
6426
|
+
}
|
|
6427
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6428
|
+
if (recovered) {
|
|
6429
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6430
|
+
}
|
|
6431
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6432
|
+
if (!low) return { warning: null, state };
|
|
6433
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6434
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6435
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6436
|
+
const worse = currentRank > previousRank;
|
|
6437
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6438
|
+
const nextState = {
|
|
6439
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6440
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6441
|
+
};
|
|
6442
|
+
return {
|
|
6443
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6444
|
+
state: nextState
|
|
6445
|
+
};
|
|
6446
|
+
}
|
|
6447
|
+
|
|
6448
|
+
// src/lib/opencode/session-store-health.ts
|
|
6449
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6450
|
+
function createSessionStoreHealth(options = {}) {
|
|
6451
|
+
const now = options.now ?? (() => Date.now());
|
|
6452
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6453
|
+
let lastUnwritableAt = null;
|
|
6454
|
+
let lastWritableAt = null;
|
|
6455
|
+
return {
|
|
6456
|
+
observeUnwritable: () => {
|
|
6457
|
+
lastUnwritableAt = now();
|
|
6458
|
+
},
|
|
6459
|
+
observeWritable: () => {
|
|
6460
|
+
lastWritableAt = now();
|
|
6461
|
+
},
|
|
6462
|
+
isUnwritable: () => {
|
|
6463
|
+
if (lastUnwritableAt === null) return false;
|
|
6464
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6465
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6466
|
+
}
|
|
6467
|
+
};
|
|
6468
|
+
}
|
|
6469
|
+
|
|
5971
6470
|
// src/lib/channels/driver.ts
|
|
5972
|
-
import { homedir as
|
|
6471
|
+
import { homedir as homedir6 } from "node:os";
|
|
5973
6472
|
|
|
5974
6473
|
// src/lib/runner-file-sync.ts
|
|
5975
6474
|
import { join as join9 } from "node:path";
|
|
@@ -6426,6 +6925,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6426
6925
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6427
6926
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6428
6927
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6928
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6429
6929
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6430
6930
|
var HEARTBEAT_MS = 6e4;
|
|
6431
6931
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6487,6 +6987,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6487
6987
|
maxActiveSessions;
|
|
6488
6988
|
watcherStallMs;
|
|
6489
6989
|
wedgeWarningIntervalMs;
|
|
6990
|
+
sessionStoreHealth;
|
|
6490
6991
|
openCodeClient;
|
|
6491
6992
|
/** Cache of conversationId → opencode sessionId. */
|
|
6492
6993
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6615,6 +7116,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6615
7116
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6616
7117
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6617
7118
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7119
|
+
/**
|
|
7120
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7121
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7122
|
+
* identical-text turn, so it never produces a correlation.
|
|
7123
|
+
*/
|
|
7124
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6618
7125
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6619
7126
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6620
7127
|
/**
|
|
@@ -6700,21 +7207,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6700
7207
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6701
7208
|
*/
|
|
6702
7209
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6703
|
-
/**
|
|
6704
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6705
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6706
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6707
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6708
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6709
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6710
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6711
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6712
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6713
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6714
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6715
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6716
|
-
*/
|
|
6717
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6718
7210
|
/**
|
|
6719
7211
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6720
7212
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6732,8 +7224,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6732
7224
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6733
7225
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6734
7226
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6735
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6736
|
-
*
|
|
7227
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7228
|
+
* the terminal-report fence and is never sent again).
|
|
6737
7229
|
*/
|
|
6738
7230
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6739
7231
|
/**
|
|
@@ -6852,10 +7344,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6852
7344
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6853
7345
|
this.now = config.now ?? (() => Date.now());
|
|
6854
7346
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6855
|
-
this.homeDir = config.homeDir ??
|
|
7347
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6856
7348
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6857
7349
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6858
7350
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7351
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6859
7352
|
}
|
|
6860
7353
|
get isV2() {
|
|
6861
7354
|
return this.openCodeClient.version === "v2";
|
|
@@ -6871,7 +7364,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6871
7364
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6872
7365
|
}
|
|
6873
7366
|
async listSessions() {
|
|
6874
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7367
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7368
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7369
|
+
});
|
|
6875
7370
|
}
|
|
6876
7371
|
async sessionExists(sessionId) {
|
|
6877
7372
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7048,6 +7543,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7048
7543
|
}
|
|
7049
7544
|
return ids;
|
|
7050
7545
|
}
|
|
7546
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7547
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7548
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7549
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7550
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7551
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7552
|
+
}
|
|
7553
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7554
|
+
}
|
|
7555
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7556
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7557
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7558
|
+
this.awaitingReadopt.delete(messageId);
|
|
7559
|
+
this.sessions.delete(conversationId);
|
|
7560
|
+
this.supersede(conversationId, sessionId);
|
|
7561
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7562
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7563
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7564
|
+
this.log({
|
|
7565
|
+
level: "error",
|
|
7566
|
+
message: errorMessage3,
|
|
7567
|
+
conversation_id: conversationId,
|
|
7568
|
+
message_id: messageId
|
|
7569
|
+
});
|
|
7570
|
+
const reportError = await this.reportUntrackableAck(
|
|
7571
|
+
conversationId,
|
|
7572
|
+
messageId,
|
|
7573
|
+
null,
|
|
7574
|
+
errorMessage3
|
|
7575
|
+
);
|
|
7576
|
+
if (reportError !== null) {
|
|
7577
|
+
this.log({
|
|
7578
|
+
level: "warn",
|
|
7579
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7580
|
+
conversation_id: conversationId,
|
|
7581
|
+
message_id: messageId
|
|
7582
|
+
});
|
|
7583
|
+
}
|
|
7584
|
+
if (source === "readopt") {
|
|
7585
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7586
|
+
}
|
|
7587
|
+
}
|
|
7051
7588
|
/**
|
|
7052
7589
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7053
7590
|
*
|
|
@@ -7227,6 +7764,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7227
7764
|
}
|
|
7228
7765
|
break;
|
|
7229
7766
|
}
|
|
7767
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7768
|
+
if (correlation) {
|
|
7769
|
+
correlation.attempts += 1;
|
|
7770
|
+
let correlatedId = null;
|
|
7771
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7772
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7773
|
+
conv,
|
|
7774
|
+
message,
|
|
7775
|
+
correlation.sessionId
|
|
7776
|
+
);
|
|
7777
|
+
if (polled.ok)
|
|
7778
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7779
|
+
} else if (correlation.knownUserIds === null) {
|
|
7780
|
+
this.log({
|
|
7781
|
+
level: "debug",
|
|
7782
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7783
|
+
conversation_id: conv.id,
|
|
7784
|
+
message_id: message.id
|
|
7785
|
+
});
|
|
7786
|
+
}
|
|
7787
|
+
if (correlatedId !== null) {
|
|
7788
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7789
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7790
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7791
|
+
this.dispatched.add(message.id);
|
|
7792
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7793
|
+
dispatched += 1;
|
|
7794
|
+
this.log({
|
|
7795
|
+
level: "info",
|
|
7796
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7797
|
+
conversation_id: conv.id,
|
|
7798
|
+
message_id: message.id
|
|
7799
|
+
});
|
|
7800
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7801
|
+
opencode_message_id: correlatedId
|
|
7802
|
+
});
|
|
7803
|
+
continue;
|
|
7804
|
+
}
|
|
7805
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7806
|
+
await this.handoffUntrackableCorrelation(
|
|
7807
|
+
conv.id,
|
|
7808
|
+
message.id,
|
|
7809
|
+
correlation.sessionId,
|
|
7810
|
+
correlation.attempts,
|
|
7811
|
+
"fresh"
|
|
7812
|
+
);
|
|
7813
|
+
break;
|
|
7814
|
+
}
|
|
7815
|
+
break;
|
|
7816
|
+
}
|
|
7230
7817
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7231
7818
|
if (effectiveOpencodeMessageId) {
|
|
7232
7819
|
const outcome = await this.resolveRedrive(
|
|
@@ -7248,6 +7835,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7248
7835
|
model: message.opencode_model ?? void 0
|
|
7249
7836
|
};
|
|
7250
7837
|
let opencodeMessageId;
|
|
7838
|
+
let preDispatchUserIds = null;
|
|
7251
7839
|
try {
|
|
7252
7840
|
this.log({
|
|
7253
7841
|
level: "info",
|
|
@@ -7277,7 +7865,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7277
7865
|
message.content,
|
|
7278
7866
|
options,
|
|
7279
7867
|
sendAttachments,
|
|
7280
|
-
this.openCodeClient
|
|
7868
|
+
this.openCodeClient,
|
|
7869
|
+
(ids) => {
|
|
7870
|
+
preDispatchUserIds = ids;
|
|
7871
|
+
}
|
|
7281
7872
|
)
|
|
7282
7873
|
);
|
|
7283
7874
|
} catch (err) {
|
|
@@ -7360,39 +7951,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7360
7951
|
if (this.isV2) {
|
|
7361
7952
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7362
7953
|
}
|
|
7363
|
-
|
|
7364
|
-
|
|
7365
|
-
|
|
7366
|
-
|
|
7367
|
-
|
|
7368
|
-
|
|
7369
|
-
message_id: message.id
|
|
7370
|
-
});
|
|
7371
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7372
|
-
continue;
|
|
7373
|
-
}
|
|
7374
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7375
|
-
this.sessions.delete(conv.id);
|
|
7376
|
-
this.supersede(conv.id, sessionId);
|
|
7377
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
7954
|
+
this.awaitingCorrelation.set(message.id, {
|
|
7955
|
+
sessionId,
|
|
7956
|
+
content: message.content,
|
|
7957
|
+
attempts: 0,
|
|
7958
|
+
knownUserIds: preDispatchUserIds
|
|
7959
|
+
});
|
|
7378
7960
|
this.log({
|
|
7379
|
-
level: "
|
|
7380
|
-
message:
|
|
7961
|
+
level: "warn",
|
|
7962
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7381
7963
|
conversation_id: conv.id,
|
|
7382
7964
|
message_id: message.id
|
|
7383
7965
|
});
|
|
7384
|
-
|
|
7385
|
-
this.log({
|
|
7386
|
-
level: "warn",
|
|
7387
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7388
|
-
conversation_id: conv.id,
|
|
7389
|
-
message_id: message.id
|
|
7390
|
-
});
|
|
7391
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7392
|
-
});
|
|
7966
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7393
7967
|
break;
|
|
7394
7968
|
}
|
|
7395
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7396
7969
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7397
7970
|
this.releasedOpencodeIds.delete(message.id);
|
|
7398
7971
|
this.dispatched.add(message.id);
|
|
@@ -7997,23 +8570,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7997
8570
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
7998
8571
|
return 1;
|
|
7999
8572
|
}
|
|
8000
|
-
/**
|
|
8001
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8002
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8003
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8004
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8005
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8006
|
-
* count, since a new session is a genuinely different attempt.
|
|
8007
|
-
*/
|
|
8008
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8009
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8010
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8011
|
-
existing.count += 1;
|
|
8012
|
-
return existing.count;
|
|
8013
|
-
}
|
|
8014
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8015
|
-
return 1;
|
|
8016
|
-
}
|
|
8017
8573
|
/** Record one queued give-up toward the never-started bound. */
|
|
8018
8574
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8019
8575
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8325,8 +8881,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8325
8881
|
b2LastDescendantCheckMs: 0,
|
|
8326
8882
|
b2RootOngoingHeldLogged: false,
|
|
8327
8883
|
b2AbandonedSignalled: false,
|
|
8884
|
+
b2AbandonedWithoutAnswer: false,
|
|
8328
8885
|
ambiguousPinnedSinceMs: 0,
|
|
8329
8886
|
ambiguousResolved: false,
|
|
8887
|
+
sessionStoreFailurePending: false,
|
|
8330
8888
|
neverStartedResolved: false
|
|
8331
8889
|
});
|
|
8332
8890
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8443,8 +9001,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8443
9001
|
b2LastDescendantCheckMs: 0,
|
|
8444
9002
|
b2RootOngoingHeldLogged: false,
|
|
8445
9003
|
b2AbandonedSignalled: false,
|
|
9004
|
+
b2AbandonedWithoutAnswer: false,
|
|
8446
9005
|
ambiguousPinnedSinceMs: 0,
|
|
8447
9006
|
ambiguousResolved: false,
|
|
9007
|
+
sessionStoreFailurePending: false,
|
|
8448
9008
|
neverStartedResolved: false
|
|
8449
9009
|
});
|
|
8450
9010
|
}
|
|
@@ -8858,6 +9418,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8858
9418
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8859
9419
|
this.neverStartedAttempts.delete(id);
|
|
8860
9420
|
}
|
|
9421
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9422
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9423
|
+
return;
|
|
9424
|
+
}
|
|
8861
9425
|
if (inFlight.neverStartedResolved) {
|
|
8862
9426
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8863
9427
|
inFlight.neverStartedResolved = false;
|
|
@@ -9017,10 +9581,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9017
9581
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9018
9582
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9019
9583
|
inFlight.b2AbandonedSignalled = false;
|
|
9584
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9020
9585
|
}
|
|
9021
9586
|
} else {
|
|
9022
9587
|
if (inFlight.b2AbandonedSignalled) {
|
|
9023
|
-
|
|
9588
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9589
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9590
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9591
|
+
sessionId,
|
|
9592
|
+
watcher,
|
|
9593
|
+
inFlight,
|
|
9594
|
+
messages,
|
|
9595
|
+
reply?.info?.finish ?? reply?.finish
|
|
9596
|
+
);
|
|
9597
|
+
} else {
|
|
9598
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9599
|
+
}
|
|
9024
9600
|
return;
|
|
9025
9601
|
}
|
|
9026
9602
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9037,17 +9613,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9037
9613
|
descendantOngoing,
|
|
9038
9614
|
rootOngoing
|
|
9039
9615
|
})) {
|
|
9616
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9617
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9618
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9619
|
+
watched_for_ms: pinnedForMs,
|
|
9620
|
+
run_state: "failed",
|
|
9621
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9622
|
+
});
|
|
9623
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9624
|
+
return;
|
|
9625
|
+
}
|
|
9040
9626
|
inFlight.b2AbandonedSignalled = true;
|
|
9627
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9628
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9629
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9630
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9631
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9632
|
+
watched_for_ms: pinnedForMs,
|
|
9633
|
+
run_state: "failed",
|
|
9634
|
+
finish,
|
|
9635
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9636
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9637
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9638
|
+
});
|
|
9639
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9640
|
+
sessionId,
|
|
9641
|
+
watcher,
|
|
9642
|
+
inFlight,
|
|
9643
|
+
messages,
|
|
9644
|
+
finish
|
|
9645
|
+
);
|
|
9646
|
+
return;
|
|
9647
|
+
}
|
|
9041
9648
|
this.log({
|
|
9042
9649
|
level: "warn",
|
|
9043
9650
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9044
9651
|
conversation_id: conv.id,
|
|
9045
9652
|
message_id: id
|
|
9046
9653
|
});
|
|
9047
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9048
9654
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9049
9655
|
watched_for_ms: pinnedForMs,
|
|
9050
|
-
finish
|
|
9656
|
+
finish,
|
|
9051
9657
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9052
9658
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9053
9659
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9095,6 +9701,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9095
9701
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9096
9702
|
sessionOngoing: ongoing
|
|
9097
9703
|
})) {
|
|
9704
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9705
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9706
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9707
|
+
watched_for_ms: pinnedForMs,
|
|
9708
|
+
run_state: "failed",
|
|
9709
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9710
|
+
});
|
|
9711
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9712
|
+
return;
|
|
9713
|
+
}
|
|
9098
9714
|
inFlight.ambiguousResolved = true;
|
|
9099
9715
|
this.log({
|
|
9100
9716
|
level: "warn",
|
|
@@ -9319,6 +9935,74 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9319
9935
|
}
|
|
9320
9936
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9321
9937
|
}
|
|
9938
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
9939
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
9940
|
+
const conv = watcher.conv;
|
|
9941
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
9942
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
9943
|
+
this.log({
|
|
9944
|
+
level: "warn",
|
|
9945
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
9946
|
+
conversation_id: conv.id,
|
|
9947
|
+
message_id: inFlight.evidentMessageId
|
|
9948
|
+
});
|
|
9949
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
9950
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
9951
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
9952
|
+
messages,
|
|
9953
|
+
inFlight.opencodeMessageId,
|
|
9954
|
+
inFlight.evidentMessageId
|
|
9955
|
+
);
|
|
9956
|
+
try {
|
|
9957
|
+
await this.markFailed(
|
|
9958
|
+
conv.id,
|
|
9959
|
+
inFlight.evidentMessageId,
|
|
9960
|
+
sessionId,
|
|
9961
|
+
error2,
|
|
9962
|
+
usage,
|
|
9963
|
+
void 0,
|
|
9964
|
+
usageAgentName,
|
|
9965
|
+
subagentInvocations
|
|
9966
|
+
);
|
|
9967
|
+
} catch (err) {
|
|
9968
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
9969
|
+
if (err instanceof ChannelTerminalError) {
|
|
9970
|
+
this.log({
|
|
9971
|
+
level: "warn",
|
|
9972
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9973
|
+
conversation_id: conv.id,
|
|
9974
|
+
message_id: inFlight.evidentMessageId
|
|
9975
|
+
});
|
|
9976
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9977
|
+
return;
|
|
9978
|
+
}
|
|
9979
|
+
if (this.now() >= inFlight.deadline) {
|
|
9980
|
+
this.log({
|
|
9981
|
+
level: "warn",
|
|
9982
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
9983
|
+
conversation_id: conv.id,
|
|
9984
|
+
message_id: inFlight.evidentMessageId
|
|
9985
|
+
});
|
|
9986
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9987
|
+
return;
|
|
9988
|
+
}
|
|
9989
|
+
this.log({
|
|
9990
|
+
level: "warn",
|
|
9991
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
9992
|
+
conversation_id: conv.id,
|
|
9993
|
+
message_id: inFlight.evidentMessageId
|
|
9994
|
+
});
|
|
9995
|
+
return;
|
|
9996
|
+
}
|
|
9997
|
+
inFlight.done = true;
|
|
9998
|
+
await this.reportSubagentAuthFailures(
|
|
9999
|
+
watcher.conv.id,
|
|
10000
|
+
inFlight.opencodeMessageId,
|
|
10001
|
+
inFlight.evidentMessageId,
|
|
10002
|
+
messages
|
|
10003
|
+
);
|
|
10004
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10005
|
+
}
|
|
9322
10006
|
/**
|
|
9323
10007
|
* Report a queued message as failed after the never-started bound. The accepted
|
|
9324
10008
|
* OpenCode copy is not cancelled, so the local fence remains on unsuccessful
|
|
@@ -9374,6 +10058,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9374
10058
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9375
10059
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9376
10060
|
}
|
|
10061
|
+
/**
|
|
10062
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10063
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10064
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10065
|
+
*/
|
|
10066
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10067
|
+
const conv = watcher.conv;
|
|
10068
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10069
|
+
this.log({
|
|
10070
|
+
level: "error",
|
|
10071
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10072
|
+
conversation_id: conv.id,
|
|
10073
|
+
message_id: inFlight.evidentMessageId
|
|
10074
|
+
});
|
|
10075
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10076
|
+
try {
|
|
10077
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10078
|
+
} catch (err) {
|
|
10079
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10080
|
+
if (err instanceof ChannelTerminalError) {
|
|
10081
|
+
this.log({
|
|
10082
|
+
level: "warn",
|
|
10083
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10084
|
+
conversation_id: conv.id,
|
|
10085
|
+
message_id: inFlight.evidentMessageId
|
|
10086
|
+
});
|
|
10087
|
+
this.recordReleasedOpencodeId(
|
|
10088
|
+
inFlight.evidentMessageId,
|
|
10089
|
+
sessionId,
|
|
10090
|
+
inFlight.opencodeMessageId
|
|
10091
|
+
);
|
|
10092
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10093
|
+
return;
|
|
10094
|
+
}
|
|
10095
|
+
if (this.now() >= inFlight.deadline) {
|
|
10096
|
+
this.log({
|
|
10097
|
+
level: "warn",
|
|
10098
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10099
|
+
conversation_id: conv.id,
|
|
10100
|
+
message_id: inFlight.evidentMessageId
|
|
10101
|
+
});
|
|
10102
|
+
this.recordReleasedOpencodeId(
|
|
10103
|
+
inFlight.evidentMessageId,
|
|
10104
|
+
sessionId,
|
|
10105
|
+
inFlight.opencodeMessageId
|
|
10106
|
+
);
|
|
10107
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10108
|
+
return;
|
|
10109
|
+
}
|
|
10110
|
+
this.log({
|
|
10111
|
+
level: "warn",
|
|
10112
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10113
|
+
conversation_id: conv.id,
|
|
10114
|
+
message_id: inFlight.evidentMessageId
|
|
10115
|
+
});
|
|
10116
|
+
return;
|
|
10117
|
+
}
|
|
10118
|
+
inFlight.done = true;
|
|
10119
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10120
|
+
}
|
|
9377
10121
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9378
10122
|
/**
|
|
9379
10123
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9412,8 +10156,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9412
10156
|
}
|
|
9413
10157
|
}
|
|
9414
10158
|
}
|
|
9415
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10159
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9416
10160
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10161
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10162
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10163
|
+
this.awaitingCorrelation.delete(id);
|
|
10164
|
+
this.log({
|
|
10165
|
+
level: "debug",
|
|
10166
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10167
|
+
message_id: id
|
|
10168
|
+
});
|
|
10169
|
+
}
|
|
10170
|
+
}
|
|
9417
10171
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9418
10172
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9419
10173
|
this.untrackableAck.delete(id);
|
|
@@ -9456,7 +10210,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9456
10210
|
bySession.set(row.opencode_session_id, list);
|
|
9457
10211
|
}
|
|
9458
10212
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9459
|
-
let messages;
|
|
10213
|
+
let messages = null;
|
|
9460
10214
|
try {
|
|
9461
10215
|
if (this.isV2) {
|
|
9462
10216
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9465,9 +10219,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9465
10219
|
level: "warn",
|
|
9466
10220
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9467
10221
|
});
|
|
9468
|
-
|
|
10222
|
+
} else {
|
|
10223
|
+
messages = snapshot;
|
|
9469
10224
|
}
|
|
9470
|
-
messages = snapshot;
|
|
9471
10225
|
} else {
|
|
9472
10226
|
const polled = await pollSessionMessagesForRedrive(
|
|
9473
10227
|
this.port,
|
|
@@ -9480,15 +10234,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9480
10234
|
level: "warn",
|
|
9481
10235
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9482
10236
|
});
|
|
9483
|
-
|
|
10237
|
+
} else {
|
|
10238
|
+
messages = polled.messages;
|
|
9484
10239
|
}
|
|
9485
|
-
messages = polled.messages;
|
|
9486
10240
|
}
|
|
9487
10241
|
} catch (err) {
|
|
9488
10242
|
this.log({
|
|
9489
10243
|
level: "warn",
|
|
9490
10244
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9491
10245
|
});
|
|
10246
|
+
}
|
|
10247
|
+
if (messages === null) {
|
|
10248
|
+
for (const row of sessionRows) {
|
|
10249
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10250
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10251
|
+
}
|
|
10252
|
+
}
|
|
9492
10253
|
continue;
|
|
9493
10254
|
}
|
|
9494
10255
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9498,6 +10259,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9498
10259
|
}
|
|
9499
10260
|
}
|
|
9500
10261
|
}
|
|
10262
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10263
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10264
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10265
|
+
if (!correlation) return false;
|
|
10266
|
+
correlation.attempts += 1;
|
|
10267
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10268
|
+
if (correlatedId !== null) {
|
|
10269
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10270
|
+
const conv = this.convForRow(sessionId, row);
|
|
10271
|
+
const message = this.queuedMessageForRow(row);
|
|
10272
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10273
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10274
|
+
this.dispatched.add(row.id);
|
|
10275
|
+
this.readopted.add(row.id);
|
|
10276
|
+
this.ensureWatcherRunning(sessionId);
|
|
10277
|
+
this.log({
|
|
10278
|
+
level: "info",
|
|
10279
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10280
|
+
conversation_id: row.conversation_id,
|
|
10281
|
+
message_id: row.id
|
|
10282
|
+
});
|
|
10283
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10284
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10285
|
+
opencode_message_id: correlatedId
|
|
10286
|
+
});
|
|
10287
|
+
return true;
|
|
10288
|
+
}
|
|
10289
|
+
if (correlation.knownUserIds === null) {
|
|
10290
|
+
this.log({
|
|
10291
|
+
level: "debug",
|
|
10292
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10293
|
+
conversation_id: row.conversation_id,
|
|
10294
|
+
message_id: row.id
|
|
10295
|
+
});
|
|
10296
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10297
|
+
this.log({
|
|
10298
|
+
level: "warn",
|
|
10299
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10300
|
+
conversation_id: row.conversation_id,
|
|
10301
|
+
message_id: row.id
|
|
10302
|
+
});
|
|
10303
|
+
}
|
|
10304
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10305
|
+
await this.handoffUntrackableCorrelation(
|
|
10306
|
+
row.conversation_id,
|
|
10307
|
+
row.id,
|
|
10308
|
+
correlation.sessionId,
|
|
10309
|
+
correlation.attempts,
|
|
10310
|
+
"readopt"
|
|
10311
|
+
);
|
|
10312
|
+
}
|
|
10313
|
+
return true;
|
|
10314
|
+
}
|
|
9501
10315
|
/**
|
|
9502
10316
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9503
10317
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9529,6 +10343,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9529
10343
|
});
|
|
9530
10344
|
return;
|
|
9531
10345
|
}
|
|
10346
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9532
10347
|
const ocId = row.opencode_message_id;
|
|
9533
10348
|
const state = messageRunState(messages, ocId ?? "");
|
|
9534
10349
|
if (state === "done") {
|
|
@@ -9797,8 +10612,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9797
10612
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9798
10613
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9799
10614
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9800
|
-
* re-entry)
|
|
9801
|
-
*
|
|
10615
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10616
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10617
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9802
10618
|
*
|
|
9803
10619
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9804
10620
|
* `processed_at` (Invariant 1).
|
|
@@ -9861,6 +10677,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9861
10677
|
);
|
|
9862
10678
|
}
|
|
9863
10679
|
let ocId;
|
|
10680
|
+
let preDispatchUserIds = null;
|
|
9864
10681
|
try {
|
|
9865
10682
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9866
10683
|
sessionId,
|
|
@@ -9870,7 +10687,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9870
10687
|
row.content,
|
|
9871
10688
|
options,
|
|
9872
10689
|
sendAttachments,
|
|
9873
|
-
this.openCodeClient
|
|
10690
|
+
this.openCodeClient,
|
|
10691
|
+
(ids) => {
|
|
10692
|
+
preDispatchUserIds = ids;
|
|
10693
|
+
}
|
|
9874
10694
|
)
|
|
9875
10695
|
);
|
|
9876
10696
|
} catch (err) {
|
|
@@ -9951,39 +10771,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9951
10771
|
}
|
|
9952
10772
|
if (ocId === null) {
|
|
9953
10773
|
this.awaitingReadopt.delete(row.id);
|
|
9954
|
-
|
|
9955
|
-
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
|
|
9960
|
-
this.log({
|
|
9961
|
-
level: "error",
|
|
9962
|
-
message: errorMessage3,
|
|
9963
|
-
conversation_id: row.conversation_id,
|
|
9964
|
-
message_id: row.id
|
|
9965
|
-
});
|
|
9966
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9967
|
-
this.log({
|
|
9968
|
-
level: "warn",
|
|
9969
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9970
|
-
conversation_id: row.conversation_id,
|
|
9971
|
-
message_id: row.id
|
|
9972
|
-
});
|
|
9973
|
-
});
|
|
9974
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9975
|
-
return;
|
|
9976
|
-
}
|
|
10774
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10775
|
+
sessionId,
|
|
10776
|
+
content: row.content,
|
|
10777
|
+
attempts: 0,
|
|
10778
|
+
knownUserIds: preDispatchUserIds
|
|
10779
|
+
});
|
|
9977
10780
|
this.log({
|
|
9978
10781
|
level: "warn",
|
|
9979
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10782
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9980
10783
|
conversation_id: row.conversation_id,
|
|
9981
10784
|
message_id: row.id
|
|
9982
10785
|
});
|
|
9983
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9984
10786
|
return;
|
|
9985
10787
|
}
|
|
9986
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9987
10788
|
const processedAtMs = this.processedAtMs(row);
|
|
9988
10789
|
this.registerReadopted(
|
|
9989
10790
|
readoptConv,
|
|
@@ -11279,8 +12080,16 @@ function checkNonInteractivePortConflict(port, isPortInUseFn) {
|
|
|
11279
12080
|
);
|
|
11280
12081
|
}
|
|
11281
12082
|
}
|
|
12083
|
+
function unknownPasswordError(port) {
|
|
12084
|
+
return new Error(
|
|
12085
|
+
`OpenCode is already running on port ${port}, but this runner cannot authenticate to it. It may have been started with OPENCODE_SERVER_PASSWORD set in its environment. Free the port or pass --port.`
|
|
12086
|
+
);
|
|
12087
|
+
}
|
|
11282
12088
|
async function ensureOpenCodeRunning(ctx) {
|
|
11283
12089
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
12090
|
+
if (healthCheck.authFailed) {
|
|
12091
|
+
throw unknownPasswordError(ctx.port);
|
|
12092
|
+
}
|
|
11284
12093
|
if (healthCheck.healthy) {
|
|
11285
12094
|
return {
|
|
11286
12095
|
port: ctx.port,
|
|
@@ -11331,6 +12140,14 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
11331
12140
|
const proc = await startOpenCode(ctx.port, { inheritStdio: ctx.inheritStdio });
|
|
11332
12141
|
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
11333
12142
|
if (!health.healthy) {
|
|
12143
|
+
if (health.authFailed) {
|
|
12144
|
+
return {
|
|
12145
|
+
port: ctx.port,
|
|
12146
|
+
process: proc,
|
|
12147
|
+
version: null,
|
|
12148
|
+
notReadyReason: "it is answering with an authentication challenge, likely because OPENCODE_SERVER_PASSWORD is set in the environment of the process that started it; a longer start timeout will not resolve this"
|
|
12149
|
+
};
|
|
12150
|
+
}
|
|
11334
12151
|
return {
|
|
11335
12152
|
port: ctx.port,
|
|
11336
12153
|
process: proc,
|
|
@@ -11400,6 +12217,9 @@ Port ${port} is already in use.`));
|
|
|
11400
12217
|
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
11401
12218
|
if (!health.healthy) {
|
|
11402
12219
|
spinner.fail("Failed to start OpenCode");
|
|
12220
|
+
if (health.authFailed) {
|
|
12221
|
+
throw unknownPasswordError(port);
|
|
12222
|
+
}
|
|
11403
12223
|
throw new Error("OpenCode failed to start");
|
|
11404
12224
|
}
|
|
11405
12225
|
spinner.stop();
|
|
@@ -11412,7 +12232,7 @@ Port ${port} is already in use.`));
|
|
|
11412
12232
|
import chalk6 from "chalk";
|
|
11413
12233
|
import ora3 from "ora";
|
|
11414
12234
|
import { select as select3 } from "@inquirer/prompts";
|
|
11415
|
-
function
|
|
12235
|
+
function unknownPasswordError2(port) {
|
|
11416
12236
|
return new Error(
|
|
11417
12237
|
`OpenCode V2 (opencode2) is already running on port ${port} with a password this runner doesn't know. Free the port or pass --port.`
|
|
11418
12238
|
);
|
|
@@ -11421,7 +12241,7 @@ var INTERACTIVE_START_TIMEOUT_MS2 = 3e4;
|
|
|
11421
12241
|
async function ensureOpenCode2Running(ctx) {
|
|
11422
12242
|
const initialHealth = await checkOpenCode2Health(ctx.port);
|
|
11423
12243
|
if (initialHealth.authFailed) {
|
|
11424
|
-
throw
|
|
12244
|
+
throw unknownPasswordError2(ctx.port);
|
|
11425
12245
|
}
|
|
11426
12246
|
if (initialHealth.healthy) {
|
|
11427
12247
|
return {
|
|
@@ -11434,7 +12254,7 @@ async function ensureOpenCode2Running(ctx) {
|
|
|
11434
12254
|
}
|
|
11435
12255
|
if (!isOpenCode2Installed()) {
|
|
11436
12256
|
throw new Error(
|
|
11437
|
-
|
|
12257
|
+
`OpenCode V2 (opencode2) is not installed. Install it with: npm install -g ${OPENCODE2_PACKAGE_SPEC}`
|
|
11438
12258
|
);
|
|
11439
12259
|
}
|
|
11440
12260
|
let port = ctx.port;
|
|
@@ -12086,6 +12906,7 @@ function createCredentialSync({
|
|
|
12086
12906
|
|
|
12087
12907
|
// src/commands/run.ts
|
|
12088
12908
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
12909
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12089
12910
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12090
12911
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12091
12912
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12241,7 +13062,13 @@ function logActivity(state, entry) {
|
|
|
12241
13062
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12242
13063
|
if (!meetsThreshold(state, level)) return;
|
|
12243
13064
|
forwardRunnerActivity(
|
|
12244
|
-
{
|
|
13065
|
+
{
|
|
13066
|
+
level,
|
|
13067
|
+
message: entry.message,
|
|
13068
|
+
error: entry.error,
|
|
13069
|
+
metadata: entry.metadata,
|
|
13070
|
+
...entry.source ? { source: entry.source } : {}
|
|
13071
|
+
},
|
|
12245
13072
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12246
13073
|
);
|
|
12247
13074
|
const fullEntry = {
|
|
@@ -12263,7 +13090,7 @@ function logActivity(state, entry) {
|
|
|
12263
13090
|
}
|
|
12264
13091
|
function reportSessionDbRecovery(state) {
|
|
12265
13092
|
try {
|
|
12266
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13093
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12267
13094
|
for (const record of report.records) {
|
|
12268
13095
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12269
13096
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12298,7 +13125,9 @@ function displayStatus(state) {
|
|
|
12298
13125
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12299
13126
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12300
13127
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12301
|
-
const detail = last ? chalk7.dim(
|
|
13128
|
+
const detail = last ? chalk7.dim(
|
|
13129
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13130
|
+
) : "";
|
|
12302
13131
|
const agent = state.agentName ?? state.agentId;
|
|
12303
13132
|
console.log(
|
|
12304
13133
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12317,7 +13146,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12317
13146
|
logActivity(state, {
|
|
12318
13147
|
type: "info",
|
|
12319
13148
|
level: "warn",
|
|
12320
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13149
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12321
13150
|
});
|
|
12322
13151
|
}
|
|
12323
13152
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12342,7 +13171,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12342
13171
|
}
|
|
12343
13172
|
const result = await importOpenCodeV2Credentials({
|
|
12344
13173
|
request: client.request,
|
|
12345
|
-
homeDir:
|
|
13174
|
+
homeDir: homedir7(),
|
|
12346
13175
|
directory,
|
|
12347
13176
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12348
13177
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12550,9 +13379,9 @@ async function driveChannels(state, driver) {
|
|
|
12550
13379
|
}
|
|
12551
13380
|
}
|
|
12552
13381
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12553
|
-
var
|
|
13382
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12554
13383
|
function shouldWarnForReclaimSkip(reason) {
|
|
12555
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13384
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12556
13385
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12557
13386
|
if (!version2) return false;
|
|
12558
13387
|
const major = Number(version2[1]);
|
|
@@ -12561,7 +13390,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12561
13390
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12562
13391
|
}
|
|
12563
13392
|
function sessionDbPath() {
|
|
12564
|
-
return join11(
|
|
13393
|
+
return join11(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
13394
|
+
}
|
|
13395
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
13396
|
+
logActivity(state, {
|
|
13397
|
+
...entry,
|
|
13398
|
+
source: "cli.session-cleanup",
|
|
13399
|
+
metadata: { ...entry.metadata, phase }
|
|
13400
|
+
});
|
|
12565
13401
|
}
|
|
12566
13402
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12567
13403
|
const record = {
|
|
@@ -12602,69 +13438,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12602
13438
|
async function runSweep(state, driver, config) {
|
|
12603
13439
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12604
13440
|
try {
|
|
12605
|
-
const
|
|
12606
|
-
|
|
13441
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13442
|
+
const sessions = enumeration.sessions;
|
|
13443
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13444
|
+
if (enumerationReason !== null) {
|
|
12607
13445
|
logActivity(state, {
|
|
12608
13446
|
type: "info",
|
|
12609
|
-
|
|
13447
|
+
level: "warn",
|
|
13448
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12610
13449
|
});
|
|
12611
|
-
return;
|
|
12612
13450
|
}
|
|
12613
|
-
const toDelete = selectSessionsToDelete(
|
|
12614
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12615
|
-
{
|
|
12616
|
-
maxAgeMs: config.maxAgeMs,
|
|
12617
|
-
maxCount: config.maxCount,
|
|
12618
|
-
nowMs: Date.now(),
|
|
12619
|
-
protectedIds: driver.protectedSessionIds()
|
|
12620
|
-
}
|
|
12621
|
-
);
|
|
12622
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12623
13451
|
let deleted = 0;
|
|
12624
13452
|
let failed = 0;
|
|
13453
|
+
let failed4xx = 0;
|
|
13454
|
+
let failed5xx = 0;
|
|
13455
|
+
let failedOtherStatus = 0;
|
|
13456
|
+
let failedThrew = 0;
|
|
12625
13457
|
let skippedNewlyActive = 0;
|
|
12626
|
-
|
|
12627
|
-
|
|
12628
|
-
|
|
12629
|
-
|
|
12630
|
-
|
|
12631
|
-
|
|
12632
|
-
|
|
12633
|
-
|
|
13458
|
+
let remaining = 0;
|
|
13459
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13460
|
+
if (sessions !== null) {
|
|
13461
|
+
const toDelete = selectSessionsToDelete(
|
|
13462
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13463
|
+
{
|
|
13464
|
+
maxAgeMs: config.maxAgeMs,
|
|
13465
|
+
maxCount: config.maxCount,
|
|
13466
|
+
nowMs: Date.now(),
|
|
13467
|
+
protectedIds: protectedNow
|
|
13468
|
+
}
|
|
13469
|
+
);
|
|
13470
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13471
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13472
|
+
protectedNow = driver.protectedSessionIds();
|
|
13473
|
+
for (const id of deleteCandidates) {
|
|
13474
|
+
if (protectedNow.has(id)) {
|
|
13475
|
+
skippedNewlyActive++;
|
|
13476
|
+
logActivity(state, {
|
|
13477
|
+
type: "info",
|
|
13478
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13479
|
+
});
|
|
13480
|
+
continue;
|
|
13481
|
+
}
|
|
13482
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13483
|
+
if (outcome.ok) {
|
|
13484
|
+
deleted++;
|
|
13485
|
+
} else if (outcome.status === "threw") {
|
|
13486
|
+
failed++;
|
|
13487
|
+
failedThrew++;
|
|
13488
|
+
} else {
|
|
13489
|
+
failed++;
|
|
13490
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13491
|
+
if (statusClass === 4) failed4xx++;
|
|
13492
|
+
else if (statusClass === 5) failed5xx++;
|
|
13493
|
+
else failedOtherStatus++;
|
|
13494
|
+
}
|
|
12634
13495
|
}
|
|
12635
|
-
|
|
12636
|
-
|
|
13496
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13497
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13498
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13499
|
+
logActivity(state, {
|
|
13500
|
+
type: "info",
|
|
13501
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13502
|
+
});
|
|
12637
13503
|
}
|
|
12638
|
-
const
|
|
12639
|
-
|
|
12640
|
-
|
|
12641
|
-
|
|
12642
|
-
|
|
12643
|
-
|
|
13504
|
+
const sessionMetadata = {
|
|
13505
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13506
|
+
sessions_deleted: deleted,
|
|
13507
|
+
sessions_failed: failed,
|
|
13508
|
+
sessions_failed_4xx: failed4xx,
|
|
13509
|
+
sessions_failed_5xx: failed5xx,
|
|
13510
|
+
sessions_failed_other: failedOtherStatus,
|
|
13511
|
+
sessions_failed_threw: failedThrew,
|
|
13512
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13513
|
+
sessions_remaining: remaining,
|
|
13514
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13515
|
+
};
|
|
12644
13516
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12645
13517
|
dbPath: sessionDbPath(),
|
|
12646
|
-
|
|
13518
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13519
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13520
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12647
13521
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12648
13522
|
});
|
|
12649
13523
|
if (reclaimResult.ok) {
|
|
13524
|
+
state.sessionStoreHealth.observeWritable();
|
|
12650
13525
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12651
13526
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12652
13527
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12653
|
-
|
|
13528
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12654
13529
|
type: "info",
|
|
12655
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${
|
|
13530
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
13531
|
+
metadata: {
|
|
13532
|
+
outcome: "reclaimed",
|
|
13533
|
+
mode: reclaimResult.mode,
|
|
13534
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
13535
|
+
after_bytes: reclaimResult.afterBytes,
|
|
13536
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
13537
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
13538
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13539
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13540
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13541
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13542
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13543
|
+
...sessionMetadata
|
|
13544
|
+
}
|
|
12656
13545
|
});
|
|
12657
13546
|
} else {
|
|
12658
|
-
|
|
13547
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13548
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13549
|
+
}
|
|
13550
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12659
13551
|
type: "info",
|
|
12660
|
-
|
|
13552
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13553
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
13554
|
+
metadata: {
|
|
13555
|
+
outcome: "reclaim-skipped",
|
|
13556
|
+
reason: reclaimResult.skipped,
|
|
13557
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
13558
|
+
...sessionMetadata
|
|
13559
|
+
}
|
|
12661
13560
|
});
|
|
12662
13561
|
}
|
|
12663
13562
|
} catch (error2) {
|
|
12664
13563
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12665
|
-
|
|
13564
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12666
13565
|
type: "error",
|
|
12667
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
13566
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
13567
|
+
metadata: {
|
|
13568
|
+
outcome: "sweep-failed",
|
|
13569
|
+
reason: "sweep-error"
|
|
13570
|
+
}
|
|
12668
13571
|
});
|
|
12669
13572
|
}
|
|
12670
13573
|
}
|
|
@@ -12680,14 +13583,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12680
13583
|
for (const warning2 of config.warnings) {
|
|
12681
13584
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12682
13585
|
}
|
|
12683
|
-
const dbBytes = statSessionDbBytes(
|
|
13586
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12684
13587
|
void (async () => {
|
|
12685
13588
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12686
13589
|
if (reclaimAvailability !== null) {
|
|
12687
|
-
|
|
13590
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12688
13591
|
type: "info",
|
|
12689
13592
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12690
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
13593
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
13594
|
+
metadata: {
|
|
13595
|
+
outcome: "preflight-skipped",
|
|
13596
|
+
reason: reclaimAvailability.reason,
|
|
13597
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
13598
|
+
}
|
|
12691
13599
|
});
|
|
12692
13600
|
}
|
|
12693
13601
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12704,9 +13612,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12704
13612
|
);
|
|
12705
13613
|
});
|
|
12706
13614
|
if (!config.enabled) return;
|
|
12707
|
-
|
|
13615
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12708
13616
|
type: "info",
|
|
12709
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
13617
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
13618
|
+
metadata: {
|
|
13619
|
+
outcome: "armed",
|
|
13620
|
+
interval_ms: config.intervalMs,
|
|
13621
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
13622
|
+
max_count: config.maxCount ?? null
|
|
13623
|
+
}
|
|
12710
13624
|
});
|
|
12711
13625
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12712
13626
|
const firstSweep = setTimeout(
|
|
@@ -12885,7 +13799,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12885
13799
|
});
|
|
12886
13800
|
return;
|
|
12887
13801
|
}
|
|
12888
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
13802
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12889
13803
|
state.stopResourceUsageSampling = stop;
|
|
12890
13804
|
let consecutiveFailures = 0;
|
|
12891
13805
|
const tick = async () => {
|
|
@@ -12898,6 +13812,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12898
13812
|
message: `Resource usage collection: ${warning2}`
|
|
12899
13813
|
});
|
|
12900
13814
|
}
|
|
13815
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13816
|
+
...usage,
|
|
13817
|
+
nowMs: Date.now(),
|
|
13818
|
+
state: state.lowDiskWarningState
|
|
13819
|
+
});
|
|
13820
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13821
|
+
if (lowDiskDecision.warning !== null) {
|
|
13822
|
+
logActivity(state, {
|
|
13823
|
+
type: "info",
|
|
13824
|
+
level: "warn",
|
|
13825
|
+
message: lowDiskDecision.warning
|
|
13826
|
+
});
|
|
13827
|
+
}
|
|
12901
13828
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12902
13829
|
if (result.ok) {
|
|
12903
13830
|
if (consecutiveFailures > 0) {
|
|
@@ -13110,7 +14037,7 @@ async function run(options) {
|
|
|
13110
14037
|
let fileSyncDirectories;
|
|
13111
14038
|
try {
|
|
13112
14039
|
logLevel = resolveLogLevel(options);
|
|
13113
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14040
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13114
14041
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13115
14042
|
throw new Error(
|
|
13116
14043
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13144,6 +14071,7 @@ async function run(options) {
|
|
|
13144
14071
|
opencodeClient: null,
|
|
13145
14072
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13146
14073
|
sessionDbProvenanceAnomaly: false,
|
|
14074
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13147
14075
|
opencodeProcess: null,
|
|
13148
14076
|
stopOpenCodeLogTail: null,
|
|
13149
14077
|
litestreamProcess: null,
|
|
@@ -13161,10 +14089,12 @@ async function run(options) {
|
|
|
13161
14089
|
openaiUsageRearm: null,
|
|
13162
14090
|
resourceUsageTimer: null,
|
|
13163
14091
|
stopResourceUsageSampling: null,
|
|
14092
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13164
14093
|
credentialSync: null,
|
|
13165
14094
|
authHeader: ""
|
|
13166
14095
|
};
|
|
13167
14096
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14097
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13168
14098
|
if (options.credentialSyncMarker) {
|
|
13169
14099
|
state.credentialSync = createCredentialSync({
|
|
13170
14100
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13350,13 +14280,21 @@ async function run(options) {
|
|
|
13350
14280
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13351
14281
|
}
|
|
13352
14282
|
state.credentialSync?.arm();
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14283
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14284
|
+
() => ({
|
|
13356
14285
|
agentId: state.agentId,
|
|
13357
14286
|
authHeader: state.authHeader
|
|
13358
|
-
})
|
|
13359
|
-
|
|
14287
|
+
}),
|
|
14288
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14289
|
+
);
|
|
14290
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14291
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14292
|
+
openCodeActivityForwarder
|
|
14293
|
+
);
|
|
14294
|
+
state.stopOpenCodeLogTail = () => {
|
|
14295
|
+
openCodeActivityForwarder.flush();
|
|
14296
|
+
openCodeLogTail.stop();
|
|
14297
|
+
};
|
|
13360
14298
|
let sessionDbVerifyFatal = false;
|
|
13361
14299
|
if (!options.restoreSessionDb) {
|
|
13362
14300
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
@@ -13368,6 +14306,12 @@ async function run(options) {
|
|
|
13368
14306
|
"Skipping session-DB restore: OpenCode is already serving this database",
|
|
13369
14307
|
"debug"
|
|
13370
14308
|
);
|
|
14309
|
+
} else if (health.authFailed) {
|
|
14310
|
+
log2(
|
|
14311
|
+
state,
|
|
14312
|
+
"Skipping session-DB restore: OpenCode answered with an authentication challenge; assuming this database is in use",
|
|
14313
|
+
"debug"
|
|
14314
|
+
);
|
|
13371
14315
|
} else {
|
|
13372
14316
|
const result = await restoreAndVerifySessionDb({
|
|
13373
14317
|
dbPath: sessionDbPath(),
|
|
@@ -13467,7 +14411,7 @@ async function run(options) {
|
|
|
13467
14411
|
const provenance = checkSessionDbProvenance({
|
|
13468
14412
|
dbPath: sessionDbPath(),
|
|
13469
14413
|
currentVersion: state.opencodeVersion,
|
|
13470
|
-
homeDir:
|
|
14414
|
+
homeDir: homedir7(),
|
|
13471
14415
|
env: process.env
|
|
13472
14416
|
});
|
|
13473
14417
|
if (provenance.anomaly) {
|
|
@@ -13634,10 +14578,11 @@ async function run(options) {
|
|
|
13634
14578
|
getAuthHeader: () => state.authHeader,
|
|
13635
14579
|
conversationFilter: state.conversationFilter,
|
|
13636
14580
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14581
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13637
14582
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13638
14583
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13639
14584
|
fileSyncDirectories,
|
|
13640
|
-
homeDir:
|
|
14585
|
+
homeDir: homedir7(),
|
|
13641
14586
|
maxActiveSessions,
|
|
13642
14587
|
log: (entry) => (
|
|
13643
14588
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13659,12 +14604,13 @@ async function run(options) {
|
|
|
13659
14604
|
openCodePassword: state.opencodePassword,
|
|
13660
14605
|
isRunning: () => state.running,
|
|
13661
14606
|
events: {
|
|
13662
|
-
onConnected: (agentId, isReconnect) => {
|
|
14607
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13663
14608
|
state.connected = true;
|
|
13664
14609
|
state.agentId = agentId;
|
|
14610
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13665
14611
|
logActivity(state, {
|
|
13666
14612
|
type: "info",
|
|
13667
|
-
message:
|
|
14613
|
+
message: connectionMessage
|
|
13668
14614
|
});
|
|
13669
14615
|
if (options.tunnelReadyFile) {
|
|
13670
14616
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13685,7 +14631,12 @@ async function run(options) {
|
|
|
13685
14631
|
version: state.opencodeVersion,
|
|
13686
14632
|
major: state.opencodeApiVersion,
|
|
13687
14633
|
connected: state.opencodeConnected
|
|
13688
|
-
})
|
|
14634
|
+
}),
|
|
14635
|
+
connection_generation: transition.generation,
|
|
14636
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14637
|
+
reconnect_attempt: transition.attempt,
|
|
14638
|
+
downtime_ms: transition.downtimeMs
|
|
14639
|
+
} : {}
|
|
13689
14640
|
});
|
|
13690
14641
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13691
14642
|
if (state.interactive) displayStatus(state);
|
|
@@ -13707,13 +14658,18 @@ async function run(options) {
|
|
|
13707
14658
|
if (state.interactive) displayStatus(state);
|
|
13708
14659
|
});
|
|
13709
14660
|
},
|
|
13710
|
-
onDisconnected: (code, reason) => {
|
|
14661
|
+
onDisconnected: (code, reason, transition) => {
|
|
13711
14662
|
state.connected = false;
|
|
14663
|
+
const safeReason = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13712
14664
|
logActivity(state, {
|
|
13713
14665
|
type: "info",
|
|
13714
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14666
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason})`
|
|
14667
|
+
});
|
|
14668
|
+
emitAgentDisconnected(state.agentId, {
|
|
14669
|
+
code,
|
|
14670
|
+
reason: safeReason,
|
|
14671
|
+
connection_generation: transition.generation
|
|
13715
14672
|
});
|
|
13716
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13717
14673
|
if (state.interactive) displayStatus(state);
|
|
13718
14674
|
},
|
|
13719
14675
|
onError: (error2) => {
|
|
@@ -13813,7 +14769,7 @@ async function run(options) {
|
|
|
13813
14769
|
},
|
|
13814
14770
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13815
14771
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13816
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14772
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13817
14773
|
firstDelayMs: firstReportDelayMs,
|
|
13818
14774
|
nextDelayMs: usageReportDelayMs,
|
|
13819
14775
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13856,7 +14812,7 @@ async function run(options) {
|
|
|
13856
14812
|
}
|
|
13857
14813
|
|
|
13858
14814
|
// src/index.ts
|
|
13859
|
-
var
|
|
14815
|
+
var version = getCliVersion();
|
|
13860
14816
|
var program = new Command();
|
|
13861
14817
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13862
14818
|
"--endpoint <url>",
|