@evident-ai/cli 3.5.2-dev.da3340e → 3.5.2-dev.df1ae49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +781 -240
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -1158,7 +1157,7 @@ import ora4 from "ora";
|
|
|
1158
1157
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1158
|
|
|
1160
1159
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.2-dev.
|
|
1160
|
+
var CLI_VERSION = (true ? "3.5.2-dev.df1ae49" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1161
|
function getCliVersion() {
|
|
1163
1162
|
return CLI_VERSION;
|
|
1164
1163
|
}
|
|
@@ -1172,6 +1171,23 @@ var authProvider = null;
|
|
|
1172
1171
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1172
|
authProvider = provider;
|
|
1174
1173
|
}
|
|
1174
|
+
var diagnosticsReporter = null;
|
|
1175
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1176
|
+
diagnosticsReporter = reporter;
|
|
1177
|
+
}
|
|
1178
|
+
function reportTelemetryDiagnostic(message) {
|
|
1179
|
+
if (!diagnosticsReporter) {
|
|
1180
|
+
console.error(message);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
try {
|
|
1184
|
+
diagnosticsReporter(message);
|
|
1185
|
+
} catch (error2) {
|
|
1186
|
+
console.error(
|
|
1187
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1175
1191
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1192
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1193
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1253,9 @@ async function flushEvents() {
|
|
|
1237
1253
|
signal: controller.signal
|
|
1238
1254
|
});
|
|
1239
1255
|
if (!response.ok) {
|
|
1240
|
-
|
|
1256
|
+
reportTelemetryDiagnostic(
|
|
1257
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1258
|
+
);
|
|
1241
1259
|
}
|
|
1242
1260
|
} finally {
|
|
1243
1261
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1265,9 @@ async function flushEvents() {
|
|
|
1247
1265
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1266
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1267
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1268
|
+
reportTelemetryDiagnostic(
|
|
1269
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1270
|
+
);
|
|
1251
1271
|
lastFlushFailureLoggedAt = now;
|
|
1252
1272
|
suppressedFlushFailureCount = 0;
|
|
1253
1273
|
} else {
|
|
@@ -1300,6 +1320,219 @@ var EventTypes = {
|
|
|
1300
1320
|
|
|
1301
1321
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
1322
|
import { homedir as homedir2 } from "node:os";
|
|
1323
|
+
|
|
1324
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1325
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1326
|
+
import { dirname as dirname2 } from "node:path";
|
|
1327
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1328
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1329
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1330
|
+
function errorMessage(error2) {
|
|
1331
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1332
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1333
|
+
}
|
|
1334
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1335
|
+
const messages = [];
|
|
1336
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1337
|
+
let current = error2;
|
|
1338
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1339
|
+
visited.add(current);
|
|
1340
|
+
if (typeof current === "string") {
|
|
1341
|
+
messages.push(current);
|
|
1342
|
+
break;
|
|
1343
|
+
}
|
|
1344
|
+
if (current instanceof Error) {
|
|
1345
|
+
messages.push(current.name, current.message);
|
|
1346
|
+
current = current.cause;
|
|
1347
|
+
continue;
|
|
1348
|
+
}
|
|
1349
|
+
if (typeof current === "object") {
|
|
1350
|
+
const record = current;
|
|
1351
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1352
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1353
|
+
current = record.cause;
|
|
1354
|
+
continue;
|
|
1355
|
+
}
|
|
1356
|
+
break;
|
|
1357
|
+
}
|
|
1358
|
+
return messages.some(
|
|
1359
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1360
|
+
message
|
|
1361
|
+
)
|
|
1362
|
+
);
|
|
1363
|
+
}
|
|
1364
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1365
|
+
try {
|
|
1366
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1367
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1368
|
+
if (availableBytes < requiredBytes) {
|
|
1369
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1370
|
+
}
|
|
1371
|
+
return null;
|
|
1372
|
+
} catch (err) {
|
|
1373
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
function readLogicalBytes(db) {
|
|
1377
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1378
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1379
|
+
return pageCount * pageSize;
|
|
1380
|
+
}
|
|
1381
|
+
function readFreelistCount(db) {
|
|
1382
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1383
|
+
}
|
|
1384
|
+
function readCheckpointResult(db) {
|
|
1385
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1386
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1387
|
+
}
|
|
1388
|
+
async function probeReclaimAvailability(input) {
|
|
1389
|
+
const { dbPath, requiredBytes } = input;
|
|
1390
|
+
let sqlite;
|
|
1391
|
+
try {
|
|
1392
|
+
sqlite = await import("node:sqlite");
|
|
1393
|
+
} catch (err) {
|
|
1394
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1395
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1396
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1397
|
+
}
|
|
1398
|
+
let autoVacuum = null;
|
|
1399
|
+
try {
|
|
1400
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1401
|
+
try {
|
|
1402
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1403
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1404
|
+
} finally {
|
|
1405
|
+
db.close();
|
|
1406
|
+
}
|
|
1407
|
+
} catch (err) {
|
|
1408
|
+
console.warn(
|
|
1409
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1410
|
+
);
|
|
1411
|
+
}
|
|
1412
|
+
if (autoVacuum !== 0) return null;
|
|
1413
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1414
|
+
}
|
|
1415
|
+
async function reclaimSessionDbSpace(input) {
|
|
1416
|
+
const {
|
|
1417
|
+
dbPath,
|
|
1418
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1419
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1420
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1421
|
+
now = () => performance.now(),
|
|
1422
|
+
allowFullVacuum = true
|
|
1423
|
+
} = input;
|
|
1424
|
+
let sqlite;
|
|
1425
|
+
try {
|
|
1426
|
+
sqlite = await import("node:sqlite");
|
|
1427
|
+
} catch (err) {
|
|
1428
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1429
|
+
console.warn(
|
|
1430
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1431
|
+
);
|
|
1432
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1433
|
+
}
|
|
1434
|
+
const { DatabaseSync } = sqlite;
|
|
1435
|
+
let db;
|
|
1436
|
+
try {
|
|
1437
|
+
db = new DatabaseSync(dbPath);
|
|
1438
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1439
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1440
|
+
if (autoVacuum === 0) {
|
|
1441
|
+
if (!allowFullVacuum) {
|
|
1442
|
+
console.warn(
|
|
1443
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1444
|
+
);
|
|
1445
|
+
return {
|
|
1446
|
+
ok: false,
|
|
1447
|
+
skipped: "full-vacuum-blocked",
|
|
1448
|
+
detail: "a session turn is live"
|
|
1449
|
+
};
|
|
1450
|
+
}
|
|
1451
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1452
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1453
|
+
if (skipReason !== null) {
|
|
1454
|
+
console.warn(
|
|
1455
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1456
|
+
);
|
|
1457
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1458
|
+
}
|
|
1459
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1460
|
+
const freelistBefore = readFreelistCount(db);
|
|
1461
|
+
const startedAt = now();
|
|
1462
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1463
|
+
db.exec("VACUUM");
|
|
1464
|
+
const afterBytes = readLogicalBytes(db);
|
|
1465
|
+
const freelistAfter = readFreelistCount(db);
|
|
1466
|
+
const checkpoint = readCheckpointResult(db);
|
|
1467
|
+
return {
|
|
1468
|
+
ok: true,
|
|
1469
|
+
mode: "convert",
|
|
1470
|
+
beforeBytes,
|
|
1471
|
+
afterBytes,
|
|
1472
|
+
freelistBefore,
|
|
1473
|
+
freelistAfter,
|
|
1474
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1475
|
+
stoppedBy: "freelist-drained",
|
|
1476
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1477
|
+
checkpoint
|
|
1478
|
+
};
|
|
1479
|
+
}
|
|
1480
|
+
if (autoVacuum === 2) {
|
|
1481
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1482
|
+
const freelistBefore = readFreelistCount(db);
|
|
1483
|
+
const startedAt = now();
|
|
1484
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1485
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1486
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1487
|
+
let pagesAttempted = 0;
|
|
1488
|
+
let freelistAfter = freelistBefore;
|
|
1489
|
+
let stoppedBy = "freelist-drained";
|
|
1490
|
+
while (freelistAfter > 0) {
|
|
1491
|
+
if (pagesAttempted >= pageLimit) {
|
|
1492
|
+
stoppedBy = "page-ceiling";
|
|
1493
|
+
break;
|
|
1494
|
+
}
|
|
1495
|
+
if (now() - startedAt >= budget) {
|
|
1496
|
+
stoppedBy = "time-budget";
|
|
1497
|
+
break;
|
|
1498
|
+
}
|
|
1499
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1500
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1501
|
+
pagesAttempted += pages;
|
|
1502
|
+
freelistAfter = readFreelistCount(db);
|
|
1503
|
+
}
|
|
1504
|
+
const afterBytes = readLogicalBytes(db);
|
|
1505
|
+
const checkpoint = readCheckpointResult(db);
|
|
1506
|
+
return {
|
|
1507
|
+
ok: true,
|
|
1508
|
+
mode: "incremental",
|
|
1509
|
+
beforeBytes,
|
|
1510
|
+
afterBytes,
|
|
1511
|
+
freelistBefore,
|
|
1512
|
+
freelistAfter,
|
|
1513
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1514
|
+
stoppedBy,
|
|
1515
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1516
|
+
checkpoint
|
|
1517
|
+
};
|
|
1518
|
+
}
|
|
1519
|
+
console.warn(
|
|
1520
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1521
|
+
);
|
|
1522
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1523
|
+
} catch (err) {
|
|
1524
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1525
|
+
return {
|
|
1526
|
+
ok: false,
|
|
1527
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1528
|
+
detail: errorMessage(err)
|
|
1529
|
+
};
|
|
1530
|
+
} finally {
|
|
1531
|
+
db?.close();
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1303
1536
|
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1304
1537
|
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1305
1538
|
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
@@ -1317,25 +1550,81 @@ function parseOpenCodeLogLine(line) {
|
|
|
1317
1550
|
const level = levelMatch[1].toUpperCase();
|
|
1318
1551
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1319
1552
|
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1320
|
-
|
|
1553
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1554
|
+
const parsed = {
|
|
1555
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1556
|
+
sessionID: sessionMatch?.[1]
|
|
1557
|
+
};
|
|
1558
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1559
|
+
if (message !== void 0) parsed.message = message;
|
|
1560
|
+
return parsed;
|
|
1321
1561
|
}
|
|
1322
1562
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1323
|
-
|
|
1563
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1564
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1324
1565
|
let buffer = Buffer.alloc(0);
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1566
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1567
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1568
|
+
const metadata = {
|
|
1569
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1570
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1571
|
+
};
|
|
1328
1572
|
forwardRunnerActivity(
|
|
1329
1573
|
{
|
|
1330
1574
|
level: parsed.level,
|
|
1331
1575
|
error: line,
|
|
1332
|
-
metadata:
|
|
1576
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1333
1577
|
source: "opencode"
|
|
1334
1578
|
},
|
|
1335
1579
|
getContext()
|
|
1336
1580
|
);
|
|
1337
1581
|
};
|
|
1338
|
-
|
|
1582
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1583
|
+
for (const [key, state] of repeatStates) {
|
|
1584
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1585
|
+
continue;
|
|
1586
|
+
}
|
|
1587
|
+
if (state.repeatsSuppressed > 0) {
|
|
1588
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1589
|
+
}
|
|
1590
|
+
repeatStates.delete(key);
|
|
1591
|
+
}
|
|
1592
|
+
};
|
|
1593
|
+
const flushLine = (line) => {
|
|
1594
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1595
|
+
try {
|
|
1596
|
+
onSessionStoreWriteFailure();
|
|
1597
|
+
} catch (err) {
|
|
1598
|
+
reportTelemetryDiagnostic(
|
|
1599
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1600
|
+
);
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1604
|
+
if (!parsed) return;
|
|
1605
|
+
const now = Date.now();
|
|
1606
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}`;
|
|
1607
|
+
expireRepeatStates(now, key);
|
|
1608
|
+
if (key === void 0) {
|
|
1609
|
+
forwardLine(line, parsed);
|
|
1610
|
+
return;
|
|
1611
|
+
}
|
|
1612
|
+
const previous = repeatStates.get(key);
|
|
1613
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1614
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1615
|
+
repeatStates.set(key, {
|
|
1616
|
+
line,
|
|
1617
|
+
parsed,
|
|
1618
|
+
lastForwardedAt: now,
|
|
1619
|
+
repeatsSuppressed: 0
|
|
1620
|
+
});
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1623
|
+
previous.line = line;
|
|
1624
|
+
previous.parsed = parsed;
|
|
1625
|
+
previous.repeatsSuppressed++;
|
|
1626
|
+
};
|
|
1627
|
+
const forward = ((chunk) => {
|
|
1339
1628
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1340
1629
|
let newlineIndex;
|
|
1341
1630
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1346,7 +1635,16 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1346
1635
|
flushLine(buffer.toString("utf-8"));
|
|
1347
1636
|
buffer = Buffer.alloc(0);
|
|
1348
1637
|
}
|
|
1638
|
+
});
|
|
1639
|
+
forward.flush = () => {
|
|
1640
|
+
for (const state of repeatStates.values()) {
|
|
1641
|
+
if (state.repeatsSuppressed > 0) {
|
|
1642
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
repeatStates.clear();
|
|
1349
1646
|
};
|
|
1647
|
+
return forward;
|
|
1350
1648
|
}
|
|
1351
1649
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1352
1650
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
@@ -1372,7 +1670,7 @@ function redactHomeDirectory(message) {
|
|
|
1372
1670
|
}
|
|
1373
1671
|
return result + message.slice(searchFrom);
|
|
1374
1672
|
}
|
|
1375
|
-
function
|
|
1673
|
+
function redactDiagnosticText(message) {
|
|
1376
1674
|
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1377
1675
|
return redactHomeDirectory(redacted);
|
|
1378
1676
|
}
|
|
@@ -1384,14 +1682,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1384
1682
|
if (!metadata) return {};
|
|
1385
1683
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1386
1684
|
if (Object.keys(metadata).length > entries.length) {
|
|
1387
|
-
|
|
1388
|
-
`
|
|
1685
|
+
reportTelemetryDiagnostic(
|
|
1686
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1389
1687
|
);
|
|
1390
1688
|
}
|
|
1391
1689
|
const sanitised = [];
|
|
1392
1690
|
for (const [key, value] of entries) {
|
|
1393
1691
|
if (typeof value === "string") {
|
|
1394
|
-
sanitised.push([
|
|
1692
|
+
sanitised.push([
|
|
1693
|
+
key,
|
|
1694
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1695
|
+
]);
|
|
1395
1696
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1396
1697
|
sanitised.push([key, value]);
|
|
1397
1698
|
}
|
|
@@ -1409,8 +1710,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1409
1710
|
}
|
|
1410
1711
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1411
1712
|
if (window.windowDroppedCount > 0) {
|
|
1412
|
-
|
|
1413
|
-
`
|
|
1713
|
+
reportTelemetryDiagnostic(
|
|
1714
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1414
1715
|
);
|
|
1415
1716
|
}
|
|
1416
1717
|
window.windowStartedAt = now;
|
|
@@ -1420,8 +1721,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1420
1721
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1421
1722
|
window.windowDroppedCount++;
|
|
1422
1723
|
if (window.windowDroppedCount === 1) {
|
|
1423
|
-
|
|
1424
|
-
`
|
|
1724
|
+
reportTelemetryDiagnostic(
|
|
1725
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1425
1726
|
);
|
|
1426
1727
|
}
|
|
1427
1728
|
return false;
|
|
@@ -1436,7 +1737,7 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1436
1737
|
if (!context.agentId || !context.authHeader) return;
|
|
1437
1738
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1438
1739
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1439
|
-
const message = truncate(
|
|
1740
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1440
1741
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1441
1742
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1442
1743
|
message,
|
|
@@ -1444,8 +1745,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1444
1745
|
agentId: context.agentId
|
|
1445
1746
|
});
|
|
1446
1747
|
} catch (err) {
|
|
1447
|
-
|
|
1448
|
-
`
|
|
1748
|
+
reportTelemetryDiagnostic(
|
|
1749
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1449
1750
|
);
|
|
1450
1751
|
}
|
|
1451
1752
|
}
|
|
@@ -1735,8 +2036,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1735
2036
|
|
|
1736
2037
|
// src/lib/http-timeout.ts
|
|
1737
2038
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2039
|
+
var RequestTimeoutError = class extends Error {
|
|
2040
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2041
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2042
|
+
this.operation = operation;
|
|
2043
|
+
this.timeoutMs = timeoutMs;
|
|
2044
|
+
this.elapsedMs = elapsedMs;
|
|
2045
|
+
this.name = "TimeoutError";
|
|
2046
|
+
}
|
|
2047
|
+
};
|
|
2048
|
+
function describeRequestOperation(input, init) {
|
|
2049
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2050
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2051
|
+
let path;
|
|
2052
|
+
try {
|
|
2053
|
+
path = new URL(rawUrl).pathname;
|
|
2054
|
+
} catch (error2) {
|
|
2055
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2056
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2057
|
+
}
|
|
2058
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2059
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2060
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2061
|
+
return `${method} ${maskedPath}`;
|
|
2062
|
+
}
|
|
1738
2063
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1739
|
-
return ((input, init) =>
|
|
2064
|
+
return (async (input, init) => {
|
|
2065
|
+
const startedAt = Date.now();
|
|
2066
|
+
try {
|
|
2067
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2068
|
+
} catch (error2) {
|
|
2069
|
+
if (error2?.name === "TimeoutError") {
|
|
2070
|
+
throw new RequestTimeoutError(
|
|
2071
|
+
describeRequestOperation(input, init),
|
|
2072
|
+
timeoutMs,
|
|
2073
|
+
Date.now() - startedAt
|
|
2074
|
+
);
|
|
2075
|
+
}
|
|
2076
|
+
throw error2;
|
|
2077
|
+
}
|
|
2078
|
+
});
|
|
1740
2079
|
}
|
|
1741
2080
|
|
|
1742
2081
|
// src/lib/opencode/client.ts
|
|
@@ -1890,6 +2229,7 @@ function isSessionActivelyGenerating(messages) {
|
|
|
1890
2229
|
if (roleOf(last) !== "assistant") return false;
|
|
1891
2230
|
return completedOf(last) == null;
|
|
1892
2231
|
}
|
|
2232
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1893
2233
|
function sessionLastActivityMs(session) {
|
|
1894
2234
|
const candidates = [
|
|
1895
2235
|
session.time?.updated,
|
|
@@ -1904,30 +2244,60 @@ function sessionLastActivityMs(session) {
|
|
|
1904
2244
|
}
|
|
1905
2245
|
return null;
|
|
1906
2246
|
}
|
|
1907
|
-
|
|
1908
|
-
|
|
2247
|
+
function sessionListPath(options) {
|
|
2248
|
+
const params = new URLSearchParams();
|
|
2249
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2250
|
+
const query = params.toString();
|
|
2251
|
+
return query ? `/session?${query}` : "/session";
|
|
2252
|
+
}
|
|
2253
|
+
async function listSessionPage(port, client, options) {
|
|
2254
|
+
const path = sessionListPath(options);
|
|
1909
2255
|
try {
|
|
1910
|
-
const res = await requestWithClient(port, client,
|
|
2256
|
+
const res = await requestWithClient(port, client, path);
|
|
1911
2257
|
if (!res.ok) return null;
|
|
1912
2258
|
const body = await res.json();
|
|
1913
2259
|
return Array.isArray(body) ? body : null;
|
|
1914
2260
|
} catch (error2) {
|
|
1915
2261
|
console.error(
|
|
1916
|
-
`[
|
|
2262
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1917
2263
|
);
|
|
1918
2264
|
return null;
|
|
1919
2265
|
}
|
|
1920
2266
|
}
|
|
2267
|
+
async function listSessions(port, client, options) {
|
|
2268
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2269
|
+
return listSessionPage(port, client, options);
|
|
2270
|
+
}
|
|
2271
|
+
async function listAllSessions(port, client) {
|
|
2272
|
+
if (client?.version === "v2") {
|
|
2273
|
+
const sessions2 = await listV2Sessions(client);
|
|
2274
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2275
|
+
}
|
|
2276
|
+
const sessions = await listSessionPage(port, client, {
|
|
2277
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2278
|
+
});
|
|
2279
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2280
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2281
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2282
|
+
}
|
|
2283
|
+
return { sessions };
|
|
2284
|
+
}
|
|
1921
2285
|
async function deleteSession(port, id, client) {
|
|
1922
2286
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1923
2287
|
try {
|
|
1924
|
-
const res = await requestWithClient(
|
|
1925
|
-
|
|
2288
|
+
const res = await requestWithClient(
|
|
2289
|
+
port,
|
|
2290
|
+
client,
|
|
2291
|
+
`/session/${id}`,
|
|
2292
|
+
{ method: "DELETE" },
|
|
2293
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2294
|
+
);
|
|
2295
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1926
2296
|
} catch (error2) {
|
|
1927
2297
|
console.error(
|
|
1928
2298
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1929
2299
|
);
|
|
1930
|
-
return false;
|
|
2300
|
+
return { ok: false, status: "threw" };
|
|
1931
2301
|
}
|
|
1932
2302
|
}
|
|
1933
2303
|
async function sessionExists(port, id, client) {
|
|
@@ -3060,13 +3430,17 @@ async function listV2Sessions(client) {
|
|
|
3060
3430
|
}
|
|
3061
3431
|
async function deleteV2Session(client, sessionId) {
|
|
3062
3432
|
try {
|
|
3063
|
-
await client.request(
|
|
3064
|
-
|
|
3433
|
+
const response = await client.request(
|
|
3434
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3435
|
+
{ method: "DELETE" },
|
|
3436
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3437
|
+
);
|
|
3438
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3065
3439
|
} catch (error2) {
|
|
3066
3440
|
console.error(
|
|
3067
3441
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3068
3442
|
);
|
|
3069
|
-
return false;
|
|
3443
|
+
return { ok: false, status: "threw" };
|
|
3070
3444
|
}
|
|
3071
3445
|
}
|
|
3072
3446
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3397,16 +3771,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3397
3771
|
].some((name) => headers.has(name));
|
|
3398
3772
|
}
|
|
3399
3773
|
async function getOpenAiUsage(port, client) {
|
|
3774
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3400
3775
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3401
3776
|
if (!credentials2) {
|
|
3402
3777
|
throw new OpenAiUsageError(
|
|
3403
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3778
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3404
3779
|
"no_credentials"
|
|
3405
3780
|
);
|
|
3406
3781
|
}
|
|
3407
3782
|
if (credentials2.expiresAt < Date.now()) {
|
|
3408
3783
|
throw new OpenAiUsageError(
|
|
3409
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3784
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3410
3785
|
"credentials_expired"
|
|
3411
3786
|
);
|
|
3412
3787
|
}
|
|
@@ -3687,9 +4062,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3687
4062
|
|
|
3688
4063
|
// src/lib/opencode/session-db-boot.ts
|
|
3689
4064
|
import { spawn as spawn2 } from "node:child_process";
|
|
3690
|
-
import { mkdirSync, statSync as
|
|
4065
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
3691
4066
|
import { homedir as homedir4 } from "node:os";
|
|
3692
|
-
import { dirname as
|
|
4067
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3693
4068
|
|
|
3694
4069
|
// src/lib/runner-synchroniser.ts
|
|
3695
4070
|
import { spawn } from "node:child_process";
|
|
@@ -3814,7 +4189,7 @@ function clearMarker(options) {
|
|
|
3814
4189
|
function markNoReplicate(options, message) {
|
|
3815
4190
|
if (options.noReplicateMarker) {
|
|
3816
4191
|
try {
|
|
3817
|
-
mkdirSync(
|
|
4192
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3818
4193
|
writeFileSync(options.noReplicateMarker, "");
|
|
3819
4194
|
} catch (error2) {
|
|
3820
4195
|
options.log(
|
|
@@ -3939,7 +4314,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3939
4314
|
return null;
|
|
3940
4315
|
}
|
|
3941
4316
|
try {
|
|
3942
|
-
if (
|
|
4317
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3943
4318
|
} catch (error2) {
|
|
3944
4319
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3945
4320
|
options.log(
|
|
@@ -3969,7 +4344,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3969
4344
|
return null;
|
|
3970
4345
|
}
|
|
3971
4346
|
try {
|
|
3972
|
-
mkdirSync(
|
|
4347
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3973
4348
|
writeFileSync(configPath, rendered.stdout);
|
|
3974
4349
|
} catch (error2) {
|
|
3975
4350
|
options.log(
|
|
@@ -4130,7 +4505,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4130
4505
|
}
|
|
4131
4506
|
function fileExists(path) {
|
|
4132
4507
|
try {
|
|
4133
|
-
|
|
4508
|
+
statSync3(path);
|
|
4134
4509
|
return true;
|
|
4135
4510
|
} catch (error2) {
|
|
4136
4511
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4219,7 +4594,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4219
4594
|
// src/lib/opencode/session-db-provenance.ts
|
|
4220
4595
|
import { createRequire } from "node:module";
|
|
4221
4596
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4222
|
-
import { dirname as
|
|
4597
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4223
4598
|
var require2 = createRequire(import.meta.url);
|
|
4224
4599
|
function readSessionDbMigrationIds(dbPath) {
|
|
4225
4600
|
let db;
|
|
@@ -4285,7 +4660,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4285
4660
|
}
|
|
4286
4661
|
function saveSessionDbProvenanceState(path, state) {
|
|
4287
4662
|
try {
|
|
4288
|
-
mkdirSync2(
|
|
4663
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4289
4664
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4290
4665
|
`, "utf8");
|
|
4291
4666
|
} catch (error2) {
|
|
@@ -4866,7 +5241,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4866
5241
|
toDelete.push(s.id);
|
|
4867
5242
|
}
|
|
4868
5243
|
}
|
|
4869
|
-
|
|
5244
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5245
|
+
return toDelete.sort(
|
|
5246
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5247
|
+
);
|
|
4870
5248
|
}
|
|
4871
5249
|
var DEFAULT_INTERVAL = "1h";
|
|
4872
5250
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4926,13 +5304,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4926
5304
|
}
|
|
4927
5305
|
|
|
4928
5306
|
// src/lib/opencode/session-db-size.ts
|
|
4929
|
-
import { statSync as
|
|
5307
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4930
5308
|
import { join as join6 } from "node:path";
|
|
4931
5309
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4932
5310
|
function statSessionDbBytes(homeDir) {
|
|
4933
5311
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4934
5312
|
try {
|
|
4935
|
-
return
|
|
5313
|
+
return statSync4(dbPath).size;
|
|
4936
5314
|
} catch (err) {
|
|
4937
5315
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4938
5316
|
if (!isMissingFile) {
|
|
@@ -4958,7 +5336,7 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4958
5336
|
}
|
|
4959
5337
|
|
|
4960
5338
|
// src/lib/opencode/log-tail.ts
|
|
4961
|
-
import { statSync as
|
|
5339
|
+
import { statSync as statSync5 } from "node:fs";
|
|
4962
5340
|
import { homedir as homedir5 } from "node:os";
|
|
4963
5341
|
import { join as join7 } from "node:path";
|
|
4964
5342
|
import { open as open2, stat } from "node:fs/promises";
|
|
@@ -4981,7 +5359,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4981
5359
|
let inode = null;
|
|
4982
5360
|
let baselineReady = true;
|
|
4983
5361
|
try {
|
|
4984
|
-
const initial =
|
|
5362
|
+
const initial = statSync5(logPath);
|
|
4985
5363
|
offset = initial.size;
|
|
4986
5364
|
inode = initial.ino;
|
|
4987
5365
|
} catch (error2) {
|
|
@@ -5040,129 +5418,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5040
5418
|
};
|
|
5041
5419
|
}
|
|
5042
5420
|
|
|
5043
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5044
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5045
|
-
import { dirname as dirname4 } from "node:path";
|
|
5046
|
-
function errorMessage(error2) {
|
|
5047
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5048
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5049
|
-
}
|
|
5050
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5051
|
-
try {
|
|
5052
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5053
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5054
|
-
if (availableBytes < requiredBytes) {
|
|
5055
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5056
|
-
}
|
|
5057
|
-
return null;
|
|
5058
|
-
} catch (err) {
|
|
5059
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5060
|
-
}
|
|
5061
|
-
}
|
|
5062
|
-
function readLogicalBytes(db) {
|
|
5063
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5064
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5065
|
-
return pageCount * pageSize;
|
|
5066
|
-
}
|
|
5067
|
-
function readCheckpointResult(db) {
|
|
5068
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5069
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5070
|
-
}
|
|
5071
|
-
async function probeReclaimAvailability(input) {
|
|
5072
|
-
const { dbPath, requiredBytes } = input;
|
|
5073
|
-
let sqlite;
|
|
5074
|
-
try {
|
|
5075
|
-
sqlite = await import("node:sqlite");
|
|
5076
|
-
} catch (err) {
|
|
5077
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5078
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5079
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5080
|
-
}
|
|
5081
|
-
let autoVacuum = null;
|
|
5082
|
-
try {
|
|
5083
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5084
|
-
try {
|
|
5085
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5086
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5087
|
-
} finally {
|
|
5088
|
-
db.close();
|
|
5089
|
-
}
|
|
5090
|
-
} catch (err) {
|
|
5091
|
-
console.warn(
|
|
5092
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5093
|
-
);
|
|
5094
|
-
}
|
|
5095
|
-
if (autoVacuum !== 0) return null;
|
|
5096
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5097
|
-
}
|
|
5098
|
-
async function reclaimSessionDbSpace(input) {
|
|
5099
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5100
|
-
let sqlite;
|
|
5101
|
-
try {
|
|
5102
|
-
sqlite = await import("node:sqlite");
|
|
5103
|
-
} catch (err) {
|
|
5104
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5105
|
-
console.warn(
|
|
5106
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5107
|
-
);
|
|
5108
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5109
|
-
}
|
|
5110
|
-
const { DatabaseSync } = sqlite;
|
|
5111
|
-
let db;
|
|
5112
|
-
try {
|
|
5113
|
-
db = new DatabaseSync(dbPath);
|
|
5114
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5115
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5116
|
-
if (autoVacuum === 0) {
|
|
5117
|
-
if (!allowFullVacuum) {
|
|
5118
|
-
console.warn(
|
|
5119
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5120
|
-
);
|
|
5121
|
-
return {
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "full-vacuum-blocked",
|
|
5124
|
-
detail: "a session turn is live"
|
|
5125
|
-
};
|
|
5126
|
-
}
|
|
5127
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5128
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5129
|
-
if (skipReason !== null) {
|
|
5130
|
-
console.warn(
|
|
5131
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5132
|
-
);
|
|
5133
|
-
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
5134
|
-
}
|
|
5135
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5136
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5137
|
-
db.exec("VACUUM");
|
|
5138
|
-
const afterBytes = readLogicalBytes(db);
|
|
5139
|
-
const checkpoint = readCheckpointResult(db);
|
|
5140
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5141
|
-
}
|
|
5142
|
-
if (autoVacuum === 2) {
|
|
5143
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5144
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5145
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5146
|
-
const afterBytes = readLogicalBytes(db);
|
|
5147
|
-
const checkpoint = readCheckpointResult(db);
|
|
5148
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5149
|
-
}
|
|
5150
|
-
console.warn(
|
|
5151
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5152
|
-
);
|
|
5153
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5154
|
-
} catch (err) {
|
|
5155
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5156
|
-
return {
|
|
5157
|
-
ok: false,
|
|
5158
|
-
skipped: "reclaim-error",
|
|
5159
|
-
detail: errorMessage(err)
|
|
5160
|
-
};
|
|
5161
|
-
} finally {
|
|
5162
|
-
db?.close();
|
|
5163
|
-
}
|
|
5164
|
-
}
|
|
5165
|
-
|
|
5166
5421
|
// src/lib/tunnel/connection.ts
|
|
5167
5422
|
import WebSocket2 from "ws";
|
|
5168
5423
|
|
|
@@ -5392,6 +5647,8 @@ var StreamForwarder = class {
|
|
|
5392
5647
|
if (done) break;
|
|
5393
5648
|
const chunk = Buffer.from(value);
|
|
5394
5649
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5650
|
+
await stream.pausePromise;
|
|
5651
|
+
if (ac.signal.aborted) break;
|
|
5395
5652
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5396
5653
|
if (this.binaryFramesSupported) {
|
|
5397
5654
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5434,6 +5691,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5434
5691
|
}
|
|
5435
5692
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5436
5693
|
var BASE_RECONNECT_DELAY = 500;
|
|
5694
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5437
5695
|
function getReconnectDelay(attempt) {
|
|
5438
5696
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5439
5697
|
const jitter = Math.random() * 1e3;
|
|
@@ -5505,10 +5763,27 @@ function connectTunnel(options) {
|
|
|
5505
5763
|
ws.close();
|
|
5506
5764
|
reject(new Error("Connection timeout"));
|
|
5507
5765
|
}, 3e4);
|
|
5766
|
+
let relaySilenceTimeout;
|
|
5767
|
+
let relaySilenceTimedOut = false;
|
|
5768
|
+
const clearRelaySilenceTimeout = () => {
|
|
5769
|
+
if (relaySilenceTimeout === void 0) return;
|
|
5770
|
+
clearTimeout(relaySilenceTimeout);
|
|
5771
|
+
relaySilenceTimeout = void 0;
|
|
5772
|
+
};
|
|
5773
|
+
const armRelaySilenceTimeout = () => {
|
|
5774
|
+
clearRelaySilenceTimeout();
|
|
5775
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
5776
|
+
relaySilenceTimeout = void 0;
|
|
5777
|
+
relaySilenceTimedOut = true;
|
|
5778
|
+
ws.terminate();
|
|
5779
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
5780
|
+
relaySilenceTimeout.unref?.();
|
|
5781
|
+
};
|
|
5508
5782
|
let upgradeRejection = null;
|
|
5509
5783
|
let upgradeRejectionReason = null;
|
|
5510
5784
|
ws.on("unexpected-response", (_req, res) => {
|
|
5511
5785
|
clearTimeout(connectionTimeout);
|
|
5786
|
+
clearRelaySilenceTimeout();
|
|
5512
5787
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5513
5788
|
upgradeRejectionReason = reason;
|
|
5514
5789
|
const chunks = [];
|
|
@@ -5538,6 +5813,7 @@ function connectTunnel(options) {
|
|
|
5538
5813
|
onInfo?.("WebSocket connection established");
|
|
5539
5814
|
});
|
|
5540
5815
|
ws.on("message", (data, isBinary) => {
|
|
5816
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5541
5817
|
if (isBinary) {
|
|
5542
5818
|
try {
|
|
5543
5819
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5571,6 +5847,7 @@ function connectTunnel(options) {
|
|
|
5571
5847
|
switch (message.type) {
|
|
5572
5848
|
case "connected": {
|
|
5573
5849
|
clearTimeout(connectionTimeout);
|
|
5850
|
+
armRelaySilenceTimeout();
|
|
5574
5851
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5575
5852
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5576
5853
|
onConnected?.(connectedAgentId);
|
|
@@ -5595,6 +5872,7 @@ function connectTunnel(options) {
|
|
|
5595
5872
|
});
|
|
5596
5873
|
ws.on("error", (error2) => {
|
|
5597
5874
|
clearTimeout(connectionTimeout);
|
|
5875
|
+
clearRelaySilenceTimeout();
|
|
5598
5876
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5599
5877
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5600
5878
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5606,7 +5884,8 @@ function connectTunnel(options) {
|
|
|
5606
5884
|
);
|
|
5607
5885
|
});
|
|
5608
5886
|
ws.on("close", (code, reason) => {
|
|
5609
|
-
|
|
5887
|
+
clearRelaySilenceTimeout();
|
|
5888
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5610
5889
|
forwarder.abortAll();
|
|
5611
5890
|
onDisconnected?.(code, reasonStr);
|
|
5612
5891
|
});
|
|
@@ -5614,9 +5893,10 @@ function connectTunnel(options) {
|
|
|
5614
5893
|
}
|
|
5615
5894
|
|
|
5616
5895
|
// src/lib/tunnel/runner-connection.ts
|
|
5617
|
-
var RunnerConnection = class {
|
|
5896
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5618
5897
|
opts;
|
|
5619
5898
|
sleep;
|
|
5899
|
+
now;
|
|
5620
5900
|
connection = null;
|
|
5621
5901
|
resolvedAgentId;
|
|
5622
5902
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5625,10 +5905,17 @@ var RunnerConnection = class {
|
|
|
5625
5905
|
reconnectPromise = null;
|
|
5626
5906
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5627
5907
|
reconnectAttempt = 0;
|
|
5908
|
+
generation = 0;
|
|
5909
|
+
disconnectedAtMs = null;
|
|
5910
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
5911
|
+
"No pong received",
|
|
5912
|
+
"Orphaned socket"
|
|
5913
|
+
]);
|
|
5628
5914
|
constructor(opts) {
|
|
5629
5915
|
this.opts = opts;
|
|
5630
5916
|
this.resolvedAgentId = opts.agentId;
|
|
5631
5917
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
5918
|
+
this.now = opts.now ?? Date.now;
|
|
5632
5919
|
}
|
|
5633
5920
|
get agentId() {
|
|
5634
5921
|
return this.resolvedAgentId;
|
|
@@ -5664,14 +5951,24 @@ var RunnerConnection = class {
|
|
|
5664
5951
|
port: this.opts.port,
|
|
5665
5952
|
openCodePassword: this.opts.openCodePassword,
|
|
5666
5953
|
onConnected: (agentId) => {
|
|
5954
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5667
5955
|
this.reconnectAttempt = 0;
|
|
5668
5956
|
this.reconnecting = false;
|
|
5669
5957
|
this.resolvedAgentId = agentId;
|
|
5670
|
-
|
|
5958
|
+
this.generation++;
|
|
5959
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
5960
|
+
this.disconnectedAtMs = null;
|
|
5961
|
+
events.onConnected(agentId, isReconnect, {
|
|
5962
|
+
generation: this.generation,
|
|
5963
|
+
attempt,
|
|
5964
|
+
downtimeMs
|
|
5965
|
+
});
|
|
5671
5966
|
},
|
|
5672
5967
|
onDisconnected: (code, reason) => {
|
|
5673
|
-
|
|
5674
|
-
|
|
5968
|
+
this.disconnectedAtMs = this.now();
|
|
5969
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
5970
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
5971
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5675
5972
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5676
5973
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5677
5974
|
});
|
|
@@ -6010,6 +6307,83 @@ function createResourceUsageCollector(homeDir) {
|
|
|
6010
6307
|
};
|
|
6011
6308
|
}
|
|
6012
6309
|
|
|
6310
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6311
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6312
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6313
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6314
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6315
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6316
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6317
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6318
|
+
var BAND_RANK = {
|
|
6319
|
+
low: 1,
|
|
6320
|
+
very_low: 2,
|
|
6321
|
+
critical: 3
|
|
6322
|
+
};
|
|
6323
|
+
function bandFor(freeBytes) {
|
|
6324
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6325
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6326
|
+
return "low";
|
|
6327
|
+
}
|
|
6328
|
+
function formatBytes(bytes) {
|
|
6329
|
+
if (bytes === null) return "unavailable";
|
|
6330
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6331
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6332
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6333
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6334
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6335
|
+
}
|
|
6336
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6337
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6338
|
+
}
|
|
6339
|
+
function evaluateLowDiskWarning(input) {
|
|
6340
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6341
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6342
|
+
return { warning: null, state };
|
|
6343
|
+
}
|
|
6344
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6345
|
+
if (recovered) {
|
|
6346
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6347
|
+
}
|
|
6348
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6349
|
+
if (!low) return { warning: null, state };
|
|
6350
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6351
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6352
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6353
|
+
const worse = currentRank > previousRank;
|
|
6354
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6355
|
+
const nextState = {
|
|
6356
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6357
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6358
|
+
};
|
|
6359
|
+
return {
|
|
6360
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6361
|
+
state: nextState
|
|
6362
|
+
};
|
|
6363
|
+
}
|
|
6364
|
+
|
|
6365
|
+
// src/lib/opencode/session-store-health.ts
|
|
6366
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6367
|
+
function createSessionStoreHealth(options = {}) {
|
|
6368
|
+
const now = options.now ?? (() => Date.now());
|
|
6369
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6370
|
+
let lastUnwritableAt = null;
|
|
6371
|
+
let lastWritableAt = null;
|
|
6372
|
+
return {
|
|
6373
|
+
observeUnwritable: () => {
|
|
6374
|
+
lastUnwritableAt = now();
|
|
6375
|
+
},
|
|
6376
|
+
observeWritable: () => {
|
|
6377
|
+
lastWritableAt = now();
|
|
6378
|
+
},
|
|
6379
|
+
isUnwritable: () => {
|
|
6380
|
+
if (lastUnwritableAt === null) return false;
|
|
6381
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6382
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6383
|
+
}
|
|
6384
|
+
};
|
|
6385
|
+
}
|
|
6386
|
+
|
|
6013
6387
|
// src/lib/channels/driver.ts
|
|
6014
6388
|
import { homedir as homedir6 } from "node:os";
|
|
6015
6389
|
|
|
@@ -6468,6 +6842,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6468
6842
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6469
6843
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6470
6844
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
6845
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6471
6846
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6472
6847
|
var HEARTBEAT_MS = 6e4;
|
|
6473
6848
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6529,6 +6904,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6529
6904
|
maxActiveSessions;
|
|
6530
6905
|
watcherStallMs;
|
|
6531
6906
|
wedgeWarningIntervalMs;
|
|
6907
|
+
sessionStoreHealth;
|
|
6532
6908
|
openCodeClient;
|
|
6533
6909
|
/** Cache of conversationId → opencode sessionId. */
|
|
6534
6910
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6898,6 +7274,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6898
7274
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6899
7275
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6900
7276
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7277
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6901
7278
|
}
|
|
6902
7279
|
get isV2() {
|
|
6903
7280
|
return this.openCodeClient.version === "v2";
|
|
@@ -6913,7 +7290,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6913
7290
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6914
7291
|
}
|
|
6915
7292
|
async listSessions() {
|
|
6916
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7293
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7294
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7295
|
+
});
|
|
6917
7296
|
}
|
|
6918
7297
|
async sessionExists(sessionId) {
|
|
6919
7298
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -8369,6 +8748,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8369
8748
|
b2AbandonedSignalled: false,
|
|
8370
8749
|
ambiguousPinnedSinceMs: 0,
|
|
8371
8750
|
ambiguousResolved: false,
|
|
8751
|
+
sessionStoreFailurePending: false,
|
|
8372
8752
|
neverStartedResolved: false
|
|
8373
8753
|
});
|
|
8374
8754
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8487,6 +8867,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8487
8867
|
b2AbandonedSignalled: false,
|
|
8488
8868
|
ambiguousPinnedSinceMs: 0,
|
|
8489
8869
|
ambiguousResolved: false,
|
|
8870
|
+
sessionStoreFailurePending: false,
|
|
8490
8871
|
neverStartedResolved: false
|
|
8491
8872
|
});
|
|
8492
8873
|
}
|
|
@@ -8900,6 +9281,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8900
9281
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8901
9282
|
this.neverStartedAttempts.delete(id);
|
|
8902
9283
|
}
|
|
9284
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9285
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9286
|
+
return;
|
|
9287
|
+
}
|
|
8903
9288
|
if (inFlight.neverStartedResolved) {
|
|
8904
9289
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8905
9290
|
inFlight.neverStartedResolved = false;
|
|
@@ -9079,6 +9464,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9079
9464
|
descendantOngoing,
|
|
9080
9465
|
rootOngoing
|
|
9081
9466
|
})) {
|
|
9467
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9468
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9469
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9470
|
+
watched_for_ms: pinnedForMs,
|
|
9471
|
+
run_state: "failed",
|
|
9472
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9473
|
+
});
|
|
9474
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9475
|
+
return;
|
|
9476
|
+
}
|
|
9082
9477
|
inFlight.b2AbandonedSignalled = true;
|
|
9083
9478
|
this.log({
|
|
9084
9479
|
level: "warn",
|
|
@@ -9137,6 +9532,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9137
9532
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9138
9533
|
sessionOngoing: ongoing
|
|
9139
9534
|
})) {
|
|
9535
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9536
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9537
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9538
|
+
watched_for_ms: pinnedForMs,
|
|
9539
|
+
run_state: "failed",
|
|
9540
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9541
|
+
});
|
|
9542
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9543
|
+
return;
|
|
9544
|
+
}
|
|
9140
9545
|
inFlight.ambiguousResolved = true;
|
|
9141
9546
|
this.log({
|
|
9142
9547
|
level: "warn",
|
|
@@ -9416,6 +9821,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9416
9821
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9417
9822
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9418
9823
|
}
|
|
9824
|
+
/**
|
|
9825
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
9826
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
9827
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
9828
|
+
*/
|
|
9829
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
9830
|
+
const conv = watcher.conv;
|
|
9831
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
9832
|
+
this.log({
|
|
9833
|
+
level: "error",
|
|
9834
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
9835
|
+
conversation_id: conv.id,
|
|
9836
|
+
message_id: inFlight.evidentMessageId
|
|
9837
|
+
});
|
|
9838
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
9839
|
+
try {
|
|
9840
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
9841
|
+
} catch (err) {
|
|
9842
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
9843
|
+
if (err instanceof ChannelTerminalError) {
|
|
9844
|
+
this.log({
|
|
9845
|
+
level: "warn",
|
|
9846
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
9847
|
+
conversation_id: conv.id,
|
|
9848
|
+
message_id: inFlight.evidentMessageId
|
|
9849
|
+
});
|
|
9850
|
+
this.recordReleasedOpencodeId(
|
|
9851
|
+
inFlight.evidentMessageId,
|
|
9852
|
+
sessionId,
|
|
9853
|
+
inFlight.opencodeMessageId
|
|
9854
|
+
);
|
|
9855
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9856
|
+
return;
|
|
9857
|
+
}
|
|
9858
|
+
if (this.now() >= inFlight.deadline) {
|
|
9859
|
+
this.log({
|
|
9860
|
+
level: "warn",
|
|
9861
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
9862
|
+
conversation_id: conv.id,
|
|
9863
|
+
message_id: inFlight.evidentMessageId
|
|
9864
|
+
});
|
|
9865
|
+
this.recordReleasedOpencodeId(
|
|
9866
|
+
inFlight.evidentMessageId,
|
|
9867
|
+
sessionId,
|
|
9868
|
+
inFlight.opencodeMessageId
|
|
9869
|
+
);
|
|
9870
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9871
|
+
return;
|
|
9872
|
+
}
|
|
9873
|
+
this.log({
|
|
9874
|
+
level: "warn",
|
|
9875
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
9876
|
+
conversation_id: conv.id,
|
|
9877
|
+
message_id: inFlight.evidentMessageId
|
|
9878
|
+
});
|
|
9879
|
+
return;
|
|
9880
|
+
}
|
|
9881
|
+
inFlight.done = true;
|
|
9882
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9883
|
+
}
|
|
9419
9884
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9420
9885
|
/**
|
|
9421
9886
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -12147,6 +12612,7 @@ function createCredentialSync({
|
|
|
12147
12612
|
|
|
12148
12613
|
// src/commands/run.ts
|
|
12149
12614
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
12615
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12150
12616
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12151
12617
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12152
12618
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12365,7 +12831,9 @@ function displayStatus(state) {
|
|
|
12365
12831
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12366
12832
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12367
12833
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12368
|
-
const detail = last ? chalk7.dim(
|
|
12834
|
+
const detail = last ? chalk7.dim(
|
|
12835
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
12836
|
+
) : "";
|
|
12369
12837
|
const agent = state.agentName ?? state.agentId;
|
|
12370
12838
|
console.log(
|
|
12371
12839
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12384,7 +12852,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12384
12852
|
logActivity(state, {
|
|
12385
12853
|
type: "info",
|
|
12386
12854
|
level: "warn",
|
|
12387
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
12855
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12388
12856
|
});
|
|
12389
12857
|
}
|
|
12390
12858
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12617,9 +13085,9 @@ async function driveChannels(state, driver) {
|
|
|
12617
13085
|
}
|
|
12618
13086
|
}
|
|
12619
13087
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12620
|
-
var
|
|
13088
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12621
13089
|
function shouldWarnForReclaimSkip(reason) {
|
|
12622
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13090
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12623
13091
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12624
13092
|
if (!version2) return false;
|
|
12625
13093
|
const major = Number(version2[1]);
|
|
@@ -12676,61 +13144,96 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12676
13144
|
async function runSweep(state, driver, config) {
|
|
12677
13145
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12678
13146
|
try {
|
|
12679
|
-
const
|
|
12680
|
-
|
|
12681
|
-
|
|
13147
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13148
|
+
const sessions = enumeration.sessions;
|
|
13149
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13150
|
+
if (enumerationReason !== null) {
|
|
13151
|
+
logActivity(state, {
|
|
12682
13152
|
type: "info",
|
|
12683
|
-
|
|
12684
|
-
|
|
12685
|
-
outcome: "sweep-skipped",
|
|
12686
|
-
reason: "opencode-unreachable"
|
|
12687
|
-
}
|
|
13153
|
+
level: "warn",
|
|
13154
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12688
13155
|
});
|
|
12689
|
-
return;
|
|
12690
13156
|
}
|
|
12691
|
-
const toDelete = selectSessionsToDelete(
|
|
12692
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12693
|
-
{
|
|
12694
|
-
maxAgeMs: config.maxAgeMs,
|
|
12695
|
-
maxCount: config.maxCount,
|
|
12696
|
-
nowMs: Date.now(),
|
|
12697
|
-
protectedIds: driver.protectedSessionIds()
|
|
12698
|
-
}
|
|
12699
|
-
);
|
|
12700
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12701
13157
|
let deleted = 0;
|
|
12702
13158
|
let failed = 0;
|
|
13159
|
+
let failed4xx = 0;
|
|
13160
|
+
let failed5xx = 0;
|
|
13161
|
+
let failedOtherStatus = 0;
|
|
13162
|
+
let failedThrew = 0;
|
|
12703
13163
|
let skippedNewlyActive = 0;
|
|
12704
|
-
|
|
12705
|
-
|
|
12706
|
-
|
|
12707
|
-
|
|
12708
|
-
|
|
12709
|
-
|
|
12710
|
-
|
|
12711
|
-
|
|
13164
|
+
let remaining = 0;
|
|
13165
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13166
|
+
if (sessions !== null) {
|
|
13167
|
+
const toDelete = selectSessionsToDelete(
|
|
13168
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13169
|
+
{
|
|
13170
|
+
maxAgeMs: config.maxAgeMs,
|
|
13171
|
+
maxCount: config.maxCount,
|
|
13172
|
+
nowMs: Date.now(),
|
|
13173
|
+
protectedIds: protectedNow
|
|
13174
|
+
}
|
|
13175
|
+
);
|
|
13176
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13177
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13178
|
+
protectedNow = driver.protectedSessionIds();
|
|
13179
|
+
for (const id of deleteCandidates) {
|
|
13180
|
+
if (protectedNow.has(id)) {
|
|
13181
|
+
skippedNewlyActive++;
|
|
13182
|
+
logActivity(state, {
|
|
13183
|
+
type: "info",
|
|
13184
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13185
|
+
});
|
|
13186
|
+
continue;
|
|
13187
|
+
}
|
|
13188
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13189
|
+
if (outcome.ok) {
|
|
13190
|
+
deleted++;
|
|
13191
|
+
} else if (outcome.status === "threw") {
|
|
13192
|
+
failed++;
|
|
13193
|
+
failedThrew++;
|
|
13194
|
+
} else {
|
|
13195
|
+
failed++;
|
|
13196
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13197
|
+
if (statusClass === 4) failed4xx++;
|
|
13198
|
+
else if (statusClass === 5) failed5xx++;
|
|
13199
|
+
else failedOtherStatus++;
|
|
13200
|
+
}
|
|
12712
13201
|
}
|
|
12713
|
-
|
|
12714
|
-
|
|
13202
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13203
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13204
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13205
|
+
logActivity(state, {
|
|
13206
|
+
type: "info",
|
|
13207
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13208
|
+
});
|
|
12715
13209
|
}
|
|
12716
|
-
const
|
|
12717
|
-
|
|
12718
|
-
|
|
12719
|
-
|
|
12720
|
-
|
|
12721
|
-
|
|
13210
|
+
const sessionMetadata = {
|
|
13211
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13212
|
+
sessions_deleted: deleted,
|
|
13213
|
+
sessions_failed: failed,
|
|
13214
|
+
sessions_failed_4xx: failed4xx,
|
|
13215
|
+
sessions_failed_5xx: failed5xx,
|
|
13216
|
+
sessions_failed_other: failedOtherStatus,
|
|
13217
|
+
sessions_failed_threw: failedThrew,
|
|
13218
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13219
|
+
sessions_remaining: remaining,
|
|
13220
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13221
|
+
};
|
|
12722
13222
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12723
13223
|
dbPath: sessionDbPath(),
|
|
12724
|
-
|
|
13224
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13225
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13226
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12725
13227
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12726
13228
|
});
|
|
12727
13229
|
if (reclaimResult.ok) {
|
|
13230
|
+
state.sessionStoreHealth.observeWritable();
|
|
12728
13231
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12729
13232
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12730
13233
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12731
13234
|
logSessionCleanupActivity(state, "sweep", {
|
|
12732
13235
|
type: "info",
|
|
12733
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${checkpointNote
|
|
13236
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
12734
13237
|
metadata: {
|
|
12735
13238
|
outcome: "reclaimed",
|
|
12736
13239
|
mode: reclaimResult.mode,
|
|
@@ -12738,25 +13241,27 @@ async function runSweep(state, driver, config) {
|
|
|
12738
13241
|
after_bytes: reclaimResult.afterBytes,
|
|
12739
13242
|
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
12740
13243
|
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
12741
|
-
|
|
12742
|
-
|
|
12743
|
-
|
|
12744
|
-
|
|
13244
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13245
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13246
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13247
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13248
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13249
|
+
...sessionMetadata
|
|
12745
13250
|
}
|
|
12746
13251
|
});
|
|
12747
13252
|
} else {
|
|
13253
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13254
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13255
|
+
}
|
|
12748
13256
|
logSessionCleanupActivity(state, "sweep", {
|
|
12749
13257
|
type: "info",
|
|
12750
|
-
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13258
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
12751
13259
|
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
12752
13260
|
metadata: {
|
|
12753
13261
|
outcome: "reclaim-skipped",
|
|
12754
13262
|
reason: reclaimResult.skipped,
|
|
12755
13263
|
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
12756
|
-
|
|
12757
|
-
sessions_deleted: deleted,
|
|
12758
|
-
sessions_failed: failed,
|
|
12759
|
-
sessions_skipped_active: skippedNewlyActive
|
|
13264
|
+
...sessionMetadata
|
|
12760
13265
|
}
|
|
12761
13266
|
});
|
|
12762
13267
|
}
|
|
@@ -13013,6 +13518,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
13013
13518
|
message: `Resource usage collection: ${warning2}`
|
|
13014
13519
|
});
|
|
13015
13520
|
}
|
|
13521
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13522
|
+
...usage,
|
|
13523
|
+
nowMs: Date.now(),
|
|
13524
|
+
state: state.lowDiskWarningState
|
|
13525
|
+
});
|
|
13526
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13527
|
+
if (lowDiskDecision.warning !== null) {
|
|
13528
|
+
logActivity(state, {
|
|
13529
|
+
type: "info",
|
|
13530
|
+
level: "warn",
|
|
13531
|
+
message: lowDiskDecision.warning
|
|
13532
|
+
});
|
|
13533
|
+
}
|
|
13016
13534
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
13017
13535
|
if (result.ok) {
|
|
13018
13536
|
if (consecutiveFailures > 0) {
|
|
@@ -13259,6 +13777,7 @@ async function run(options) {
|
|
|
13259
13777
|
opencodeClient: null,
|
|
13260
13778
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13261
13779
|
sessionDbProvenanceAnomaly: false,
|
|
13780
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13262
13781
|
opencodeProcess: null,
|
|
13263
13782
|
stopOpenCodeLogTail: null,
|
|
13264
13783
|
litestreamProcess: null,
|
|
@@ -13276,10 +13795,12 @@ async function run(options) {
|
|
|
13276
13795
|
openaiUsageRearm: null,
|
|
13277
13796
|
resourceUsageTimer: null,
|
|
13278
13797
|
stopResourceUsageSampling: null,
|
|
13798
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13279
13799
|
credentialSync: null,
|
|
13280
13800
|
authHeader: ""
|
|
13281
13801
|
};
|
|
13282
13802
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
13803
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13283
13804
|
if (options.credentialSyncMarker) {
|
|
13284
13805
|
state.credentialSync = createCredentialSync({
|
|
13285
13806
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13465,13 +13986,21 @@ async function run(options) {
|
|
|
13465
13986
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13466
13987
|
}
|
|
13467
13988
|
state.credentialSync?.arm();
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
createOpenCodeActivityForwarder(() => ({
|
|
13989
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
13990
|
+
() => ({
|
|
13471
13991
|
agentId: state.agentId,
|
|
13472
13992
|
authHeader: state.authHeader
|
|
13473
|
-
})
|
|
13474
|
-
|
|
13993
|
+
}),
|
|
13994
|
+
state.sessionStoreHealth.observeUnwritable
|
|
13995
|
+
);
|
|
13996
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
13997
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
13998
|
+
openCodeActivityForwarder
|
|
13999
|
+
);
|
|
14000
|
+
state.stopOpenCodeLogTail = () => {
|
|
14001
|
+
openCodeActivityForwarder.flush();
|
|
14002
|
+
openCodeLogTail.stop();
|
|
14003
|
+
};
|
|
13475
14004
|
let sessionDbVerifyFatal = false;
|
|
13476
14005
|
if (!options.restoreSessionDb) {
|
|
13477
14006
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
@@ -13755,6 +14284,7 @@ async function run(options) {
|
|
|
13755
14284
|
getAuthHeader: () => state.authHeader,
|
|
13756
14285
|
conversationFilter: state.conversationFilter,
|
|
13757
14286
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14287
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13758
14288
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13759
14289
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13760
14290
|
fileSyncDirectories,
|
|
@@ -13780,12 +14310,13 @@ async function run(options) {
|
|
|
13780
14310
|
openCodePassword: state.opencodePassword,
|
|
13781
14311
|
isRunning: () => state.running,
|
|
13782
14312
|
events: {
|
|
13783
|
-
onConnected: (agentId, isReconnect) => {
|
|
14313
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13784
14314
|
state.connected = true;
|
|
13785
14315
|
state.agentId = agentId;
|
|
14316
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13786
14317
|
logActivity(state, {
|
|
13787
14318
|
type: "info",
|
|
13788
|
-
message:
|
|
14319
|
+
message: connectionMessage
|
|
13789
14320
|
});
|
|
13790
14321
|
if (options.tunnelReadyFile) {
|
|
13791
14322
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13806,7 +14337,12 @@ async function run(options) {
|
|
|
13806
14337
|
version: state.opencodeVersion,
|
|
13807
14338
|
major: state.opencodeApiVersion,
|
|
13808
14339
|
connected: state.opencodeConnected
|
|
13809
|
-
})
|
|
14340
|
+
}),
|
|
14341
|
+
connection_generation: transition.generation,
|
|
14342
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14343
|
+
reconnect_attempt: transition.attempt,
|
|
14344
|
+
downtime_ms: transition.downtimeMs
|
|
14345
|
+
} : {}
|
|
13810
14346
|
});
|
|
13811
14347
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13812
14348
|
if (state.interactive) displayStatus(state);
|
|
@@ -13828,13 +14364,18 @@ async function run(options) {
|
|
|
13828
14364
|
if (state.interactive) displayStatus(state);
|
|
13829
14365
|
});
|
|
13830
14366
|
},
|
|
13831
|
-
onDisconnected: (code, reason) => {
|
|
14367
|
+
onDisconnected: (code, reason, transition) => {
|
|
13832
14368
|
state.connected = false;
|
|
14369
|
+
const safeReason = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13833
14370
|
logActivity(state, {
|
|
13834
14371
|
type: "info",
|
|
13835
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14372
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason})`
|
|
14373
|
+
});
|
|
14374
|
+
emitAgentDisconnected(state.agentId, {
|
|
14375
|
+
code,
|
|
14376
|
+
reason: safeReason,
|
|
14377
|
+
connection_generation: transition.generation
|
|
13836
14378
|
});
|
|
13837
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13838
14379
|
if (state.interactive) displayStatus(state);
|
|
13839
14380
|
},
|
|
13840
14381
|
onError: (error2) => {
|
|
@@ -13934,7 +14475,7 @@ async function run(options) {
|
|
|
13934
14475
|
},
|
|
13935
14476
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13936
14477
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13937
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14478
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13938
14479
|
firstDelayMs: firstReportDelayMs,
|
|
13939
14480
|
nextDelayMs: usageReportDelayMs,
|
|
13940
14481
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13977,7 +14518,7 @@ async function run(options) {
|
|
|
13977
14518
|
}
|
|
13978
14519
|
|
|
13979
14520
|
// src/index.ts
|
|
13980
|
-
var
|
|
14521
|
+
var version = getCliVersion();
|
|
13981
14522
|
var program = new Command();
|
|
13982
14523
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13983
14524
|
"--endpoint <url>",
|