@evident-ai/cli 3.5.1 → 3.5.2-dev.06654ec
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1526 -414
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { createRequire as createRequire2 } from "module";
|
|
5
4
|
import { Command } from "commander";
|
|
6
5
|
|
|
7
6
|
// src/commands/login.ts
|
|
@@ -704,6 +703,31 @@ async function reportResourceUsage(agentId, authHeader, usage) {
|
|
|
704
703
|
opencode_db_bytes: usage.opencodeDbBytes
|
|
705
704
|
});
|
|
706
705
|
}
|
|
706
|
+
async function getRunnerCredentialVault(agentId, authHeader) {
|
|
707
|
+
const empty = { prefix: null, vaultId: null, source: null };
|
|
708
|
+
try {
|
|
709
|
+
const apiUrl = getApiUrlConfig();
|
|
710
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/credential-vault`, {
|
|
711
|
+
headers: { Authorization: authHeader },
|
|
712
|
+
signal: AbortSignal.timeout(1e4)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) {
|
|
715
|
+
const serverMessage = await readErrorMessage(response);
|
|
716
|
+
return {
|
|
717
|
+
...empty,
|
|
718
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const data = await response.json();
|
|
722
|
+
return {
|
|
723
|
+
prefix: data.credential_vault_prefix,
|
|
724
|
+
vaultId: data.vault_id,
|
|
725
|
+
source: data.source
|
|
726
|
+
};
|
|
727
|
+
} catch (error2) {
|
|
728
|
+
return { ...empty, error: describeTimeoutError(error2, 1e4) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
707
731
|
async function getAgentInfo(agentId, authHeader) {
|
|
708
732
|
const apiUrl = getApiUrlConfig();
|
|
709
733
|
try {
|
|
@@ -878,6 +902,76 @@ async function status(options = {}) {
|
|
|
878
902
|
process.exit(result.exitCode);
|
|
879
903
|
}
|
|
880
904
|
|
|
905
|
+
// src/commands/credential-vault-env.ts
|
|
906
|
+
function shellQuote(value) {
|
|
907
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
908
|
+
}
|
|
909
|
+
function printAssignment(prefix) {
|
|
910
|
+
console.log(`CREDENTIAL_VAULT_PREFIX=${shellQuote(prefix ?? "")}`);
|
|
911
|
+
}
|
|
912
|
+
function safeReason(reason, credentials2, authHeader) {
|
|
913
|
+
let safe = reason;
|
|
914
|
+
for (const secret of [
|
|
915
|
+
authHeader,
|
|
916
|
+
credentials2?.token,
|
|
917
|
+
process.env.EVIDENT_RUNNER_KEY,
|
|
918
|
+
process.env.EVIDENT_AGENT_KEY,
|
|
919
|
+
process.env.EVIDENT_TOKEN
|
|
920
|
+
]) {
|
|
921
|
+
if (secret) safe = safe.replaceAll(secret, "[redacted]");
|
|
922
|
+
}
|
|
923
|
+
return safe.replace(/[\r\n]+/g, " ");
|
|
924
|
+
}
|
|
925
|
+
function printFallback(reason, credentials2, authHeader) {
|
|
926
|
+
printAssignment(null);
|
|
927
|
+
console.error(
|
|
928
|
+
`WARNING: ${safeReason(reason, credentials2, authHeader)}; using this runner's own credential prefix.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
async function credentialVaultEnv() {
|
|
932
|
+
let credentials2 = null;
|
|
933
|
+
let authHeader;
|
|
934
|
+
try {
|
|
935
|
+
credentials2 = await getAuthCredentials();
|
|
936
|
+
if (!credentials2) {
|
|
937
|
+
printFallback("no runner credentials are configured");
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
authHeader = getAuthHeader(credentials2);
|
|
941
|
+
const runner = await resolveAgentIdFromKey(authHeader);
|
|
942
|
+
if (!runner.agent_id) {
|
|
943
|
+
printFallback(
|
|
944
|
+
runner.error ?? "the runner identity could not be resolved",
|
|
945
|
+
credentials2,
|
|
946
|
+
authHeader
|
|
947
|
+
);
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const lookup = await getRunnerCredentialVault(runner.agent_id, authHeader);
|
|
951
|
+
if (lookup.error) {
|
|
952
|
+
printFallback(`credential vault lookup failed: ${lookup.error}`, credentials2, authHeader);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
if (lookup.prefix === null) {
|
|
956
|
+
printFallback("this runner is not linked to a credential vault", credentials2, authHeader);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (/[\r\n]/.test(lookup.prefix)) {
|
|
960
|
+
printFallback("credential vault lookup returned an invalid prefix", credentials2, authHeader);
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
printAssignment(lookup.prefix);
|
|
964
|
+
console.error(
|
|
965
|
+
`VAULT-MODE: credential stores use vault=${lookup.vaultId} source=${lookup.source} prefix=${lookup.prefix}.`
|
|
966
|
+
);
|
|
967
|
+
} catch (error2) {
|
|
968
|
+
const detail = credentials2 === null ? "credential resolution failed" : `credential vault lookup could not be completed: ${error2 instanceof Error ? error2.message : String(error2)}`;
|
|
969
|
+
printFallback(detail, credentials2, authHeader);
|
|
970
|
+
} finally {
|
|
971
|
+
process.exitCode = 0;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
881
975
|
// src/lib/claude-usage.ts
|
|
882
976
|
import { execFileSync } from "node:child_process";
|
|
883
977
|
import { readFileSync } from "node:fs";
|
|
@@ -1063,13 +1157,83 @@ async function claudeUsage() {
|
|
|
1063
1157
|
|
|
1064
1158
|
// src/commands/run.ts
|
|
1065
1159
|
import { chmodSync as chmodSync3, existsSync as existsSync3, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs";
|
|
1066
|
-
import { homedir as
|
|
1160
|
+
import { homedir as homedir7 } from "node:os";
|
|
1067
1161
|
import { isAbsolute as isAbsolute3, join as join11, parse, resolve as resolvePath2 } from "node:path";
|
|
1068
1162
|
import chalk7 from "chalk";
|
|
1069
1163
|
|
|
1070
1164
|
// ../../packages/types/src/agents/index.ts
|
|
1071
1165
|
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1072
1166
|
|
|
1167
|
+
// ../../packages/types/src/opencode/index.ts
|
|
1168
|
+
function roleOf(m) {
|
|
1169
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1170
|
+
if (typeof m.role === "string") return m.role;
|
|
1171
|
+
const infoRole = m.info?.role;
|
|
1172
|
+
return typeof infoRole === "string" ? infoRole : void 0;
|
|
1173
|
+
}
|
|
1174
|
+
function idOf(m) {
|
|
1175
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1176
|
+
if (typeof m.id === "string") return m.id;
|
|
1177
|
+
const infoId = m.info?.id;
|
|
1178
|
+
return typeof infoId === "string" ? infoId : void 0;
|
|
1179
|
+
}
|
|
1180
|
+
function parentIdOf(m) {
|
|
1181
|
+
if (!m || typeof m !== "object") return void 0;
|
|
1182
|
+
if (typeof m.parentID === "string") return m.parentID;
|
|
1183
|
+
const infoParent = m.info?.parentID;
|
|
1184
|
+
return typeof infoParent === "string" ? infoParent : void 0;
|
|
1185
|
+
}
|
|
1186
|
+
function textPartsOf(m) {
|
|
1187
|
+
if (!m || typeof m !== "object") return [];
|
|
1188
|
+
const parts = Array.isArray(m.parts) ? m.parts : [];
|
|
1189
|
+
return parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text ?? "").filter(Boolean);
|
|
1190
|
+
}
|
|
1191
|
+
function correlatedAssistantRun(messages, userMessageId) {
|
|
1192
|
+
if (!messages || messages.length === 0) return [];
|
|
1193
|
+
const byParent = messages.filter(
|
|
1194
|
+
(m) => roleOf(m) === "assistant" && parentIdOf(m) === userMessageId
|
|
1195
|
+
);
|
|
1196
|
+
if (byParent.length > 0) return byParent;
|
|
1197
|
+
const userIndex = messages.findIndex((m) => idOf(m) === userMessageId);
|
|
1198
|
+
if (userIndex === -1) return [];
|
|
1199
|
+
const run2 = [];
|
|
1200
|
+
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
1201
|
+
const role = roleOf(messages[i]);
|
|
1202
|
+
if (role === "user") break;
|
|
1203
|
+
if (role === "assistant") run2.push(messages[i]);
|
|
1204
|
+
}
|
|
1205
|
+
return run2;
|
|
1206
|
+
}
|
|
1207
|
+
function lastText(assistantRun) {
|
|
1208
|
+
for (let i = assistantRun.length - 1; i >= 0; i--) {
|
|
1209
|
+
const textParts = textPartsOf(assistantRun[i]);
|
|
1210
|
+
if (textParts.length > 0) return textParts.join("\n").trim() || null;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function effectiveCorrelatedRun(messages, userMessageId) {
|
|
1215
|
+
const run2 = correlatedAssistantRun(messages, userMessageId);
|
|
1216
|
+
if (lastText(run2) !== null) return run2;
|
|
1217
|
+
const userIndex = messages?.findIndex((m) => idOf(m) === userMessageId) ?? -1;
|
|
1218
|
+
if (userIndex === -1) return run2;
|
|
1219
|
+
let continuation;
|
|
1220
|
+
for (let i = userIndex + 1; messages && i < messages.length; i++) {
|
|
1221
|
+
if (roleOf(messages[i]) === "user") {
|
|
1222
|
+
continuation = messages[i];
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
const continuationId = idOf(continuation);
|
|
1227
|
+
if (roleOf(continuation) === "user" && textPartsOf(continuation).join("\n").trim().toLowerCase() === "continue" && continuationId) {
|
|
1228
|
+
const continuationRun = correlatedAssistantRun(messages, continuationId);
|
|
1229
|
+
if (continuationRun.length > 0) return continuationRun;
|
|
1230
|
+
}
|
|
1231
|
+
return run2;
|
|
1232
|
+
}
|
|
1233
|
+
function extractReplyTextForMessage(messages, opencodeUserMessageId) {
|
|
1234
|
+
return lastText(effectiveCorrelatedRun(messages, opencodeUserMessageId));
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1073
1237
|
// ../../packages/types/src/telemetry/index.ts
|
|
1074
1238
|
var TelemetryEventTypes = {
|
|
1075
1239
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -1158,7 +1322,7 @@ import ora4 from "ora";
|
|
|
1158
1322
|
import { select as select4 } from "@inquirer/prompts";
|
|
1159
1323
|
|
|
1160
1324
|
// src/lib/telemetry.ts
|
|
1161
|
-
var CLI_VERSION = (true ? "3.5.
|
|
1325
|
+
var CLI_VERSION = (true ? "3.5.2-dev.06654ec" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1162
1326
|
function getCliVersion() {
|
|
1163
1327
|
return CLI_VERSION;
|
|
1164
1328
|
}
|
|
@@ -1172,6 +1336,23 @@ var authProvider = null;
|
|
|
1172
1336
|
function setTelemetryAuthProvider(provider) {
|
|
1173
1337
|
authProvider = provider;
|
|
1174
1338
|
}
|
|
1339
|
+
var diagnosticsReporter = null;
|
|
1340
|
+
function setTelemetryDiagnosticsReporter(reporter) {
|
|
1341
|
+
diagnosticsReporter = reporter;
|
|
1342
|
+
}
|
|
1343
|
+
function reportTelemetryDiagnostic(message) {
|
|
1344
|
+
if (!diagnosticsReporter) {
|
|
1345
|
+
console.error(message);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
try {
|
|
1349
|
+
diagnosticsReporter(message);
|
|
1350
|
+
} catch (error2) {
|
|
1351
|
+
console.error(
|
|
1352
|
+
`Telemetry diagnostics reporter failed: ${error2 instanceof Error ? error2.message : String(error2)}; original diagnostic: ${message}`
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1175
1356
|
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1176
1357
|
var lastFlushFailureLoggedAt = 0;
|
|
1177
1358
|
var suppressedFlushFailureCount = 0;
|
|
@@ -1237,7 +1418,9 @@ async function flushEvents() {
|
|
|
1237
1418
|
signal: controller.signal
|
|
1238
1419
|
});
|
|
1239
1420
|
if (!response.ok) {
|
|
1240
|
-
|
|
1421
|
+
reportTelemetryDiagnostic(
|
|
1422
|
+
`Telemetry could not send events (HTTP ${response.status}); runner and OpenCode are unaffected`
|
|
1423
|
+
);
|
|
1241
1424
|
}
|
|
1242
1425
|
} finally {
|
|
1243
1426
|
clearTimeout(timeout);
|
|
@@ -1247,7 +1430,9 @@ async function flushEvents() {
|
|
|
1247
1430
|
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1248
1431
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1249
1432
|
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1250
|
-
|
|
1433
|
+
reportTelemetryDiagnostic(
|
|
1434
|
+
`Telemetry flush failed: ${message}; runner and OpenCode are unaffected${suffix}`
|
|
1435
|
+
);
|
|
1251
1436
|
lastFlushFailureLoggedAt = now;
|
|
1252
1437
|
suppressedFlushFailureCount = 0;
|
|
1253
1438
|
} else {
|
|
@@ -1299,8 +1484,227 @@ var EventTypes = {
|
|
|
1299
1484
|
};
|
|
1300
1485
|
|
|
1301
1486
|
// src/lib/runner-activity-telemetry.ts
|
|
1302
|
-
|
|
1487
|
+
import { homedir as homedir2 } from "node:os";
|
|
1488
|
+
|
|
1489
|
+
// src/lib/opencode/session-db-reclaim.ts
|
|
1490
|
+
import { statSync as statSync2, statfsSync } from "node:fs";
|
|
1491
|
+
import { dirname as dirname2 } from "node:path";
|
|
1492
|
+
var SESSION_DB_RECLAIM_CHUNK_PAGES = 2e3;
|
|
1493
|
+
var SESSION_DB_RECLAIM_TIME_BUDGET_MS = 500;
|
|
1494
|
+
var SESSION_DB_RECLAIM_PAGE_CEILING = 4e4;
|
|
1495
|
+
function errorMessage(error2) {
|
|
1496
|
+
if (!(error2 instanceof Error)) return String(error2);
|
|
1497
|
+
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
1498
|
+
}
|
|
1499
|
+
function isSessionStoreWriteFailure(error2) {
|
|
1500
|
+
const messages = [];
|
|
1501
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1502
|
+
let current = error2;
|
|
1503
|
+
while (current !== null && current !== void 0 && !visited.has(current)) {
|
|
1504
|
+
visited.add(current);
|
|
1505
|
+
if (typeof current === "string") {
|
|
1506
|
+
messages.push(current);
|
|
1507
|
+
break;
|
|
1508
|
+
}
|
|
1509
|
+
if (current instanceof Error) {
|
|
1510
|
+
messages.push(current.name, current.message);
|
|
1511
|
+
current = current.cause;
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
if (typeof current === "object") {
|
|
1515
|
+
const record = current;
|
|
1516
|
+
if (typeof record.code === "string") messages.push(record.code);
|
|
1517
|
+
if (typeof record.message === "string") messages.push(record.message);
|
|
1518
|
+
current = record.cause;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
break;
|
|
1522
|
+
}
|
|
1523
|
+
return messages.some(
|
|
1524
|
+
(message) => /(?:database or disk is full|disk is full|no space left on device|SQLITE_(?:FULL|IOERR)(?:\b|_))/i.test(
|
|
1525
|
+
message
|
|
1526
|
+
)
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
1530
|
+
try {
|
|
1531
|
+
const fsStats = statfsSync(dirname2(dbPath));
|
|
1532
|
+
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
1533
|
+
if (availableBytes < requiredBytes) {
|
|
1534
|
+
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
1535
|
+
}
|
|
1536
|
+
return null;
|
|
1537
|
+
} catch (err) {
|
|
1538
|
+
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
function readLogicalBytes(db) {
|
|
1542
|
+
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
1543
|
+
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
1544
|
+
return pageCount * pageSize;
|
|
1545
|
+
}
|
|
1546
|
+
function readFreelistCount(db) {
|
|
1547
|
+
return db.prepare("PRAGMA freelist_count").get().freelist_count;
|
|
1548
|
+
}
|
|
1549
|
+
function readCheckpointResult(db) {
|
|
1550
|
+
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
1551
|
+
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
1552
|
+
}
|
|
1553
|
+
async function probeReclaimAvailability(input) {
|
|
1554
|
+
const { dbPath, requiredBytes } = input;
|
|
1555
|
+
let sqlite;
|
|
1556
|
+
try {
|
|
1557
|
+
sqlite = await import("node:sqlite");
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1560
|
+
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
1561
|
+
return { reason: "sqlite-unavailable", detail };
|
|
1562
|
+
}
|
|
1563
|
+
let autoVacuum = null;
|
|
1564
|
+
try {
|
|
1565
|
+
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
1566
|
+
try {
|
|
1567
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1568
|
+
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1569
|
+
} finally {
|
|
1570
|
+
db.close();
|
|
1571
|
+
}
|
|
1572
|
+
} catch (err) {
|
|
1573
|
+
console.warn(
|
|
1574
|
+
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
1575
|
+
);
|
|
1576
|
+
}
|
|
1577
|
+
if (autoVacuum !== 0) return null;
|
|
1578
|
+
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
1579
|
+
}
|
|
1580
|
+
async function reclaimSessionDbSpace(input) {
|
|
1581
|
+
const {
|
|
1582
|
+
dbPath,
|
|
1583
|
+
pageCeiling = SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
1584
|
+
timeBudgetMs = SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
1585
|
+
chunkPages = SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
1586
|
+
now = () => performance.now(),
|
|
1587
|
+
allowFullVacuum = true
|
|
1588
|
+
} = input;
|
|
1589
|
+
let sqlite;
|
|
1590
|
+
try {
|
|
1591
|
+
sqlite = await import("node:sqlite");
|
|
1592
|
+
} catch (err) {
|
|
1593
|
+
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
1594
|
+
console.warn(
|
|
1595
|
+
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
1596
|
+
);
|
|
1597
|
+
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
1598
|
+
}
|
|
1599
|
+
const { DatabaseSync } = sqlite;
|
|
1600
|
+
let db;
|
|
1601
|
+
try {
|
|
1602
|
+
db = new DatabaseSync(dbPath);
|
|
1603
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
1604
|
+
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
1605
|
+
if (autoVacuum === 0) {
|
|
1606
|
+
if (!allowFullVacuum) {
|
|
1607
|
+
console.warn(
|
|
1608
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
1609
|
+
);
|
|
1610
|
+
return {
|
|
1611
|
+
ok: false,
|
|
1612
|
+
skipped: "full-vacuum-blocked",
|
|
1613
|
+
detail: "a session turn is live"
|
|
1614
|
+
};
|
|
1615
|
+
}
|
|
1616
|
+
const fileBytesForGuard = statSync2(dbPath).size;
|
|
1617
|
+
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
1618
|
+
if (skipReason !== null) {
|
|
1619
|
+
console.warn(
|
|
1620
|
+
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
1621
|
+
);
|
|
1622
|
+
return { ok: false, skipped: "insufficient-disk-space", detail: skipReason };
|
|
1623
|
+
}
|
|
1624
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1625
|
+
const freelistBefore = readFreelistCount(db);
|
|
1626
|
+
const startedAt = now();
|
|
1627
|
+
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
1628
|
+
db.exec("VACUUM");
|
|
1629
|
+
const afterBytes = readLogicalBytes(db);
|
|
1630
|
+
const freelistAfter = readFreelistCount(db);
|
|
1631
|
+
const checkpoint = readCheckpointResult(db);
|
|
1632
|
+
return {
|
|
1633
|
+
ok: true,
|
|
1634
|
+
mode: "convert",
|
|
1635
|
+
beforeBytes,
|
|
1636
|
+
afterBytes,
|
|
1637
|
+
freelistBefore,
|
|
1638
|
+
freelistAfter,
|
|
1639
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1640
|
+
stoppedBy: "freelist-drained",
|
|
1641
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1642
|
+
checkpoint
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
if (autoVacuum === 2) {
|
|
1646
|
+
const beforeBytes = readLogicalBytes(db);
|
|
1647
|
+
const freelistBefore = readFreelistCount(db);
|
|
1648
|
+
const startedAt = now();
|
|
1649
|
+
const pageLimit = Math.max(0, Math.trunc(pageCeiling));
|
|
1650
|
+
const chunkLimit = Math.max(1, Math.trunc(chunkPages));
|
|
1651
|
+
const budget = Math.max(0, timeBudgetMs);
|
|
1652
|
+
let pagesAttempted = 0;
|
|
1653
|
+
let freelistAfter = freelistBefore;
|
|
1654
|
+
let stoppedBy = "freelist-drained";
|
|
1655
|
+
while (freelistAfter > 0) {
|
|
1656
|
+
if (pagesAttempted >= pageLimit) {
|
|
1657
|
+
stoppedBy = "page-ceiling";
|
|
1658
|
+
break;
|
|
1659
|
+
}
|
|
1660
|
+
if (now() - startedAt >= budget) {
|
|
1661
|
+
stoppedBy = "time-budget";
|
|
1662
|
+
break;
|
|
1663
|
+
}
|
|
1664
|
+
const pages = Math.min(chunkLimit, pageLimit - pagesAttempted, freelistAfter);
|
|
1665
|
+
db.exec(`PRAGMA incremental_vacuum(${pages})`);
|
|
1666
|
+
pagesAttempted += pages;
|
|
1667
|
+
freelistAfter = readFreelistCount(db);
|
|
1668
|
+
}
|
|
1669
|
+
const afterBytes = readLogicalBytes(db);
|
|
1670
|
+
const checkpoint = readCheckpointResult(db);
|
|
1671
|
+
return {
|
|
1672
|
+
ok: true,
|
|
1673
|
+
mode: "incremental",
|
|
1674
|
+
beforeBytes,
|
|
1675
|
+
afterBytes,
|
|
1676
|
+
freelistBefore,
|
|
1677
|
+
freelistAfter,
|
|
1678
|
+
pagesReclaimed: Math.max(0, freelistBefore - freelistAfter),
|
|
1679
|
+
stoppedBy,
|
|
1680
|
+
elapsedMs: Math.max(0, now() - startedAt),
|
|
1681
|
+
checkpoint
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
console.warn(
|
|
1685
|
+
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
1686
|
+
);
|
|
1687
|
+
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
1688
|
+
} catch (err) {
|
|
1689
|
+
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
1690
|
+
return {
|
|
1691
|
+
ok: false,
|
|
1692
|
+
skipped: isSessionStoreWriteFailure(err) ? "disk-exhaustion" : "reclaim-error",
|
|
1693
|
+
detail: errorMessage(err)
|
|
1694
|
+
};
|
|
1695
|
+
} finally {
|
|
1696
|
+
db?.close();
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1701
|
+
var FORWARDED_LEVELS_BY_SOURCE = {
|
|
1702
|
+
"cli.run": /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1703
|
+
opencode: /* @__PURE__ */ new Set(["warn", "error"]),
|
|
1704
|
+
"cli.session-cleanup": /* @__PURE__ */ new Set(["info", "warn", "error"])
|
|
1705
|
+
};
|
|
1303
1706
|
var SEVERITY_BY_LEVEL = {
|
|
1707
|
+
info: "info",
|
|
1304
1708
|
warn: "warning",
|
|
1305
1709
|
error: "error"
|
|
1306
1710
|
};
|
|
@@ -1310,26 +1714,85 @@ function parseOpenCodeLogLine(line) {
|
|
|
1310
1714
|
if (!levelMatch) return null;
|
|
1311
1715
|
const level = levelMatch[1].toUpperCase();
|
|
1312
1716
|
if (level !== "WARN" && level !== "ERROR") return null;
|
|
1313
|
-
const sessionMatch = normalisedLine.match(/(?:^|\s)sessionID=(\S+)/);
|
|
1314
|
-
|
|
1717
|
+
const sessionMatch = normalisedLine.match(/(?:^|\s)(?:sessionID|session\.id)=(\S+)/);
|
|
1718
|
+
const messageIDMatch = normalisedLine.match(/(?:^|\s)messageID=(\S+)/);
|
|
1719
|
+
const messageMatch = normalisedLine.match(/(?:^|\s)message=(?:"((?:\\.|[^"])*)"|(\S+))/);
|
|
1720
|
+
const parsed = {
|
|
1721
|
+
level: level === "WARN" ? "warn" : "error",
|
|
1722
|
+
sessionID: sessionMatch?.[1],
|
|
1723
|
+
messageID: messageIDMatch?.[1]
|
|
1724
|
+
};
|
|
1725
|
+
const message = messageMatch?.[1] ?? messageMatch?.[2];
|
|
1726
|
+
if (message !== void 0) parsed.message = message;
|
|
1727
|
+
return parsed;
|
|
1315
1728
|
}
|
|
1316
1729
|
var MAX_LINE_BUFFER_BYTES = 16 * 1024;
|
|
1317
|
-
|
|
1730
|
+
var REPEAT_SUPPRESSION_WINDOW_MS = 6e4;
|
|
1731
|
+
function createOpenCodeActivityForwarder(getContext, onSessionStoreWriteFailure) {
|
|
1318
1732
|
let buffer = Buffer.alloc(0);
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1733
|
+
const repeatStates = /* @__PURE__ */ new Map();
|
|
1734
|
+
const forwardLine = (line, parsed, repeatsSuppressed = 0) => {
|
|
1735
|
+
const metadata = {
|
|
1736
|
+
...parsed.sessionID ? { sessionID: parsed.sessionID } : {},
|
|
1737
|
+
...parsed.messageID ? { messageID: parsed.messageID } : {},
|
|
1738
|
+
...repeatsSuppressed > 0 ? { repeats_suppressed: repeatsSuppressed } : {}
|
|
1739
|
+
};
|
|
1322
1740
|
forwardRunnerActivity(
|
|
1323
1741
|
{
|
|
1324
1742
|
level: parsed.level,
|
|
1325
1743
|
error: line,
|
|
1326
|
-
metadata:
|
|
1744
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : void 0,
|
|
1327
1745
|
source: "opencode"
|
|
1328
1746
|
},
|
|
1329
1747
|
getContext()
|
|
1330
1748
|
);
|
|
1331
1749
|
};
|
|
1332
|
-
|
|
1750
|
+
const expireRepeatStates = (now, currentKey) => {
|
|
1751
|
+
for (const [key, state] of repeatStates) {
|
|
1752
|
+
if (key === currentKey || now - state.lastForwardedAt < REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1753
|
+
continue;
|
|
1754
|
+
}
|
|
1755
|
+
if (state.repeatsSuppressed > 0) {
|
|
1756
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1757
|
+
}
|
|
1758
|
+
repeatStates.delete(key);
|
|
1759
|
+
}
|
|
1760
|
+
};
|
|
1761
|
+
const flushLine = (line) => {
|
|
1762
|
+
if (onSessionStoreWriteFailure && isSessionStoreWriteFailure(line)) {
|
|
1763
|
+
try {
|
|
1764
|
+
onSessionStoreWriteFailure();
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
reportTelemetryDiagnostic(
|
|
1767
|
+
`Runner activity telemetry could not record session-store write failure: ${err instanceof Error ? err.message : String(err)}`
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
const parsed = parseOpenCodeLogLine(line);
|
|
1772
|
+
if (!parsed) return;
|
|
1773
|
+
const now = Date.now();
|
|
1774
|
+
const key = parsed.message === void 0 ? void 0 : `${parsed.level}\0${parsed.message}\0${parsed.sessionID ?? ""}\0${parsed.messageID ?? ""}`;
|
|
1775
|
+
expireRepeatStates(now, key);
|
|
1776
|
+
if (key === void 0) {
|
|
1777
|
+
forwardLine(line, parsed);
|
|
1778
|
+
return;
|
|
1779
|
+
}
|
|
1780
|
+
const previous = repeatStates.get(key);
|
|
1781
|
+
if (!previous || now - previous.lastForwardedAt >= REPEAT_SUPPRESSION_WINDOW_MS) {
|
|
1782
|
+
forwardLine(line, parsed, previous?.repeatsSuppressed ?? 0);
|
|
1783
|
+
repeatStates.set(key, {
|
|
1784
|
+
line,
|
|
1785
|
+
parsed,
|
|
1786
|
+
lastForwardedAt: now,
|
|
1787
|
+
repeatsSuppressed: 0
|
|
1788
|
+
});
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
previous.line = line;
|
|
1792
|
+
previous.parsed = parsed;
|
|
1793
|
+
previous.repeatsSuppressed++;
|
|
1794
|
+
};
|
|
1795
|
+
const forward = ((chunk) => {
|
|
1333
1796
|
buffer = Buffer.concat([buffer, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf-8")]);
|
|
1334
1797
|
let newlineIndex;
|
|
1335
1798
|
while ((newlineIndex = buffer.indexOf(10)) !== -1) {
|
|
@@ -1340,14 +1803,44 @@ function createOpenCodeActivityForwarder(getContext) {
|
|
|
1340
1803
|
flushLine(buffer.toString("utf-8"));
|
|
1341
1804
|
buffer = Buffer.alloc(0);
|
|
1342
1805
|
}
|
|
1806
|
+
});
|
|
1807
|
+
forward.flush = () => {
|
|
1808
|
+
for (const state of repeatStates.values()) {
|
|
1809
|
+
if (state.repeatsSuppressed > 0) {
|
|
1810
|
+
forwardLine(state.line, state.parsed, state.repeatsSuppressed);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
repeatStates.clear();
|
|
1343
1814
|
};
|
|
1815
|
+
return forward;
|
|
1344
1816
|
}
|
|
1345
1817
|
var MAX_MESSAGE_LENGTH = 500;
|
|
1346
1818
|
var MAX_METADATA_VALUE_LENGTH = 200;
|
|
1347
1819
|
var MAX_METADATA_ENTRIES = 20;
|
|
1348
1820
|
var TRUNCATION_MARKER = "\u2026";
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1821
|
+
function redactHomeDirectory(message) {
|
|
1822
|
+
const home = homedir2();
|
|
1823
|
+
if (!home || home === "/") {
|
|
1824
|
+
return message;
|
|
1825
|
+
}
|
|
1826
|
+
const pathSegmentCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-";
|
|
1827
|
+
let result = "";
|
|
1828
|
+
let searchFrom = 0;
|
|
1829
|
+
let homeIndex;
|
|
1830
|
+
while ((homeIndex = message.indexOf(home, searchFrom)) !== -1) {
|
|
1831
|
+
const homeEnd = homeIndex + home.length;
|
|
1832
|
+
const previousCharacter = message[homeIndex - 1];
|
|
1833
|
+
const nextCharacter = message[homeEnd];
|
|
1834
|
+
const atPathBoundary = home === "/" || (!previousCharacter || !pathSegmentCharacters.includes(previousCharacter)) && (!nextCharacter || !pathSegmentCharacters.includes(nextCharacter));
|
|
1835
|
+
result += message.slice(searchFrom, homeIndex);
|
|
1836
|
+
result += atPathBoundary ? "~" : home;
|
|
1837
|
+
searchFrom = homeEnd;
|
|
1838
|
+
}
|
|
1839
|
+
return result + message.slice(searchFrom);
|
|
1840
|
+
}
|
|
1841
|
+
function redactDiagnosticText(message) {
|
|
1842
|
+
const redacted = message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/(?<![A-Za-z0-9_-])sk-ant-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-ant-***").replace(/(?<![A-Za-z0-9_-])sk-proj-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])/g, "sk-proj-***").replace(/(?<![A-Za-z0-9_-])sk-[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "sk-***").replace(/(?<![A-Za-z0-9_-])(gh[oprsu])_[A-Za-z0-9]{20,}(?![A-Za-z0-9_-])/g, "$1_***").replace(/(?<![A-Za-z0-9_-])github_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_-])/g, "github_pat_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1843
|
+
return redactHomeDirectory(redacted);
|
|
1351
1844
|
}
|
|
1352
1845
|
function truncate(message) {
|
|
1353
1846
|
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
@@ -1357,14 +1850,17 @@ function sanitiseMetadata(metadata) {
|
|
|
1357
1850
|
if (!metadata) return {};
|
|
1358
1851
|
const entries = Object.entries(metadata).slice(0, MAX_METADATA_ENTRIES);
|
|
1359
1852
|
if (Object.keys(metadata).length > entries.length) {
|
|
1360
|
-
|
|
1361
|
-
`
|
|
1853
|
+
reportTelemetryDiagnostic(
|
|
1854
|
+
`Runner activity telemetry dropped ${Object.keys(metadata).length - entries.length} metadata entries; those entries were not sent`
|
|
1362
1855
|
);
|
|
1363
1856
|
}
|
|
1364
1857
|
const sanitised = [];
|
|
1365
1858
|
for (const [key, value] of entries) {
|
|
1366
1859
|
if (typeof value === "string") {
|
|
1367
|
-
sanitised.push([
|
|
1860
|
+
sanitised.push([
|
|
1861
|
+
key,
|
|
1862
|
+
truncate(redactDiagnosticText(value).slice(0, MAX_METADATA_VALUE_LENGTH))
|
|
1863
|
+
]);
|
|
1368
1864
|
} else if (value === null || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) {
|
|
1369
1865
|
sanitised.push([key, value]);
|
|
1370
1866
|
}
|
|
@@ -1382,8 +1878,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1382
1878
|
}
|
|
1383
1879
|
if (now - window.windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1384
1880
|
if (window.windowDroppedCount > 0) {
|
|
1385
|
-
|
|
1386
|
-
`
|
|
1881
|
+
reportTelemetryDiagnostic(
|
|
1882
|
+
`Runner activity telemetry rate cap reached: dropped ${window.windowDroppedCount} ${window.windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (limit ${RATE_LIMIT_MAX_EVENTS}/min); those entries were not sent for source "${source}"`
|
|
1387
1883
|
);
|
|
1388
1884
|
}
|
|
1389
1885
|
window.windowStartedAt = now;
|
|
@@ -1393,8 +1889,8 @@ function admitUnderRateLimit(source, now) {
|
|
|
1393
1889
|
if (window.windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1394
1890
|
window.windowDroppedCount++;
|
|
1395
1891
|
if (window.windowDroppedCount === 1) {
|
|
1396
|
-
|
|
1397
|
-
`
|
|
1892
|
+
reportTelemetryDiagnostic(
|
|
1893
|
+
`Runner activity telemetry rate cap reached (${RATE_LIMIT_MAX_EVENTS} entries per ${RATE_LIMIT_WINDOW_MS / 1e3}s limit; further entries were not sent for source "${source}"`
|
|
1398
1894
|
);
|
|
1399
1895
|
}
|
|
1400
1896
|
return false;
|
|
@@ -1404,12 +1900,12 @@ function admitUnderRateLimit(source, now) {
|
|
|
1404
1900
|
}
|
|
1405
1901
|
function forwardRunnerActivity(entry, context) {
|
|
1406
1902
|
try {
|
|
1407
|
-
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1408
|
-
if (!context.agentId || !context.authHeader) return;
|
|
1409
1903
|
const source = entry.source ?? "cli.run";
|
|
1904
|
+
if (!FORWARDED_LEVELS_BY_SOURCE[source].has(entry.level)) return;
|
|
1905
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1410
1906
|
if (!admitUnderRateLimit(source, Date.now())) return;
|
|
1411
1907
|
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1412
|
-
const message = truncate(
|
|
1908
|
+
const message = truncate(redactDiagnosticText(rawMessage));
|
|
1413
1909
|
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1414
1910
|
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1415
1911
|
message,
|
|
@@ -1417,8 +1913,8 @@ function forwardRunnerActivity(entry, context) {
|
|
|
1417
1913
|
agentId: context.agentId
|
|
1418
1914
|
});
|
|
1419
1915
|
} catch (err) {
|
|
1420
|
-
|
|
1421
|
-
`
|
|
1916
|
+
reportTelemetryDiagnostic(
|
|
1917
|
+
`Runner activity telemetry failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1422
1918
|
);
|
|
1423
1919
|
}
|
|
1424
1920
|
}
|
|
@@ -1708,8 +2204,46 @@ async function waitForOpenCode2Health(port, password, timeoutMs = 3e4) {
|
|
|
1708
2204
|
|
|
1709
2205
|
// src/lib/http-timeout.ts
|
|
1710
2206
|
var REQUEST_TIMEOUT_MS = 6e4;
|
|
2207
|
+
var RequestTimeoutError = class extends Error {
|
|
2208
|
+
constructor(operation, timeoutMs, elapsedMs) {
|
|
2209
|
+
super(`${operation} timed out after ${elapsedMs}ms (limit ${timeoutMs}ms)`);
|
|
2210
|
+
this.operation = operation;
|
|
2211
|
+
this.timeoutMs = timeoutMs;
|
|
2212
|
+
this.elapsedMs = elapsedMs;
|
|
2213
|
+
this.name = "TimeoutError";
|
|
2214
|
+
}
|
|
2215
|
+
};
|
|
2216
|
+
function describeRequestOperation(input, init) {
|
|
2217
|
+
const method = (init?.method ?? input.method ?? "GET").toUpperCase();
|
|
2218
|
+
const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
|
2219
|
+
let path;
|
|
2220
|
+
try {
|
|
2221
|
+
path = new URL(rawUrl).pathname;
|
|
2222
|
+
} catch (error2) {
|
|
2223
|
+
if (!(error2 instanceof TypeError)) throw error2;
|
|
2224
|
+
path = rawUrl.split(/[?#]/, 1)[0];
|
|
2225
|
+
}
|
|
2226
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2227
|
+
const prefixedIdentifier = /^[a-z]+_[A-Za-z0-9_-]{10,}$/;
|
|
2228
|
+
const maskedPath = path.split("/").map((segment) => uuid.test(segment) || prefixedIdentifier.test(segment) ? ":id" : segment).join("/");
|
|
2229
|
+
return `${method} ${maskedPath}`;
|
|
2230
|
+
}
|
|
1711
2231
|
function withRequestTimeout(fetchImpl, timeoutMs) {
|
|
1712
|
-
return ((input, init) =>
|
|
2232
|
+
return (async (input, init) => {
|
|
2233
|
+
const startedAt = Date.now();
|
|
2234
|
+
try {
|
|
2235
|
+
return await fetchImpl(input, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
2236
|
+
} catch (error2) {
|
|
2237
|
+
if (error2?.name === "TimeoutError") {
|
|
2238
|
+
throw new RequestTimeoutError(
|
|
2239
|
+
describeRequestOperation(input, init),
|
|
2240
|
+
timeoutMs,
|
|
2241
|
+
Date.now() - startedAt
|
|
2242
|
+
);
|
|
2243
|
+
}
|
|
2244
|
+
throw error2;
|
|
2245
|
+
}
|
|
2246
|
+
});
|
|
1713
2247
|
}
|
|
1714
2248
|
|
|
1715
2249
|
// src/lib/opencode/client.ts
|
|
@@ -1775,7 +2309,7 @@ async function getOpenCodeDirectory(port, client) {
|
|
|
1775
2309
|
return null;
|
|
1776
2310
|
}
|
|
1777
2311
|
}
|
|
1778
|
-
function
|
|
2312
|
+
function roleOf2(m) {
|
|
1779
2313
|
if (!m || typeof m !== "object") return void 0;
|
|
1780
2314
|
if (typeof m.role === "string") return m.role;
|
|
1781
2315
|
const infoRole = m.info?.role;
|
|
@@ -1789,13 +2323,13 @@ function createdOf(m) {
|
|
|
1789
2323
|
if (!m || typeof m !== "object") return void 0;
|
|
1790
2324
|
return m.info?.time?.created ?? m.time?.created;
|
|
1791
2325
|
}
|
|
1792
|
-
function
|
|
2326
|
+
function idOf2(m) {
|
|
1793
2327
|
if (!m || typeof m !== "object") return void 0;
|
|
1794
2328
|
if (typeof m.id === "string") return m.id;
|
|
1795
2329
|
const infoId = m.info?.id;
|
|
1796
2330
|
return typeof infoId === "string" ? infoId : void 0;
|
|
1797
2331
|
}
|
|
1798
|
-
function
|
|
2332
|
+
function parentIdOf2(m) {
|
|
1799
2333
|
if (!m || typeof m !== "object") return void 0;
|
|
1800
2334
|
if (typeof m.parentID === "string") return m.parentID;
|
|
1801
2335
|
const infoParent = m.info?.parentID;
|
|
@@ -1860,9 +2394,10 @@ async function pollSessionMessagesForRedrive(port, sessionId, client) {
|
|
|
1860
2394
|
function isSessionActivelyGenerating(messages) {
|
|
1861
2395
|
if (!messages || messages.length === 0) return false;
|
|
1862
2396
|
const last = messages[messages.length - 1];
|
|
1863
|
-
if (
|
|
2397
|
+
if (roleOf2(last) !== "assistant") return false;
|
|
1864
2398
|
return completedOf(last) == null;
|
|
1865
2399
|
}
|
|
2400
|
+
var FULL_SESSION_ENUMERATION_MAX_SESSIONS = 3200;
|
|
1866
2401
|
function sessionLastActivityMs(session) {
|
|
1867
2402
|
const candidates = [
|
|
1868
2403
|
session.time?.updated,
|
|
@@ -1877,30 +2412,60 @@ function sessionLastActivityMs(session) {
|
|
|
1877
2412
|
}
|
|
1878
2413
|
return null;
|
|
1879
2414
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
2415
|
+
function sessionListPath(options) {
|
|
2416
|
+
const params = new URLSearchParams();
|
|
2417
|
+
if (options?.limit !== void 0) params.set("limit", String(options.limit));
|
|
2418
|
+
const query = params.toString();
|
|
2419
|
+
return query ? `/session?${query}` : "/session";
|
|
2420
|
+
}
|
|
2421
|
+
async function listSessionPage(port, client, options) {
|
|
2422
|
+
const path = sessionListPath(options);
|
|
1882
2423
|
try {
|
|
1883
|
-
const res = await requestWithClient(port, client,
|
|
2424
|
+
const res = await requestWithClient(port, client, path);
|
|
1884
2425
|
if (!res.ok) return null;
|
|
1885
2426
|
const body = await res.json();
|
|
1886
2427
|
return Array.isArray(body) ? body : null;
|
|
1887
2428
|
} catch (error2) {
|
|
1888
2429
|
console.error(
|
|
1889
|
-
`[
|
|
2430
|
+
`[listSessionPage] GET ${path} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1890
2431
|
);
|
|
1891
2432
|
return null;
|
|
1892
2433
|
}
|
|
1893
2434
|
}
|
|
2435
|
+
async function listSessions(port, client, options) {
|
|
2436
|
+
if (client?.version === "v2") return listV2Sessions(client);
|
|
2437
|
+
return listSessionPage(port, client, options);
|
|
2438
|
+
}
|
|
2439
|
+
async function listAllSessions(port, client) {
|
|
2440
|
+
if (client?.version === "v2") {
|
|
2441
|
+
const sessions2 = await listV2Sessions(client);
|
|
2442
|
+
return sessions2 === null ? { sessions: null, reason: "opencode-unreachable" } : { sessions: sessions2 };
|
|
2443
|
+
}
|
|
2444
|
+
const sessions = await listSessionPage(port, client, {
|
|
2445
|
+
limit: FULL_SESSION_ENUMERATION_MAX_SESSIONS
|
|
2446
|
+
});
|
|
2447
|
+
if (sessions === null) return { sessions: null, reason: "opencode-unreachable" };
|
|
2448
|
+
if (sessions.length >= FULL_SESSION_ENUMERATION_MAX_SESSIONS) {
|
|
2449
|
+
return { sessions: null, reason: "session-enumeration-limit-bound" };
|
|
2450
|
+
}
|
|
2451
|
+
return { sessions };
|
|
2452
|
+
}
|
|
1894
2453
|
async function deleteSession(port, id, client) {
|
|
1895
2454
|
if (client?.version === "v2") return deleteV2Session(client, id);
|
|
1896
2455
|
try {
|
|
1897
|
-
const res = await requestWithClient(
|
|
1898
|
-
|
|
2456
|
+
const res = await requestWithClient(
|
|
2457
|
+
port,
|
|
2458
|
+
client,
|
|
2459
|
+
`/session/${id}`,
|
|
2460
|
+
{ method: "DELETE" },
|
|
2461
|
+
{ allowStatuses: ALL_HTTP_STATUSES }
|
|
2462
|
+
);
|
|
2463
|
+
return { ok: res.status >= 200 && res.status < 300, status: res.status };
|
|
1899
2464
|
} catch (error2) {
|
|
1900
2465
|
console.error(
|
|
1901
2466
|
`[deleteSession] DELETE /session/${id} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
1902
2467
|
);
|
|
1903
|
-
return false;
|
|
2468
|
+
return { ok: false, status: "threw" };
|
|
1904
2469
|
}
|
|
1905
2470
|
}
|
|
1906
2471
|
async function sessionExists(port, id, client) {
|
|
@@ -2138,11 +2703,31 @@ function messageText(m) {
|
|
|
2138
2703
|
if (!m || !Array.isArray(m.parts)) return "";
|
|
2139
2704
|
return m.parts.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text).join("");
|
|
2140
2705
|
}
|
|
2141
|
-
|
|
2706
|
+
function findDispatchedUserMessageId(messages, content, excludeIds) {
|
|
2707
|
+
if (!messages || messages.length === 0) return null;
|
|
2708
|
+
let best = null;
|
|
2709
|
+
for (const message of messages) {
|
|
2710
|
+
if (roleOf2(message) !== "user") continue;
|
|
2711
|
+
const id = idOf2(message);
|
|
2712
|
+
if (typeof id !== "string" || excludeIds?.has(id)) continue;
|
|
2713
|
+
if (messageText(message) !== content) continue;
|
|
2714
|
+
const created = createdOf(message) ?? 0;
|
|
2715
|
+
if (best === null || created > best.created) best = { id, created };
|
|
2716
|
+
}
|
|
2717
|
+
return best?.id ?? null;
|
|
2718
|
+
}
|
|
2719
|
+
async function sendPromptAsync(port, sessionId, content, options, attachments, client, callbacks) {
|
|
2142
2720
|
const before = await getSessionMessages(port, sessionId, client);
|
|
2143
2721
|
const knownUserIds = new Set(
|
|
2144
|
-
(before ?? []).filter((m) =>
|
|
2722
|
+
(before ?? []).filter((m) => roleOf2(m) === "user").map((m) => idOf2(m)).filter((id) => typeof id === "string")
|
|
2145
2723
|
);
|
|
2724
|
+
if (callbacks) {
|
|
2725
|
+
try {
|
|
2726
|
+
callbacks.onKnownUserIds(before === null ? null : knownUserIds);
|
|
2727
|
+
} catch (error2) {
|
|
2728
|
+
callbacks.onKnownUserIdsError(error2);
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2146
2731
|
const parts = [{ type: "text", text: content }];
|
|
2147
2732
|
let pendingOutcomes = null;
|
|
2148
2733
|
if (attachments && attachments.inputs.length > 0) {
|
|
@@ -2181,22 +2766,10 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2181
2766
|
const READ_BACK_DELAY_MS = 150;
|
|
2182
2767
|
for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
|
|
2183
2768
|
const after = await getSessionMessages(port, sessionId, client);
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
const id = idOf(m);
|
|
2189
|
-
if (typeof id !== "string" || knownUserIds.has(id)) continue;
|
|
2190
|
-
if (messageText(m) !== content) continue;
|
|
2191
|
-
const created = createdOf(m) ?? 0;
|
|
2192
|
-
if (best === null || created > best.created) {
|
|
2193
|
-
best = { id, created };
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
if (best) {
|
|
2197
|
-
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2198
|
-
return best.id;
|
|
2199
|
-
}
|
|
2769
|
+
const dispatchedUserMessageId = findDispatchedUserMessageId(after, content, knownUserIds);
|
|
2770
|
+
if (dispatchedUserMessageId !== null) {
|
|
2771
|
+
if (pendingOutcomes && attachments?.onOutcomes) attachments.onOutcomes(pendingOutcomes);
|
|
2772
|
+
return dispatchedUserMessageId;
|
|
2200
2773
|
}
|
|
2201
2774
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
2202
2775
|
await new Promise((resolve4) => setTimeout(resolve4, READ_BACK_DELAY_MS));
|
|
@@ -2207,13 +2780,13 @@ async function sendPromptAsync(port, sessionId, content, options, attachments, c
|
|
|
2207
2780
|
function findAssistantReplyAfter(messages, userMessageId) {
|
|
2208
2781
|
if (!messages || messages.length === 0) return null;
|
|
2209
2782
|
const byParent = messages.find(
|
|
2210
|
-
(m) =>
|
|
2783
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2211
2784
|
);
|
|
2212
2785
|
if (byParent) return byParent;
|
|
2213
|
-
const userIndex = messages.findIndex((m) =>
|
|
2786
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2214
2787
|
if (userIndex === -1) return null;
|
|
2215
2788
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2216
|
-
if (
|
|
2789
|
+
if (roleOf2(messages[i]) === "assistant") return messages[i];
|
|
2217
2790
|
}
|
|
2218
2791
|
return null;
|
|
2219
2792
|
}
|
|
@@ -2223,7 +2796,7 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2223
2796
|
let lastNonErrored = null;
|
|
2224
2797
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2225
2798
|
const m = messages[i];
|
|
2226
|
-
if (
|
|
2799
|
+
if (roleOf2(m) !== "assistant" || parentIdOf2(m) !== userMessageId) continue;
|
|
2227
2800
|
if (lastCorrelated === null) lastCorrelated = m;
|
|
2228
2801
|
if (errorOf(m) == null) {
|
|
2229
2802
|
lastNonErrored = m;
|
|
@@ -2231,12 +2804,12 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2231
2804
|
}
|
|
2232
2805
|
}
|
|
2233
2806
|
if (lastCorrelated) return lastNonErrored ?? lastCorrelated;
|
|
2234
|
-
const userIndex = messages.findIndex((m) =>
|
|
2807
|
+
const userIndex = messages.findIndex((m) => idOf2(m) === userMessageId);
|
|
2235
2808
|
if (userIndex === -1) return null;
|
|
2236
2809
|
let last = null;
|
|
2237
2810
|
let lastOk = null;
|
|
2238
2811
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2239
|
-
const role =
|
|
2812
|
+
const role = roleOf2(messages[i]);
|
|
2240
2813
|
if (role === "user") break;
|
|
2241
2814
|
if (role === "assistant") {
|
|
2242
2815
|
last = messages[i];
|
|
@@ -2248,16 +2821,16 @@ function findLastAssistantReplyFor(messages, userMessageId) {
|
|
|
2248
2821
|
function collectSubagentSessions(messages, userMessageId) {
|
|
2249
2822
|
if (!messages || messages.length === 0) return [];
|
|
2250
2823
|
const byParent = messages.filter(
|
|
2251
|
-
(message) =>
|
|
2824
|
+
(message) => roleOf2(message) === "assistant" && parentIdOf2(message) === userMessageId
|
|
2252
2825
|
);
|
|
2253
2826
|
const assistants = byParent.length > 0 ? byParent : [];
|
|
2254
2827
|
if (assistants.length === 0) {
|
|
2255
|
-
const userIndex = messages.findIndex((message) =>
|
|
2828
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2256
2829
|
if (userIndex === -1) return [];
|
|
2257
2830
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2258
2831
|
const message = messages[i];
|
|
2259
|
-
if (
|
|
2260
|
-
if (
|
|
2832
|
+
if (roleOf2(message) === "user") break;
|
|
2833
|
+
if (roleOf2(message) === "assistant") assistants.push(message);
|
|
2261
2834
|
}
|
|
2262
2835
|
}
|
|
2263
2836
|
const refs = [];
|
|
@@ -2297,7 +2870,7 @@ function collectTaskCalls(messages, userMessageId) {
|
|
|
2297
2870
|
if (!messages || messages.length === 0) return [];
|
|
2298
2871
|
const calls = [];
|
|
2299
2872
|
for (const message of messages) {
|
|
2300
|
-
if (
|
|
2873
|
+
if (roleOf2(message) !== "assistant" || parentIdOf2(message) !== userMessageId) continue;
|
|
2301
2874
|
for (const part of message.parts ?? []) {
|
|
2302
2875
|
if (part.tool !== "task" || !part.callID || !part.state || part.state.status === "pending") {
|
|
2303
2876
|
continue;
|
|
@@ -2327,7 +2900,7 @@ function attributeTaskCallUsage(messages, windows) {
|
|
|
2327
2900
|
for (const window of eligibleWindows) assignments.set(window.callID, []);
|
|
2328
2901
|
const unattributed = [];
|
|
2329
2902
|
for (const message of messages ?? []) {
|
|
2330
|
-
if (
|
|
2903
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2331
2904
|
const created = finiteNumber(createdOf(message));
|
|
2332
2905
|
const matching = created === null ? [] : eligibleWindows.filter(
|
|
2333
2906
|
(window) => window.timeStart <= created && (window.timeEnd === null || window.timeEnd === void 0 || created <= window.timeEnd)
|
|
@@ -2404,7 +2977,7 @@ function sumAssistantUsage(messages) {
|
|
|
2404
2977
|
function messageUsage(messages, userMessageId) {
|
|
2405
2978
|
if (!messages || messages.length === 0) return null;
|
|
2406
2979
|
const byParentAll = messages.filter(
|
|
2407
|
-
(m) =>
|
|
2980
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) === userMessageId
|
|
2408
2981
|
);
|
|
2409
2982
|
const byParentNonErrored = byParentAll.filter((m) => errorOf(m) == null);
|
|
2410
2983
|
const byParent = byParentNonErrored.length > 0 ? byParentNonErrored : byParentAll;
|
|
@@ -2420,7 +2993,7 @@ function messageUsage(messages, userMessageId) {
|
|
|
2420
2993
|
}
|
|
2421
2994
|
function messageRunState(messages, userMessageId) {
|
|
2422
2995
|
if (!messages || messages.length === 0) return "unknown";
|
|
2423
|
-
const hasUser = messages.some((m) =>
|
|
2996
|
+
const hasUser = messages.some((m) => idOf2(m) === userMessageId);
|
|
2424
2997
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2425
2998
|
if (!hasUser) {
|
|
2426
2999
|
if (!reply) return "unknown";
|
|
@@ -2512,7 +3085,7 @@ function findLatestSubagentAuthOutcome(messages, sinceMs) {
|
|
|
2512
3085
|
if (!messages || messages.length === 0) return null;
|
|
2513
3086
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2514
3087
|
const message = messages[i];
|
|
2515
|
-
if (
|
|
3088
|
+
if (roleOf2(message) !== "assistant") continue;
|
|
2516
3089
|
const created = createdOf(message);
|
|
2517
3090
|
if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
|
|
2518
3091
|
const failure = classifyReplyAuthError(message);
|
|
@@ -2544,25 +3117,25 @@ function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProvi
|
|
|
2544
3117
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
2545
3118
|
if (!messages || messages.length === 0) return false;
|
|
2546
3119
|
return messages.some(
|
|
2547
|
-
(m) =>
|
|
3120
|
+
(m) => roleOf2(m) === "assistant" && parentIdOf2(m) !== exceptUserMessageId && isAssistantInFlight(m)
|
|
2548
3121
|
);
|
|
2549
3122
|
}
|
|
2550
3123
|
function hasLaterSiblingTurnStarted(messages, userMessageId, siblingUserMessageIds) {
|
|
2551
3124
|
if (!messages || messages.length === 0) return false;
|
|
2552
|
-
const userIndex = messages.findIndex((message) =>
|
|
3125
|
+
const userIndex = messages.findIndex((message) => idOf2(message) === userMessageId);
|
|
2553
3126
|
if (userIndex === -1) return false;
|
|
2554
3127
|
let hasLaterUser = false;
|
|
2555
3128
|
let hasStartedLaterUser = false;
|
|
2556
3129
|
for (let i = userIndex + 1; i < messages.length; i++) {
|
|
2557
3130
|
const message = messages[i];
|
|
2558
|
-
if (
|
|
3131
|
+
if (roleOf2(message) !== "user") continue;
|
|
2559
3132
|
hasLaterUser = true;
|
|
2560
|
-
const laterUserMessageId =
|
|
3133
|
+
const laterUserMessageId = idOf2(message);
|
|
2561
3134
|
if (laterUserMessageId === void 0 || !siblingUserMessageIds.has(laterUserMessageId)) {
|
|
2562
3135
|
return false;
|
|
2563
3136
|
}
|
|
2564
3137
|
if (messages.some(
|
|
2565
|
-
(candidate) =>
|
|
3138
|
+
(candidate) => roleOf2(candidate) === "assistant" && parentIdOf2(candidate) === laterUserMessageId
|
|
2566
3139
|
)) {
|
|
2567
3140
|
hasStartedLaterUser = true;
|
|
2568
3141
|
}
|
|
@@ -3033,13 +3606,17 @@ async function listV2Sessions(client) {
|
|
|
3033
3606
|
}
|
|
3034
3607
|
async function deleteV2Session(client, sessionId) {
|
|
3035
3608
|
try {
|
|
3036
|
-
await client.request(
|
|
3037
|
-
|
|
3609
|
+
const response = await client.request(
|
|
3610
|
+
`/api/session/${encodeURIComponent(sessionId)}`,
|
|
3611
|
+
{ method: "DELETE" },
|
|
3612
|
+
{ allowStatuses: Array.from({ length: 500 }, (_, index) => index + 100) }
|
|
3613
|
+
);
|
|
3614
|
+
return { ok: response.status >= 200 && response.status < 300, status: response.status };
|
|
3038
3615
|
} catch (error2) {
|
|
3039
3616
|
console.error(
|
|
3040
3617
|
`[deleteV2Session] DELETE /api/session/${sessionId} failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
3041
3618
|
);
|
|
3042
|
-
return false;
|
|
3619
|
+
return { ok: false, status: "threw" };
|
|
3043
3620
|
}
|
|
3044
3621
|
}
|
|
3045
3622
|
async function v2SessionExists(client, sessionId) {
|
|
@@ -3214,7 +3791,7 @@ import { join as join4 } from "node:path";
|
|
|
3214
3791
|
|
|
3215
3792
|
// src/lib/openai-usage.ts
|
|
3216
3793
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
3217
|
-
import { homedir as
|
|
3794
|
+
import { homedir as homedir3 } from "node:os";
|
|
3218
3795
|
import { join as join3 } from "node:path";
|
|
3219
3796
|
var CODEX_RESPONSES_URL = "https://chatgpt.com/backend-api/codex/responses";
|
|
3220
3797
|
var OPENCODE_AUTH_SEGMENTS = [".local", "share", "opencode", "auth.json"];
|
|
@@ -3229,7 +3806,7 @@ function isLocalCredentialProblem2(err) {
|
|
|
3229
3806
|
}
|
|
3230
3807
|
function readOpenCodeChatGptCredentials() {
|
|
3231
3808
|
try {
|
|
3232
|
-
const raw = readFileSync3(join3(
|
|
3809
|
+
const raw = readFileSync3(join3(homedir3(), ...OPENCODE_AUTH_SEGMENTS), "utf-8");
|
|
3233
3810
|
let parsed;
|
|
3234
3811
|
try {
|
|
3235
3812
|
parsed = JSON.parse(raw);
|
|
@@ -3370,16 +3947,17 @@ function hasPrimaryHeaders(headers) {
|
|
|
3370
3947
|
].some((name) => headers.has(name));
|
|
3371
3948
|
}
|
|
3372
3949
|
async function getOpenAiUsage(port, client) {
|
|
3950
|
+
const openCodeVersion = client?.version ?? "v1";
|
|
3373
3951
|
const credentials2 = readOpenCodeChatGptCredentials();
|
|
3374
3952
|
if (!credentials2) {
|
|
3375
3953
|
throw new OpenAiUsageError(
|
|
3376
|
-
"No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3954
|
+
openCodeVersion === "v2" ? "OpenAI usage reporting is not available on OpenCode V2 because V2 keeps a connected ChatGPT login in a store Evident cannot read." : "No ChatGPT login found. Connect a ChatGPT account to this runner, or run `opencode auth login`.",
|
|
3377
3955
|
"no_credentials"
|
|
3378
3956
|
);
|
|
3379
3957
|
}
|
|
3380
3958
|
if (credentials2.expiresAt < Date.now()) {
|
|
3381
3959
|
throw new OpenAiUsageError(
|
|
3382
|
-
"ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3960
|
+
openCodeVersion === "v2" ? "The delivered ChatGPT login has expired. OpenCode V2 cannot be given a replacement by file; connect the account again from the runner's page in Evident." : "ChatGPT credentials have expired. Run `opencode auth login` to refresh them.",
|
|
3383
3961
|
"credentials_expired"
|
|
3384
3962
|
);
|
|
3385
3963
|
}
|
|
@@ -3660,9 +4238,9 @@ async function importOpenCodeV2Credentials({
|
|
|
3660
4238
|
|
|
3661
4239
|
// src/lib/opencode/session-db-boot.ts
|
|
3662
4240
|
import { spawn as spawn2 } from "node:child_process";
|
|
3663
|
-
import { mkdirSync, statSync as
|
|
3664
|
-
import { homedir as
|
|
3665
|
-
import { dirname as
|
|
4241
|
+
import { mkdirSync, statSync as statSync3, unlinkSync as unlinkSync2, writeFileSync } from "node:fs";
|
|
4242
|
+
import { homedir as homedir4 } from "node:os";
|
|
4243
|
+
import { dirname as dirname3, resolve as resolvePath } from "node:path";
|
|
3666
4244
|
|
|
3667
4245
|
// src/lib/runner-synchroniser.ts
|
|
3668
4246
|
import { spawn } from "node:child_process";
|
|
@@ -3787,7 +4365,7 @@ function clearMarker(options) {
|
|
|
3787
4365
|
function markNoReplicate(options, message) {
|
|
3788
4366
|
if (options.noReplicateMarker) {
|
|
3789
4367
|
try {
|
|
3790
|
-
mkdirSync(
|
|
4368
|
+
mkdirSync(dirname3(options.noReplicateMarker), { recursive: true });
|
|
3791
4369
|
writeFileSync(options.noReplicateMarker, "");
|
|
3792
4370
|
} catch (error2) {
|
|
3793
4371
|
options.log(
|
|
@@ -3912,7 +4490,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3912
4490
|
return null;
|
|
3913
4491
|
}
|
|
3914
4492
|
try {
|
|
3915
|
-
if (
|
|
4493
|
+
if (statSync3(configPath).size > 0) return configPath;
|
|
3916
4494
|
} catch (error2) {
|
|
3917
4495
|
if (!(error2 instanceof Error && "code" in error2 && error2.code === "ENOENT")) {
|
|
3918
4496
|
options.log(
|
|
@@ -3942,7 +4520,7 @@ async function ensureLitestreamConfig(options, env) {
|
|
|
3942
4520
|
return null;
|
|
3943
4521
|
}
|
|
3944
4522
|
try {
|
|
3945
|
-
mkdirSync(
|
|
4523
|
+
mkdirSync(dirname3(configPath), { recursive: true });
|
|
3946
4524
|
writeFileSync(configPath, rendered.stdout);
|
|
3947
4525
|
} catch (error2) {
|
|
3948
4526
|
options.log(
|
|
@@ -4017,14 +4595,14 @@ async function restoreSessionDb(options, configPath, env) {
|
|
|
4017
4595
|
case 0:
|
|
4018
4596
|
return;
|
|
4019
4597
|
case 31:
|
|
4020
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4598
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4021
4599
|
options.log(
|
|
4022
4600
|
"SESSION-DB-REPLICA-UNUSABLE: booting with a fresh opencode.db and replicating into the existing prefix this boot",
|
|
4023
4601
|
"warn"
|
|
4024
4602
|
);
|
|
4025
4603
|
return;
|
|
4026
4604
|
case 32:
|
|
4027
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4605
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4028
4606
|
discardSessionDbDebris(options);
|
|
4029
4607
|
markNoReplicate(
|
|
4030
4608
|
options,
|
|
@@ -4103,7 +4681,7 @@ async function verifySessionDb(options, configPath, env) {
|
|
|
4103
4681
|
}
|
|
4104
4682
|
function fileExists(path) {
|
|
4105
4683
|
try {
|
|
4106
|
-
|
|
4684
|
+
statSync3(path);
|
|
4107
4685
|
return true;
|
|
4108
4686
|
} catch (error2) {
|
|
4109
4687
|
if (error2 instanceof Error && "code" in error2 && error2.code === "ENOENT") return false;
|
|
@@ -4113,7 +4691,7 @@ function fileExists(path) {
|
|
|
4113
4691
|
async function restoreAndVerifySessionDb(options) {
|
|
4114
4692
|
const env = options.env ?? process.env;
|
|
4115
4693
|
clearMarker(options);
|
|
4116
|
-
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(
|
|
4694
|
+
acknowledgeSessionDbRecoveryReport(sessionDbRecoveryReportPath(homedir4(), env));
|
|
4117
4695
|
const synchroniserEnv = await runSynchroniser(["env"], {
|
|
4118
4696
|
timeoutMs: SESSION_DB_SYNCHRONISER_TIMEOUT_MS,
|
|
4119
4697
|
env
|
|
@@ -4192,7 +4770,7 @@ async function restoreAndVerifySessionDb(options) {
|
|
|
4192
4770
|
// src/lib/opencode/session-db-provenance.ts
|
|
4193
4771
|
import { createRequire } from "node:module";
|
|
4194
4772
|
import { mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "node:fs";
|
|
4195
|
-
import { dirname as
|
|
4773
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
4196
4774
|
var require2 = createRequire(import.meta.url);
|
|
4197
4775
|
function readSessionDbMigrationIds(dbPath) {
|
|
4198
4776
|
let db;
|
|
@@ -4258,7 +4836,7 @@ function loadSessionDbProvenanceState(path) {
|
|
|
4258
4836
|
}
|
|
4259
4837
|
function saveSessionDbProvenanceState(path, state) {
|
|
4260
4838
|
try {
|
|
4261
|
-
mkdirSync2(
|
|
4839
|
+
mkdirSync2(dirname4(path), { recursive: true });
|
|
4262
4840
|
writeFileSync2(path, `${JSON.stringify(state, null, 2)}
|
|
4263
4841
|
`, "utf8");
|
|
4264
4842
|
} catch (error2) {
|
|
@@ -4493,12 +5071,15 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4493
5071
|
}
|
|
4494
5072
|
} catch {
|
|
4495
5073
|
try {
|
|
4496
|
-
const psOutput = execSync(
|
|
5074
|
+
const psOutput = execSync("ps aux", {
|
|
4497
5075
|
encoding: "utf-8",
|
|
4498
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
5076
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
5077
|
+
// The whole table reaches Node; the 1 MiB default would surface a spurious failure.
|
|
5078
|
+
maxBuffer: 16 * 1024 * 1024
|
|
4499
5079
|
}).trim();
|
|
4500
5080
|
if (psOutput) {
|
|
4501
5081
|
for (const line of psOutput.split("\n")) {
|
|
5082
|
+
if (!psPattern.test(line)) continue;
|
|
4502
5083
|
const parts = line.trim().split(/\s+/);
|
|
4503
5084
|
if (parts.length >= 2) {
|
|
4504
5085
|
const pid = parseInt(parts[1], 10);
|
|
@@ -4507,8 +5088,9 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4507
5088
|
}
|
|
4508
5089
|
}
|
|
4509
5090
|
} catch (err) {
|
|
5091
|
+
const reason = (typeof err === "object" && err !== null && "stderr" in err && typeof err.stderr === "string" ? err.stderr : "").split("\n").map((line) => line.trim()).find(Boolean) ?? (err instanceof Error ? err.message : String(err)).split("\n").map((line) => line.trim()).find(Boolean) ?? "no error detail available";
|
|
4510
5092
|
console.warn(
|
|
4511
|
-
`
|
|
5093
|
+
`Could not list running processes, so an already-running OpenCode may not be detected: ${reason}`
|
|
4512
5094
|
);
|
|
4513
5095
|
}
|
|
4514
5096
|
}
|
|
@@ -4540,7 +5122,7 @@ function findProcessesByPattern(pgrepPattern, psPattern) {
|
|
|
4540
5122
|
return instances;
|
|
4541
5123
|
}
|
|
4542
5124
|
function findOpenCodeProcesses() {
|
|
4543
|
-
return findProcessesByPattern("opencode serve|opencode-serve",
|
|
5125
|
+
return findProcessesByPattern("opencode serve|opencode-serve", /opencode (serve|--port)/);
|
|
4544
5126
|
}
|
|
4545
5127
|
async function scanPortsForOpenCode() {
|
|
4546
5128
|
const instances = [];
|
|
@@ -4835,7 +5417,10 @@ function selectSessionsToDelete(sessions, opts) {
|
|
|
4835
5417
|
toDelete.push(s.id);
|
|
4836
5418
|
}
|
|
4837
5419
|
}
|
|
4838
|
-
|
|
5420
|
+
const activityById = new Map(sessions.map((session) => [session.id, session.lastActivityMs]));
|
|
5421
|
+
return toDelete.sort(
|
|
5422
|
+
(a, b) => (activityById.get(a) ?? -Infinity) - (activityById.get(b) ?? -Infinity)
|
|
5423
|
+
);
|
|
4839
5424
|
}
|
|
4840
5425
|
var DEFAULT_INTERVAL = "1h";
|
|
4841
5426
|
function resolve(flag, envValue, fallback) {
|
|
@@ -4895,13 +5480,13 @@ function resolveSessionCleanupConfig(flags, env = process.env) {
|
|
|
4895
5480
|
}
|
|
4896
5481
|
|
|
4897
5482
|
// src/lib/opencode/session-db-size.ts
|
|
4898
|
-
import { statSync as
|
|
5483
|
+
import { statSync as statSync4 } from "node:fs";
|
|
4899
5484
|
import { join as join6 } from "node:path";
|
|
4900
5485
|
var LARGE_DB_THRESHOLD_BYTES = 268435456;
|
|
4901
5486
|
function statSessionDbBytes(homeDir) {
|
|
4902
5487
|
const dbPath = join6(homeDir, ".local", "share", "opencode", "opencode.db");
|
|
4903
5488
|
try {
|
|
4904
|
-
return
|
|
5489
|
+
return statSync4(dbPath).size;
|
|
4905
5490
|
} catch (err) {
|
|
4906
5491
|
const isMissingFile = err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4907
5492
|
if (!isMissingFile) {
|
|
@@ -4927,12 +5512,12 @@ function buildSessionStoreSizeWarning(input) {
|
|
|
4927
5512
|
}
|
|
4928
5513
|
|
|
4929
5514
|
// src/lib/opencode/log-tail.ts
|
|
4930
|
-
import { statSync as
|
|
4931
|
-
import { homedir as
|
|
5515
|
+
import { statSync as statSync5 } from "node:fs";
|
|
5516
|
+
import { homedir as homedir5 } from "node:os";
|
|
4932
5517
|
import { join as join7 } from "node:path";
|
|
4933
5518
|
import { open as open2, stat } from "node:fs/promises";
|
|
4934
5519
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
4935
|
-
function resolveOpenCodeLogPath(homeDir =
|
|
5520
|
+
function resolveOpenCodeLogPath(homeDir = homedir5(), env = process.env) {
|
|
4936
5521
|
const dataDir = env.XDG_DATA_HOME || join7(homeDir, ".local", "share");
|
|
4937
5522
|
return join7(dataDir, "opencode", "log", "opencode.log");
|
|
4938
5523
|
}
|
|
@@ -4950,7 +5535,7 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
4950
5535
|
let inode = null;
|
|
4951
5536
|
let baselineReady = true;
|
|
4952
5537
|
try {
|
|
4953
|
-
const initial =
|
|
5538
|
+
const initial = statSync5(logPath);
|
|
4954
5539
|
offset = initial.size;
|
|
4955
5540
|
inode = initial.ino;
|
|
4956
5541
|
} catch (error2) {
|
|
@@ -5009,125 +5594,6 @@ function tailOpenCodeLogFile(logPath, onChunk, opts = {}) {
|
|
|
5009
5594
|
};
|
|
5010
5595
|
}
|
|
5011
5596
|
|
|
5012
|
-
// src/lib/opencode/session-db-reclaim.ts
|
|
5013
|
-
import { statSync as statSync5, statfsSync } from "node:fs";
|
|
5014
|
-
import { dirname as dirname4 } from "node:path";
|
|
5015
|
-
function errorMessage(error2) {
|
|
5016
|
-
if (!(error2 instanceof Error)) return String(error2);
|
|
5017
|
-
return error2.cause instanceof Error ? error2.cause.message : error2.message;
|
|
5018
|
-
}
|
|
5019
|
-
function insufficientSpaceReason(dbPath, requiredBytes) {
|
|
5020
|
-
try {
|
|
5021
|
-
const fsStats = statfsSync(dirname4(dbPath));
|
|
5022
|
-
const availableBytes = fsStats.bavail * fsStats.bsize;
|
|
5023
|
-
if (availableBytes < requiredBytes) {
|
|
5024
|
-
return `only ${availableBytes} bytes free, need ${requiredBytes} for a second copy`;
|
|
5025
|
-
}
|
|
5026
|
-
return null;
|
|
5027
|
-
} catch (err) {
|
|
5028
|
-
return `could not check free space (${err instanceof Error ? err.message : String(err)}); refusing to guess`;
|
|
5029
|
-
}
|
|
5030
|
-
}
|
|
5031
|
-
function readLogicalBytes(db) {
|
|
5032
|
-
const pageCount = db.prepare("PRAGMA page_count").get().page_count;
|
|
5033
|
-
const pageSize = db.prepare("PRAGMA page_size").get().page_size;
|
|
5034
|
-
return pageCount * pageSize;
|
|
5035
|
-
}
|
|
5036
|
-
function readCheckpointResult(db) {
|
|
5037
|
-
const row = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
5038
|
-
return { busy: row.busy !== 0, log: row.log, checkpointed: row.checkpointed };
|
|
5039
|
-
}
|
|
5040
|
-
async function probeReclaimAvailability(input) {
|
|
5041
|
-
const { dbPath, requiredBytes } = input;
|
|
5042
|
-
let sqlite;
|
|
5043
|
-
try {
|
|
5044
|
-
sqlite = await import("node:sqlite");
|
|
5045
|
-
} catch (err) {
|
|
5046
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5047
|
-
console.warn(`[probeReclaimAvailability] node:sqlite unavailable for ${dbPath}: ` + detail);
|
|
5048
|
-
return { reason: "sqlite-unavailable", detail };
|
|
5049
|
-
}
|
|
5050
|
-
let autoVacuum = null;
|
|
5051
|
-
try {
|
|
5052
|
-
const db = new sqlite.DatabaseSync(dbPath, { readOnly: true });
|
|
5053
|
-
try {
|
|
5054
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5055
|
-
autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5056
|
-
} finally {
|
|
5057
|
-
db.close();
|
|
5058
|
-
}
|
|
5059
|
-
} catch (err) {
|
|
5060
|
-
console.warn(
|
|
5061
|
-
`[probeReclaimAvailability] could not read auto_vacuum mode for ${dbPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
5062
|
-
);
|
|
5063
|
-
}
|
|
5064
|
-
if (autoVacuum !== 0) return null;
|
|
5065
|
-
return insufficientSpaceReason(dbPath, requiredBytes) !== null ? { reason: "insufficient-disk-space" } : null;
|
|
5066
|
-
}
|
|
5067
|
-
async function reclaimSessionDbSpace(input) {
|
|
5068
|
-
const { dbPath, maxPages, allowFullVacuum = true } = input;
|
|
5069
|
-
let sqlite;
|
|
5070
|
-
try {
|
|
5071
|
-
sqlite = await import("node:sqlite");
|
|
5072
|
-
} catch (err) {
|
|
5073
|
-
const detail = `Node ${process.version}: ${errorMessage(err)}`;
|
|
5074
|
-
console.warn(
|
|
5075
|
-
`[reclaimSessionDbSpace] node:sqlite unavailable (need Node >=22.5, >=22.13 unflagged) for ${dbPath}: ${detail}`
|
|
5076
|
-
);
|
|
5077
|
-
return { ok: false, skipped: "sqlite-unavailable", detail };
|
|
5078
|
-
}
|
|
5079
|
-
const { DatabaseSync } = sqlite;
|
|
5080
|
-
let db;
|
|
5081
|
-
try {
|
|
5082
|
-
db = new DatabaseSync(dbPath);
|
|
5083
|
-
db.exec("PRAGMA busy_timeout=5000");
|
|
5084
|
-
const autoVacuum = db.prepare("PRAGMA auto_vacuum").get().auto_vacuum;
|
|
5085
|
-
if (autoVacuum === 0) {
|
|
5086
|
-
if (!allowFullVacuum) {
|
|
5087
|
-
console.warn(
|
|
5088
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: a session turn is live`
|
|
5089
|
-
);
|
|
5090
|
-
return { ok: false, skipped: "full-vacuum-blocked" };
|
|
5091
|
-
}
|
|
5092
|
-
const fileBytesForGuard = statSync5(dbPath).size;
|
|
5093
|
-
const skipReason = insufficientSpaceReason(dbPath, fileBytesForGuard);
|
|
5094
|
-
if (skipReason !== null) {
|
|
5095
|
-
console.warn(
|
|
5096
|
-
`[reclaimSessionDbSpace] skipping VACUUM conversion of ${dbPath}: ${skipReason}`
|
|
5097
|
-
);
|
|
5098
|
-
return { ok: false, skipped: "insufficient-disk-space" };
|
|
5099
|
-
}
|
|
5100
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5101
|
-
db.exec("PRAGMA auto_vacuum=INCREMENTAL");
|
|
5102
|
-
db.exec("VACUUM");
|
|
5103
|
-
const afterBytes = readLogicalBytes(db);
|
|
5104
|
-
const checkpoint = readCheckpointResult(db);
|
|
5105
|
-
return { ok: true, mode: "convert", beforeBytes, afterBytes, checkpoint };
|
|
5106
|
-
}
|
|
5107
|
-
if (autoVacuum === 2) {
|
|
5108
|
-
const beforeBytes = readLogicalBytes(db);
|
|
5109
|
-
const bound = Math.max(0, Math.trunc(maxPages));
|
|
5110
|
-
db.exec(`PRAGMA incremental_vacuum(${bound})`);
|
|
5111
|
-
const afterBytes = readLogicalBytes(db);
|
|
5112
|
-
const checkpoint = readCheckpointResult(db);
|
|
5113
|
-
return { ok: true, mode: "incremental", beforeBytes, afterBytes, checkpoint };
|
|
5114
|
-
}
|
|
5115
|
-
console.warn(
|
|
5116
|
-
`[reclaimSessionDbSpace] ${dbPath} has auto_vacuum=${autoVacuum} (neither NONE nor INCREMENTAL); nothing to reclaim`
|
|
5117
|
-
);
|
|
5118
|
-
return { ok: false, skipped: "auto-vacuum-not-applicable" };
|
|
5119
|
-
} catch (err) {
|
|
5120
|
-
console.error(`[reclaimSessionDbSpace] reclaim failed for ${dbPath}: ` + errorMessage(err));
|
|
5121
|
-
return {
|
|
5122
|
-
ok: false,
|
|
5123
|
-
skipped: "reclaim-error",
|
|
5124
|
-
detail: errorMessage(err)
|
|
5125
|
-
};
|
|
5126
|
-
} finally {
|
|
5127
|
-
db?.close();
|
|
5128
|
-
}
|
|
5129
|
-
}
|
|
5130
|
-
|
|
5131
5597
|
// src/lib/tunnel/connection.ts
|
|
5132
5598
|
import WebSocket2 from "ws";
|
|
5133
5599
|
|
|
@@ -5357,6 +5823,8 @@ var StreamForwarder = class {
|
|
|
5357
5823
|
if (done) break;
|
|
5358
5824
|
const chunk = Buffer.from(value);
|
|
5359
5825
|
for (let i = 0; i < chunk.length; i += MAX_FRAME_BYTES) {
|
|
5826
|
+
await stream.pausePromise;
|
|
5827
|
+
if (ac.signal.aborted) break;
|
|
5360
5828
|
const slice = chunk.subarray(i, i + MAX_FRAME_BYTES);
|
|
5361
5829
|
if (this.binaryFramesSupported) {
|
|
5362
5830
|
if (this.ws.readyState === WebSocket.OPEN) {
|
|
@@ -5399,6 +5867,7 @@ function classifyUpgradeRejection(headers) {
|
|
|
5399
5867
|
}
|
|
5400
5868
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
5401
5869
|
var BASE_RECONNECT_DELAY = 500;
|
|
5870
|
+
var RELAY_SILENCE_TIMEOUT_MS = 9e4;
|
|
5402
5871
|
function getReconnectDelay(attempt) {
|
|
5403
5872
|
const exponentialDelay = BASE_RECONNECT_DELAY * Math.pow(2, attempt);
|
|
5404
5873
|
const jitter = Math.random() * 1e3;
|
|
@@ -5470,10 +5939,27 @@ function connectTunnel(options) {
|
|
|
5470
5939
|
ws.close();
|
|
5471
5940
|
reject(new Error("Connection timeout"));
|
|
5472
5941
|
}, 3e4);
|
|
5942
|
+
let relaySilenceTimeout;
|
|
5943
|
+
let relaySilenceTimedOut = false;
|
|
5944
|
+
const clearRelaySilenceTimeout = () => {
|
|
5945
|
+
if (relaySilenceTimeout === void 0) return;
|
|
5946
|
+
clearTimeout(relaySilenceTimeout);
|
|
5947
|
+
relaySilenceTimeout = void 0;
|
|
5948
|
+
};
|
|
5949
|
+
const armRelaySilenceTimeout = () => {
|
|
5950
|
+
clearRelaySilenceTimeout();
|
|
5951
|
+
relaySilenceTimeout = setTimeout(() => {
|
|
5952
|
+
relaySilenceTimeout = void 0;
|
|
5953
|
+
relaySilenceTimedOut = true;
|
|
5954
|
+
ws.terminate();
|
|
5955
|
+
}, RELAY_SILENCE_TIMEOUT_MS);
|
|
5956
|
+
relaySilenceTimeout.unref?.();
|
|
5957
|
+
};
|
|
5473
5958
|
let upgradeRejection = null;
|
|
5474
5959
|
let upgradeRejectionReason = null;
|
|
5475
5960
|
ws.on("unexpected-response", (_req, res) => {
|
|
5476
5961
|
clearTimeout(connectionTimeout);
|
|
5962
|
+
clearRelaySilenceTimeout();
|
|
5477
5963
|
const reason = classifyUpgradeRejection(res.headers);
|
|
5478
5964
|
upgradeRejectionReason = reason;
|
|
5479
5965
|
const chunks = [];
|
|
@@ -5503,6 +5989,7 @@ function connectTunnel(options) {
|
|
|
5503
5989
|
onInfo?.("WebSocket connection established");
|
|
5504
5990
|
});
|
|
5505
5991
|
ws.on("message", (data, isBinary) => {
|
|
5992
|
+
if (relaySilenceTimeout !== void 0) armRelaySilenceTimeout();
|
|
5506
5993
|
if (isBinary) {
|
|
5507
5994
|
try {
|
|
5508
5995
|
const frame = decodeBinaryBodyFrame(toUint8Array(data));
|
|
@@ -5536,6 +6023,7 @@ function connectTunnel(options) {
|
|
|
5536
6023
|
switch (message.type) {
|
|
5537
6024
|
case "connected": {
|
|
5538
6025
|
clearTimeout(connectionTimeout);
|
|
6026
|
+
armRelaySilenceTimeout();
|
|
5539
6027
|
forwarder.setBinaryFramesSupported(message.binary_frames === true);
|
|
5540
6028
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
5541
6029
|
onConnected?.(connectedAgentId);
|
|
@@ -5560,6 +6048,7 @@ function connectTunnel(options) {
|
|
|
5560
6048
|
});
|
|
5561
6049
|
ws.on("error", (error2) => {
|
|
5562
6050
|
clearTimeout(connectionTimeout);
|
|
6051
|
+
clearRelaySilenceTimeout();
|
|
5563
6052
|
const detail = upgradeRejection ?? describeSocketError(error2, url);
|
|
5564
6053
|
if (upgradeRejectionReason === "do_code_updated") {
|
|
5565
6054
|
onWarning?.("Relay redeployed \u2014 reconnecting");
|
|
@@ -5571,7 +6060,8 @@ function connectTunnel(options) {
|
|
|
5571
6060
|
);
|
|
5572
6061
|
});
|
|
5573
6062
|
ws.on("close", (code, reason) => {
|
|
5574
|
-
|
|
6063
|
+
clearRelaySilenceTimeout();
|
|
6064
|
+
const reasonStr = (relaySilenceTimedOut ? "no relay heartbeat for 90s \u2014 reconnecting" : void 0) || reason.toString() || upgradeRejection || (code === 1006 ? "abnormal closure" : "No reason provided");
|
|
5575
6065
|
forwarder.abortAll();
|
|
5576
6066
|
onDisconnected?.(code, reasonStr);
|
|
5577
6067
|
});
|
|
@@ -5579,9 +6069,10 @@ function connectTunnel(options) {
|
|
|
5579
6069
|
}
|
|
5580
6070
|
|
|
5581
6071
|
// src/lib/tunnel/runner-connection.ts
|
|
5582
|
-
var RunnerConnection = class {
|
|
6072
|
+
var RunnerConnection = class _RunnerConnection {
|
|
5583
6073
|
opts;
|
|
5584
6074
|
sleep;
|
|
6075
|
+
now;
|
|
5585
6076
|
connection = null;
|
|
5586
6077
|
resolvedAgentId;
|
|
5587
6078
|
/** True while a (re)connect loop is in flight. */
|
|
@@ -5590,10 +6081,17 @@ var RunnerConnection = class {
|
|
|
5590
6081
|
reconnectPromise = null;
|
|
5591
6082
|
/** 1-based count of the current reconnect attempt streak. */
|
|
5592
6083
|
reconnectAttempt = 0;
|
|
6084
|
+
generation = 0;
|
|
6085
|
+
disconnectedAtMs = null;
|
|
6086
|
+
static RELAY_RECONNECT_CLOSE_REASONS = /* @__PURE__ */ new Set([
|
|
6087
|
+
"No pong received",
|
|
6088
|
+
"Orphaned socket"
|
|
6089
|
+
]);
|
|
5593
6090
|
constructor(opts) {
|
|
5594
6091
|
this.opts = opts;
|
|
5595
6092
|
this.resolvedAgentId = opts.agentId;
|
|
5596
6093
|
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
6094
|
+
this.now = opts.now ?? Date.now;
|
|
5597
6095
|
}
|
|
5598
6096
|
get agentId() {
|
|
5599
6097
|
return this.resolvedAgentId;
|
|
@@ -5629,14 +6127,24 @@ var RunnerConnection = class {
|
|
|
5629
6127
|
port: this.opts.port,
|
|
5630
6128
|
openCodePassword: this.opts.openCodePassword,
|
|
5631
6129
|
onConnected: (agentId) => {
|
|
6130
|
+
const attempt = this.reconnectAttempt + 1;
|
|
5632
6131
|
this.reconnectAttempt = 0;
|
|
5633
6132
|
this.reconnecting = false;
|
|
5634
6133
|
this.resolvedAgentId = agentId;
|
|
5635
|
-
|
|
6134
|
+
this.generation++;
|
|
6135
|
+
const downtimeMs = this.disconnectedAtMs === null ? null : this.now() - this.disconnectedAtMs;
|
|
6136
|
+
this.disconnectedAtMs = null;
|
|
6137
|
+
events.onConnected(agentId, isReconnect, {
|
|
6138
|
+
generation: this.generation,
|
|
6139
|
+
attempt,
|
|
6140
|
+
downtimeMs
|
|
6141
|
+
});
|
|
5636
6142
|
},
|
|
5637
6143
|
onDisconnected: (code, reason) => {
|
|
5638
|
-
|
|
5639
|
-
|
|
6144
|
+
this.disconnectedAtMs = this.now();
|
|
6145
|
+
events.onDisconnected(code, reason, { generation: this.generation });
|
|
6146
|
+
const relayRequestedReconnect = code === 1e3 && _RunnerConnection.RELAY_RECONNECT_CLOSE_REASONS.has(reason);
|
|
6147
|
+
if (this.opts.isRunning() && (code !== 1e3 || relayRequestedReconnect) && !this.reconnecting) {
|
|
5640
6148
|
this.reconnectPromise = this.connectWithRetry(true).catch((err) => {
|
|
5641
6149
|
events.onError?.(`Reconnection failed: ${err.message}`);
|
|
5642
6150
|
});
|
|
@@ -5975,8 +6483,85 @@ function createResourceUsageCollector(homeDir) {
|
|
|
5975
6483
|
};
|
|
5976
6484
|
}
|
|
5977
6485
|
|
|
6486
|
+
// src/lib/resource-usage-low-disk.ts
|
|
6487
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_BYTES = 4 * 1024 ** 3;
|
|
6488
|
+
var RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION = 0.25;
|
|
6489
|
+
var RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES = 1 * 1024 ** 3;
|
|
6490
|
+
var RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES = 256 * 1024 ** 2;
|
|
6491
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES = 6 * 1024 ** 3;
|
|
6492
|
+
var RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION = 0.375;
|
|
6493
|
+
var LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
6494
|
+
var BAND_RANK = {
|
|
6495
|
+
low: 1,
|
|
6496
|
+
very_low: 2,
|
|
6497
|
+
critical: 3
|
|
6498
|
+
};
|
|
6499
|
+
function bandFor(freeBytes) {
|
|
6500
|
+
if (freeBytes < RUNNER_DISK_ALERT_CRITICAL_FREE_BYTES) return "critical";
|
|
6501
|
+
if (freeBytes < RUNNER_DISK_ALERT_VERY_LOW_FREE_BYTES) return "very_low";
|
|
6502
|
+
return "low";
|
|
6503
|
+
}
|
|
6504
|
+
function formatBytes(bytes) {
|
|
6505
|
+
if (bytes === null) return "unavailable";
|
|
6506
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
6507
|
+
const unitIndex = Math.min(Math.floor(Math.log2(Math.max(bytes, 1)) / 10), units.length - 1);
|
|
6508
|
+
const value = bytes / 1024 ** unitIndex;
|
|
6509
|
+
const digits = unitIndex === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
|
|
6510
|
+
return `${value.toFixed(digits)} ${units[unitIndex]}`;
|
|
6511
|
+
}
|
|
6512
|
+
function warningMessage(band, diskFreeBytes, diskTotalBytes, opencodeDbBytes) {
|
|
6513
|
+
return `Low disk warning (${band} severity): ${formatBytes(diskFreeBytes)} free of ${formatBytes(diskTotalBytes)}; opencode.db is ${formatBytes(opencodeDbBytes)}. Clear space on the machine running this runner before session history or new work can no longer be saved.`;
|
|
6514
|
+
}
|
|
6515
|
+
function evaluateLowDiskWarning(input) {
|
|
6516
|
+
const { diskFreeBytes, diskTotalBytes, opencodeDbBytes, nowMs, state } = input;
|
|
6517
|
+
if (diskFreeBytes === null || diskTotalBytes === null || diskTotalBytes <= 0 || diskFreeBytes >= diskTotalBytes) {
|
|
6518
|
+
return { warning: null, state };
|
|
6519
|
+
}
|
|
6520
|
+
const recovered = diskFreeBytes >= RUNNER_DISK_ALERT_RECOVERY_FREE_BYTES || diskFreeBytes >= diskTotalBytes * RUNNER_DISK_ALERT_RECOVERY_FREE_FRACTION;
|
|
6521
|
+
if (recovered) {
|
|
6522
|
+
return { warning: null, state: { worstBand: null, warnedAtMs: null } };
|
|
6523
|
+
}
|
|
6524
|
+
const low = diskFreeBytes < RUNNER_DISK_ALERT_ENTRY_FREE_BYTES && diskFreeBytes < diskTotalBytes * RUNNER_DISK_ALERT_ENTRY_FREE_FRACTION;
|
|
6525
|
+
if (!low) return { warning: null, state };
|
|
6526
|
+
const currentBand = bandFor(diskFreeBytes);
|
|
6527
|
+
const currentRank = BAND_RANK[currentBand];
|
|
6528
|
+
const previousRank = state.worstBand === null ? 0 : BAND_RANK[state.worstBand];
|
|
6529
|
+
const worse = currentRank > previousRank;
|
|
6530
|
+
const reassert = state.warnedAtMs !== null && nowMs - state.warnedAtMs >= LOCAL_DISK_WARNING_REASSERTION_INTERVAL_MS;
|
|
6531
|
+
const nextState = {
|
|
6532
|
+
worstBand: worse ? currentBand : state.worstBand ?? currentBand,
|
|
6533
|
+
warnedAtMs: worse || state.warnedAtMs === null || reassert ? nowMs : state.warnedAtMs
|
|
6534
|
+
};
|
|
6535
|
+
return {
|
|
6536
|
+
warning: worse || state.warnedAtMs === null || reassert ? warningMessage(nextState.worstBand, diskFreeBytes, diskTotalBytes, opencodeDbBytes) : null,
|
|
6537
|
+
state: nextState
|
|
6538
|
+
};
|
|
6539
|
+
}
|
|
6540
|
+
|
|
6541
|
+
// src/lib/opencode/session-store-health.ts
|
|
6542
|
+
var SESSION_STORE_HEALTH_STALENESS_MS = 5 * 6e4;
|
|
6543
|
+
function createSessionStoreHealth(options = {}) {
|
|
6544
|
+
const now = options.now ?? (() => Date.now());
|
|
6545
|
+
const stalenessMs = Math.max(0, options.stalenessMs ?? SESSION_STORE_HEALTH_STALENESS_MS);
|
|
6546
|
+
let lastUnwritableAt = null;
|
|
6547
|
+
let lastWritableAt = null;
|
|
6548
|
+
return {
|
|
6549
|
+
observeUnwritable: () => {
|
|
6550
|
+
lastUnwritableAt = now();
|
|
6551
|
+
},
|
|
6552
|
+
observeWritable: () => {
|
|
6553
|
+
lastWritableAt = now();
|
|
6554
|
+
},
|
|
6555
|
+
isUnwritable: () => {
|
|
6556
|
+
if (lastUnwritableAt === null) return false;
|
|
6557
|
+
if (lastWritableAt !== null && lastWritableAt >= lastUnwritableAt) return false;
|
|
6558
|
+
return now() - lastUnwritableAt < stalenessMs;
|
|
6559
|
+
}
|
|
6560
|
+
};
|
|
6561
|
+
}
|
|
6562
|
+
|
|
5978
6563
|
// src/lib/channels/driver.ts
|
|
5979
|
-
import { homedir as
|
|
6564
|
+
import { homedir as homedir6 } from "node:os";
|
|
5980
6565
|
|
|
5981
6566
|
// src/lib/runner-file-sync.ts
|
|
5982
6567
|
import { join as join9 } from "node:path";
|
|
@@ -6433,6 +7018,7 @@ var SESSION_ERROR_BUFFER_TTL_MS = SESSION_ERROR_STREAM_HEALTHY_MS;
|
|
|
6433
7018
|
var MAX_BUFFERED_SESSION_ERRORS = 256;
|
|
6434
7019
|
var DEFAULT_PAUSED_POLL_INTERVAL_MS = 2e3;
|
|
6435
7020
|
var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
7021
|
+
var DRIVER_SESSION_ENUMERATION_LIMIT = 20;
|
|
6436
7022
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
6437
7023
|
var HEARTBEAT_MS = 6e4;
|
|
6438
7024
|
var ALIVE_WITHHELD_THRESHOLD_MS = 3 * HEARTBEAT_MS;
|
|
@@ -6494,6 +7080,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6494
7080
|
maxActiveSessions;
|
|
6495
7081
|
watcherStallMs;
|
|
6496
7082
|
wedgeWarningIntervalMs;
|
|
7083
|
+
sessionStoreHealth;
|
|
6497
7084
|
openCodeClient;
|
|
6498
7085
|
/** Cache of conversationId → opencode sessionId. */
|
|
6499
7086
|
sessions = /* @__PURE__ */ new Map();
|
|
@@ -6618,10 +7205,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6618
7205
|
* row is `processing` or `pending`; clear it only when absent from both lists.
|
|
6619
7206
|
*/
|
|
6620
7207
|
untrackableAck = /* @__PURE__ */ new Map();
|
|
7208
|
+
/** Session binding used by the first untrackable-ack report, for retries. */
|
|
7209
|
+
untrackableAckSessionIds = /* @__PURE__ */ new Map();
|
|
6621
7210
|
/** Messages whose untrackable-ack failure signal was emitted while fenced. */
|
|
6622
7211
|
untrackableAckSignalled = /* @__PURE__ */ new Set();
|
|
6623
7212
|
/** Messages whose first untrackable-ack fence warning was already logged. */
|
|
6624
7213
|
untrackableAckWarned = /* @__PURE__ */ new Set();
|
|
7214
|
+
/**
|
|
7215
|
+
* Accepted V1 prompts whose assigned ids were not read back; these rows are
|
|
7216
|
+
* re-read instead of POSTed again. A null snapshot cannot safely exclude an older
|
|
7217
|
+
* identical-text turn, so it never produces a correlation.
|
|
7218
|
+
*/
|
|
7219
|
+
awaitingCorrelation = /* @__PURE__ */ new Map();
|
|
6625
7220
|
/** Pending rows seen in the current drain, used to retain terminal dispatch fences. */
|
|
6626
7221
|
pendingMessageIds = /* @__PURE__ */ new Set();
|
|
6627
7222
|
/**
|
|
@@ -6707,21 +7302,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6707
7302
|
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
6708
7303
|
*/
|
|
6709
7304
|
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
6710
|
-
/**
|
|
6711
|
-
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
6712
|
-
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
6713
|
-
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
6714
|
-
* read-back retries can never confirm the assigned id when the session's
|
|
6715
|
-
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
6716
|
-
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
6717
|
-
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
6718
|
-
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
6719
|
-
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
6720
|
-
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
6721
|
-
* after abandonment) starts a new streak rather than inheriting the old
|
|
6722
|
-
* session's count — same shape as `redrivePollFailures` above.
|
|
6723
|
-
*/
|
|
6724
|
-
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
6725
7305
|
/**
|
|
6726
7306
|
* Consecutive give-ups for a message whose OpenCode user row remains queued.
|
|
6727
7307
|
* The session id scopes the count: a fresh session is a new attempt after a
|
|
@@ -6739,8 +7319,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6739
7319
|
* short-circuits while it is present, so a null-id row is re-dispatched AT MOST
|
|
6740
7320
|
* ONCE per outstanding read-back. Cleared on a SUCCESSFUL dispatch+read-back (the
|
|
6741
7321
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
6742
|
-
* re-entry) OR on a failed/unresolved dispatch (the
|
|
6743
|
-
*
|
|
7322
|
+
* re-entry) OR on a failed/unresolved dispatch (the accepted prompt is handed to
|
|
7323
|
+
* the terminal-report fence and is never sent again).
|
|
6744
7324
|
*/
|
|
6745
7325
|
awaitingReadopt = /* @__PURE__ */ new Set();
|
|
6746
7326
|
/**
|
|
@@ -6859,10 +7439,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6859
7439
|
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
6860
7440
|
this.now = config.now ?? (() => Date.now());
|
|
6861
7441
|
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
6862
|
-
this.homeDir = config.homeDir ??
|
|
7442
|
+
this.homeDir = config.homeDir ?? homedir6();
|
|
6863
7443
|
this.maxActiveSessions = config.maxActiveSessions;
|
|
6864
7444
|
this.watcherStallMs = config.watcherStallMs ?? WATCHER_STALL_MS;
|
|
6865
7445
|
this.wedgeWarningIntervalMs = config.wedgeWarningIntervalMs ?? WEDGE_WARNING_INTERVAL_MS;
|
|
7446
|
+
this.sessionStoreHealth = config.sessionStoreHealth;
|
|
6866
7447
|
}
|
|
6867
7448
|
get isV2() {
|
|
6868
7449
|
return this.openCodeClient.version === "v2";
|
|
@@ -6878,7 +7459,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
6878
7459
|
return fetchSessionMessages(this.port, sessionId, this.openCodeClient);
|
|
6879
7460
|
}
|
|
6880
7461
|
async listSessions() {
|
|
6881
|
-
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient
|
|
7462
|
+
return this.isV2 ? listV2Sessions(this.openCodeClient) : listSessions(this.port, this.openCodeClient, {
|
|
7463
|
+
limit: DRIVER_SESSION_ENUMERATION_LIMIT
|
|
7464
|
+
});
|
|
6882
7465
|
}
|
|
6883
7466
|
async sessionExists(sessionId) {
|
|
6884
7467
|
return this.isV2 ? v2SessionExists(this.openCodeClient, sessionId) : sessionExists(this.port, sessionId, this.openCodeClient);
|
|
@@ -7055,6 +7638,49 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7055
7638
|
}
|
|
7056
7639
|
return ids;
|
|
7057
7640
|
}
|
|
7641
|
+
/** Find an accepted prompt's user row without adopting an older or concurrent turn. */
|
|
7642
|
+
findAwaitingCorrelationId(entry, sessionId, messages) {
|
|
7643
|
+
if (entry.sessionId !== sessionId || entry.knownUserIds === null) return null;
|
|
7644
|
+
const excludeIds = new Set(entry.knownUserIds);
|
|
7645
|
+
for (const inFlight of this.watchers.get(entry.sessionId)?.inFlight.values() ?? []) {
|
|
7646
|
+
excludeIds.add(inFlight.opencodeMessageId);
|
|
7647
|
+
}
|
|
7648
|
+
return findDispatchedUserMessageId(messages, entry.content, excludeIds);
|
|
7649
|
+
}
|
|
7650
|
+
/** Hand an exhausted accepted-but-untrackable prompt to the terminal-report fence. */
|
|
7651
|
+
async handoffUntrackableCorrelation(conversationId, messageId, sessionId, attempts, source) {
|
|
7652
|
+
this.awaitingCorrelation.delete(messageId);
|
|
7653
|
+
this.awaitingReadopt.delete(messageId);
|
|
7654
|
+
this.sessions.delete(conversationId);
|
|
7655
|
+
this.supersede(conversationId, sessionId);
|
|
7656
|
+
const dispatchDescription = source === "readopt" ? "re-dispatched message" : "message";
|
|
7657
|
+
const errorMessage3 = `OpenCode accepted this ${dispatchDescription} once, but its assigned id was never confirmed after ${attempts} attempts to match it up \u2014 the message was deliberately not sent again, so no duplicate turn was created; session ${sessionId.slice(0, 8)} was abandoned; a fresh one is used for further messages.`;
|
|
7658
|
+
this.untrackableAck.set(messageId, errorMessage3);
|
|
7659
|
+
this.untrackableAckSessionIds.set(messageId, null);
|
|
7660
|
+
this.log({
|
|
7661
|
+
level: "error",
|
|
7662
|
+
message: errorMessage3,
|
|
7663
|
+
conversation_id: conversationId,
|
|
7664
|
+
message_id: messageId
|
|
7665
|
+
});
|
|
7666
|
+
const reportError = await this.reportUntrackableAck(
|
|
7667
|
+
conversationId,
|
|
7668
|
+
messageId,
|
|
7669
|
+
null,
|
|
7670
|
+
errorMessage3
|
|
7671
|
+
);
|
|
7672
|
+
if (reportError !== null) {
|
|
7673
|
+
this.log({
|
|
7674
|
+
level: "warn",
|
|
7675
|
+
message: `markFailed PATCH for message ${messageId.slice(0, 8)} (conversation ${conversationId.slice(0, 8)}) failed: ${reportError.message}`,
|
|
7676
|
+
conversation_id: conversationId,
|
|
7677
|
+
message_id: messageId
|
|
7678
|
+
});
|
|
7679
|
+
}
|
|
7680
|
+
if (source === "readopt") {
|
|
7681
|
+
void this.postSignal(conversationId, messageId, "readopt_orphan_unsent");
|
|
7682
|
+
}
|
|
7683
|
+
}
|
|
7058
7684
|
/**
|
|
7059
7685
|
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
7060
7686
|
*
|
|
@@ -7234,6 +7860,56 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7234
7860
|
}
|
|
7235
7861
|
break;
|
|
7236
7862
|
}
|
|
7863
|
+
const correlation = this.awaitingCorrelation.get(message.id);
|
|
7864
|
+
if (correlation) {
|
|
7865
|
+
correlation.attempts += 1;
|
|
7866
|
+
let correlatedId = null;
|
|
7867
|
+
if (correlation.knownUserIds !== null && correlation.sessionId === sessionId) {
|
|
7868
|
+
const polled = await this.pollSessionMessagesForRedrive(
|
|
7869
|
+
conv,
|
|
7870
|
+
message,
|
|
7871
|
+
correlation.sessionId
|
|
7872
|
+
);
|
|
7873
|
+
if (polled.ok)
|
|
7874
|
+
correlatedId = this.findAwaitingCorrelationId(correlation, sessionId, polled.messages);
|
|
7875
|
+
} else if (correlation.knownUserIds === null) {
|
|
7876
|
+
this.log({
|
|
7877
|
+
level: "debug",
|
|
7878
|
+
message: `Message ${message.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
7879
|
+
conversation_id: conv.id,
|
|
7880
|
+
message_id: message.id
|
|
7881
|
+
});
|
|
7882
|
+
}
|
|
7883
|
+
if (correlatedId !== null) {
|
|
7884
|
+
this.awaitingCorrelation.delete(message.id);
|
|
7885
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7886
|
+
this.releasedOpencodeIds.delete(message.id);
|
|
7887
|
+
this.dispatched.add(message.id);
|
|
7888
|
+
this.registerInFlight(conv, correlation.sessionId, message, correlatedId);
|
|
7889
|
+
dispatched += 1;
|
|
7890
|
+
this.log({
|
|
7891
|
+
level: "info",
|
|
7892
|
+
message: `Matched accepted message ${message.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
7893
|
+
conversation_id: conv.id,
|
|
7894
|
+
message_id: message.id
|
|
7895
|
+
});
|
|
7896
|
+
void this.postSignal(conv.id, message.id, "ack_correlated", {
|
|
7897
|
+
opencode_message_id: correlatedId
|
|
7898
|
+
});
|
|
7899
|
+
continue;
|
|
7900
|
+
}
|
|
7901
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
7902
|
+
await this.handoffUntrackableCorrelation(
|
|
7903
|
+
conv.id,
|
|
7904
|
+
message.id,
|
|
7905
|
+
correlation.sessionId,
|
|
7906
|
+
correlation.attempts,
|
|
7907
|
+
"fresh"
|
|
7908
|
+
);
|
|
7909
|
+
break;
|
|
7910
|
+
}
|
|
7911
|
+
break;
|
|
7912
|
+
}
|
|
7237
7913
|
const effectiveOpencodeMessageId = message.opencode_message_id ?? this.releasedOpencodeIds.get(message.id)?.opencodeMessageId ?? null;
|
|
7238
7914
|
if (effectiveOpencodeMessageId) {
|
|
7239
7915
|
const outcome = await this.resolveRedrive(
|
|
@@ -7255,6 +7931,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7255
7931
|
model: message.opencode_model ?? void 0
|
|
7256
7932
|
};
|
|
7257
7933
|
let opencodeMessageId;
|
|
7934
|
+
let preDispatchUserIds = null;
|
|
7258
7935
|
try {
|
|
7259
7936
|
this.log({
|
|
7260
7937
|
level: "info",
|
|
@@ -7284,7 +7961,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7284
7961
|
message.content,
|
|
7285
7962
|
options,
|
|
7286
7963
|
sendAttachments,
|
|
7287
|
-
this.openCodeClient
|
|
7964
|
+
this.openCodeClient,
|
|
7965
|
+
{
|
|
7966
|
+
onKnownUserIds: (ids) => {
|
|
7967
|
+
preDispatchUserIds = ids;
|
|
7968
|
+
},
|
|
7969
|
+
onKnownUserIdsError: (error2) => {
|
|
7970
|
+
this.log({
|
|
7971
|
+
level: "warn",
|
|
7972
|
+
message: `Pre-dispatch snapshot callback failed for message ${message.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
7973
|
+
conversation_id: conv.id,
|
|
7974
|
+
message_id: message.id
|
|
7975
|
+
});
|
|
7976
|
+
}
|
|
7977
|
+
}
|
|
7288
7978
|
)
|
|
7289
7979
|
);
|
|
7290
7980
|
} catch (err) {
|
|
@@ -7292,6 +7982,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7292
7982
|
if (this.isV2 && err instanceof OpenCodeV2PromptAckError) {
|
|
7293
7983
|
const errorMessage4 = err instanceof Error ? err.message : String(err);
|
|
7294
7984
|
this.untrackableAck.set(message.id, errorMessage4);
|
|
7985
|
+
this.untrackableAckSessionIds.set(message.id, null);
|
|
7295
7986
|
this.log({
|
|
7296
7987
|
level: "error",
|
|
7297
7988
|
message: `V2 prompt dispatch for message ${message.id.slice(0, 8)} failed after a positive ack with no usable id: ${errorMessage4}`,
|
|
@@ -7367,39 +8058,21 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
7367
8058
|
if (this.isV2) {
|
|
7368
8059
|
throw new Error("V2 prompt dispatch completed without an acknowledged message id");
|
|
7369
8060
|
}
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
message_id: message.id
|
|
7377
|
-
});
|
|
7378
|
-
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7379
|
-
continue;
|
|
7380
|
-
}
|
|
7381
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7382
|
-
this.sessions.delete(conv.id);
|
|
7383
|
-
this.supersede(conv.id, sessionId);
|
|
7384
|
-
const errorMessage3 = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
8061
|
+
this.awaitingCorrelation.set(message.id, {
|
|
8062
|
+
sessionId,
|
|
8063
|
+
content: message.content,
|
|
8064
|
+
attempts: 0,
|
|
8065
|
+
knownUserIds: preDispatchUserIds
|
|
8066
|
+
});
|
|
7385
8067
|
this.log({
|
|
7386
|
-
level: "
|
|
7387
|
-
message:
|
|
8068
|
+
level: "warn",
|
|
8069
|
+
message: `OpenCode accepted message ${message.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
7388
8070
|
conversation_id: conv.id,
|
|
7389
8071
|
message_id: message.id
|
|
7390
8072
|
});
|
|
7391
|
-
|
|
7392
|
-
this.log({
|
|
7393
|
-
level: "warn",
|
|
7394
|
-
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
7395
|
-
conversation_id: conv.id,
|
|
7396
|
-
message_id: message.id
|
|
7397
|
-
});
|
|
7398
|
-
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
7399
|
-
});
|
|
8073
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
7400
8074
|
break;
|
|
7401
8075
|
}
|
|
7402
|
-
this.unconfirmedDispatchFailures.delete(message.id);
|
|
7403
8076
|
this.dispatchNotStartedSignalled.delete(message.id);
|
|
7404
8077
|
this.releasedOpencodeIds.delete(message.id);
|
|
7405
8078
|
this.dispatched.add(message.id);
|
|
@@ -8004,23 +8677,6 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8004
8677
|
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
8005
8678
|
return 1;
|
|
8006
8679
|
}
|
|
8007
|
-
/**
|
|
8008
|
-
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
8009
|
-
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
8010
|
-
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
8011
|
-
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
8012
|
-
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
8013
|
-
* count, since a new session is a genuinely different attempt.
|
|
8014
|
-
*/
|
|
8015
|
-
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
8016
|
-
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
8017
|
-
if (existing && existing.sessionId === sessionId) {
|
|
8018
|
-
existing.count += 1;
|
|
8019
|
-
return existing.count;
|
|
8020
|
-
}
|
|
8021
|
-
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
8022
|
-
return 1;
|
|
8023
|
-
}
|
|
8024
8680
|
/** Record one queued give-up toward the never-started bound. */
|
|
8025
8681
|
recordNeverStartedAttempt(messageId, sessionId) {
|
|
8026
8682
|
const existing = this.neverStartedAttempts.get(messageId);
|
|
@@ -8332,8 +8988,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8332
8988
|
b2LastDescendantCheckMs: 0,
|
|
8333
8989
|
b2RootOngoingHeldLogged: false,
|
|
8334
8990
|
b2AbandonedSignalled: false,
|
|
8991
|
+
b2AbandonedWithoutAnswer: false,
|
|
8335
8992
|
ambiguousPinnedSinceMs: 0,
|
|
8336
8993
|
ambiguousResolved: false,
|
|
8994
|
+
sessionStoreFailurePending: false,
|
|
8337
8995
|
neverStartedResolved: false
|
|
8338
8996
|
});
|
|
8339
8997
|
const buffered = this.bufferedSessionErrors.get(sessionId);
|
|
@@ -8450,8 +9108,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8450
9108
|
b2LastDescendantCheckMs: 0,
|
|
8451
9109
|
b2RootOngoingHeldLogged: false,
|
|
8452
9110
|
b2AbandonedSignalled: false,
|
|
9111
|
+
b2AbandonedWithoutAnswer: false,
|
|
8453
9112
|
ambiguousPinnedSinceMs: 0,
|
|
8454
9113
|
ambiguousResolved: false,
|
|
9114
|
+
sessionStoreFailurePending: false,
|
|
8455
9115
|
neverStartedResolved: false
|
|
8456
9116
|
});
|
|
8457
9117
|
}
|
|
@@ -8865,6 +9525,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
8865
9525
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8866
9526
|
this.neverStartedAttempts.delete(id);
|
|
8867
9527
|
}
|
|
9528
|
+
if (inFlight.sessionStoreFailurePending) {
|
|
9529
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9530
|
+
return;
|
|
9531
|
+
}
|
|
8868
9532
|
if (inFlight.neverStartedResolved) {
|
|
8869
9533
|
if (state === "running" || state === "done" || state === "failed") {
|
|
8870
9534
|
inFlight.neverStartedResolved = false;
|
|
@@ -9024,10 +9688,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9024
9688
|
inFlight.b2LastDescendantCheckMs = 0;
|
|
9025
9689
|
inFlight.b2RootOngoingHeldLogged = false;
|
|
9026
9690
|
inFlight.b2AbandonedSignalled = false;
|
|
9691
|
+
inFlight.b2AbandonedWithoutAnswer = false;
|
|
9027
9692
|
}
|
|
9028
9693
|
} else {
|
|
9029
9694
|
if (inFlight.b2AbandonedSignalled) {
|
|
9030
|
-
|
|
9695
|
+
if (inFlight.b2AbandonedWithoutAnswer) {
|
|
9696
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9697
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9698
|
+
sessionId,
|
|
9699
|
+
watcher,
|
|
9700
|
+
inFlight,
|
|
9701
|
+
messages,
|
|
9702
|
+
reply?.info?.finish ?? reply?.finish
|
|
9703
|
+
);
|
|
9704
|
+
} else {
|
|
9705
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
9706
|
+
}
|
|
9031
9707
|
return;
|
|
9032
9708
|
}
|
|
9033
9709
|
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
@@ -9044,17 +9720,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9044
9720
|
descendantOngoing,
|
|
9045
9721
|
rootOngoing
|
|
9046
9722
|
})) {
|
|
9723
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9724
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9725
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9726
|
+
watched_for_ms: pinnedForMs,
|
|
9727
|
+
run_state: "failed",
|
|
9728
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9729
|
+
});
|
|
9730
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9731
|
+
return;
|
|
9732
|
+
}
|
|
9047
9733
|
inFlight.b2AbandonedSignalled = true;
|
|
9734
|
+
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9735
|
+
const finish = reply?.info?.finish ?? reply?.finish;
|
|
9736
|
+
if (extractReplyTextForMessage(messages, inFlight.opencodeMessageId) === null) {
|
|
9737
|
+
inFlight.b2AbandonedWithoutAnswer = true;
|
|
9738
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9739
|
+
watched_for_ms: pinnedForMs,
|
|
9740
|
+
run_state: "failed",
|
|
9741
|
+
finish,
|
|
9742
|
+
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9743
|
+
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9744
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9745
|
+
});
|
|
9746
|
+
await this.settleMessageAbandonedWithoutAnswer(
|
|
9747
|
+
sessionId,
|
|
9748
|
+
watcher,
|
|
9749
|
+
inFlight,
|
|
9750
|
+
messages,
|
|
9751
|
+
finish
|
|
9752
|
+
);
|
|
9753
|
+
return;
|
|
9754
|
+
}
|
|
9048
9755
|
this.log({
|
|
9049
9756
|
level: "warn",
|
|
9050
9757
|
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
9051
9758
|
conversation_id: conv.id,
|
|
9052
9759
|
message_id: id
|
|
9053
9760
|
});
|
|
9054
|
-
const reply = findLastAssistantReplyFor(messages, inFlight.opencodeMessageId);
|
|
9055
9761
|
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
9056
9762
|
watched_for_ms: pinnedForMs,
|
|
9057
|
-
finish
|
|
9763
|
+
finish,
|
|
9058
9764
|
...rootOngoing != null ? { root_ongoing: rootOngoing } : {},
|
|
9059
9765
|
...descendantOngoing != null ? { descendant_ongoing: descendantOngoing } : {},
|
|
9060
9766
|
opencode_message_id: inFlight.opencodeMessageId
|
|
@@ -9102,6 +9808,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9102
9808
|
maxPinnedMs: AMBIGUOUS_FINISH_MAX_PINNED_MS,
|
|
9103
9809
|
sessionOngoing: ongoing
|
|
9104
9810
|
})) {
|
|
9811
|
+
if (this.sessionStoreHealth?.isUnwritable()) {
|
|
9812
|
+
inFlight.sessionStoreFailurePending = true;
|
|
9813
|
+
void this.postSignal(conv.id, id, "ambiguous_finish_resolved", {
|
|
9814
|
+
watched_for_ms: pinnedForMs,
|
|
9815
|
+
run_state: "failed",
|
|
9816
|
+
opencode_message_id: inFlight.opencodeMessageId
|
|
9817
|
+
});
|
|
9818
|
+
await this.settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight);
|
|
9819
|
+
return;
|
|
9820
|
+
}
|
|
9105
9821
|
inFlight.ambiguousResolved = true;
|
|
9106
9822
|
this.log({
|
|
9107
9823
|
level: "warn",
|
|
@@ -9310,20 +10026,88 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9310
10026
|
}
|
|
9311
10027
|
this.log({
|
|
9312
10028
|
level: "warn",
|
|
9313
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10029
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10030
|
+
conversation_id: conv.id,
|
|
10031
|
+
message_id: inFlight.evidentMessageId
|
|
10032
|
+
});
|
|
10033
|
+
return;
|
|
10034
|
+
}
|
|
10035
|
+
inFlight.done = true;
|
|
10036
|
+
await this.reportSubagentAuthFailures(
|
|
10037
|
+
watcher.conv.id,
|
|
10038
|
+
inFlight.opencodeMessageId,
|
|
10039
|
+
inFlight.evidentMessageId,
|
|
10040
|
+
messages
|
|
10041
|
+
);
|
|
10042
|
+
}
|
|
10043
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10044
|
+
}
|
|
10045
|
+
/** Report a confirmed b2 abandonment that produced no answer text. */
|
|
10046
|
+
async settleMessageAbandonedWithoutAnswer(sessionId, watcher, inFlight, messages, finish) {
|
|
10047
|
+
const conv = watcher.conv;
|
|
10048
|
+
const error2 = `OpenCode ended this turn after a tool call (finish="${finish ?? "(unknown)"}") without writing an answer, and confirmed it is no longer working on this conversation. Evident is reporting the run as failed so the failure and its reason are visible in failed runs.`;
|
|
10049
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10050
|
+
this.log({
|
|
10051
|
+
level: "warn",
|
|
10052
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was abandoned without an answer \u2014 ${error2}`,
|
|
10053
|
+
conversation_id: conv.id,
|
|
10054
|
+
message_id: inFlight.evidentMessageId
|
|
10055
|
+
});
|
|
10056
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
10057
|
+
const usageAgentName = this.usageAgentName(messages ?? [], inFlight.opencodeMessageId);
|
|
10058
|
+
const subagentInvocations = await this.resolveSubagentInvocations(
|
|
10059
|
+
messages,
|
|
10060
|
+
inFlight.opencodeMessageId,
|
|
10061
|
+
inFlight.evidentMessageId
|
|
10062
|
+
);
|
|
10063
|
+
try {
|
|
10064
|
+
await this.markFailed(
|
|
10065
|
+
conv.id,
|
|
10066
|
+
inFlight.evidentMessageId,
|
|
10067
|
+
sessionId,
|
|
10068
|
+
error2,
|
|
10069
|
+
usage,
|
|
10070
|
+
void 0,
|
|
10071
|
+
usageAgentName,
|
|
10072
|
+
subagentInvocations
|
|
10073
|
+
);
|
|
10074
|
+
} catch (err) {
|
|
10075
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10076
|
+
if (err instanceof ChannelTerminalError) {
|
|
10077
|
+
this.log({
|
|
10078
|
+
level: "warn",
|
|
10079
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 leaving it for the cron safety net: ${err.message}`,
|
|
9314
10080
|
conversation_id: conv.id,
|
|
9315
10081
|
message_id: inFlight.evidentMessageId
|
|
9316
10082
|
});
|
|
10083
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9317
10084
|
return;
|
|
9318
10085
|
}
|
|
9319
|
-
inFlight.
|
|
9320
|
-
|
|
9321
|
-
|
|
9322
|
-
|
|
9323
|
-
|
|
9324
|
-
|
|
9325
|
-
|
|
10086
|
+
if (this.now() >= inFlight.deadline) {
|
|
10087
|
+
this.log({
|
|
10088
|
+
level: "warn",
|
|
10089
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 leaving it for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
10090
|
+
conversation_id: conv.id,
|
|
10091
|
+
message_id: inFlight.evidentMessageId
|
|
10092
|
+
});
|
|
10093
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10094
|
+
return;
|
|
10095
|
+
}
|
|
10096
|
+
this.log({
|
|
10097
|
+
level: "warn",
|
|
10098
|
+
message: `Failed to report abandoned no-answer message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10099
|
+
conversation_id: conv.id,
|
|
10100
|
+
message_id: inFlight.evidentMessageId
|
|
10101
|
+
});
|
|
10102
|
+
return;
|
|
9326
10103
|
}
|
|
10104
|
+
inFlight.done = true;
|
|
10105
|
+
await this.reportSubagentAuthFailures(
|
|
10106
|
+
watcher.conv.id,
|
|
10107
|
+
inFlight.opencodeMessageId,
|
|
10108
|
+
inFlight.evidentMessageId,
|
|
10109
|
+
messages
|
|
10110
|
+
);
|
|
9327
10111
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9328
10112
|
}
|
|
9329
10113
|
/**
|
|
@@ -9381,6 +10165,66 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9381
10165
|
this.neverStartedAttempts.delete(inFlight.evidentMessageId);
|
|
9382
10166
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
9383
10167
|
}
|
|
10168
|
+
/**
|
|
10169
|
+
* Report a turn as failed when the session database was observed unwritable.
|
|
10170
|
+
* The queue row is retained as failed, while the OpenCode copy stays fenced so
|
|
10171
|
+
* a later drain cannot blindly prompt the same turn again.
|
|
10172
|
+
*/
|
|
10173
|
+
async settleMessageSessionStoreUnwritable(sessionId, watcher, inFlight) {
|
|
10174
|
+
const conv = watcher.conv;
|
|
10175
|
+
const error2 = "The runner session database could not be written, so this turn did not finish. Nothing was lost from the queue. Clear disk space and retry.";
|
|
10176
|
+
this.log({
|
|
10177
|
+
level: "error",
|
|
10178
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} was not resolved as done: ${error2}`,
|
|
10179
|
+
conversation_id: conv.id,
|
|
10180
|
+
message_id: inFlight.evidentMessageId
|
|
10181
|
+
});
|
|
10182
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
10183
|
+
try {
|
|
10184
|
+
await this.markFailed(conv.id, inFlight.evidentMessageId, sessionId, error2);
|
|
10185
|
+
} catch (err) {
|
|
10186
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
10187
|
+
if (err instanceof ChannelTerminalError) {
|
|
10188
|
+
this.log({
|
|
10189
|
+
level: "warn",
|
|
10190
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (terminal HTTP ${err.status}) \u2014 keeping its OpenCode copy fenced: ${err.message}`,
|
|
10191
|
+
conversation_id: conv.id,
|
|
10192
|
+
message_id: inFlight.evidentMessageId
|
|
10193
|
+
});
|
|
10194
|
+
this.recordReleasedOpencodeId(
|
|
10195
|
+
inFlight.evidentMessageId,
|
|
10196
|
+
sessionId,
|
|
10197
|
+
inFlight.opencodeMessageId
|
|
10198
|
+
);
|
|
10199
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10200
|
+
return;
|
|
10201
|
+
}
|
|
10202
|
+
if (this.now() >= inFlight.deadline) {
|
|
10203
|
+
this.log({
|
|
10204
|
+
level: "warn",
|
|
10205
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed within the delivery window \u2014 keeping its OpenCode copy fenced: ${err instanceof Error ? err.message : String(err)}`,
|
|
10206
|
+
conversation_id: conv.id,
|
|
10207
|
+
message_id: inFlight.evidentMessageId
|
|
10208
|
+
});
|
|
10209
|
+
this.recordReleasedOpencodeId(
|
|
10210
|
+
inFlight.evidentMessageId,
|
|
10211
|
+
sessionId,
|
|
10212
|
+
inFlight.opencodeMessageId
|
|
10213
|
+
);
|
|
10214
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10215
|
+
return;
|
|
10216
|
+
}
|
|
10217
|
+
this.log({
|
|
10218
|
+
level: "warn",
|
|
10219
|
+
message: `Failed to report session-store failure for message ${inFlight.evidentMessageId.slice(0, 8)} as failed (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
10220
|
+
conversation_id: conv.id,
|
|
10221
|
+
message_id: inFlight.evidentMessageId
|
|
10222
|
+
});
|
|
10223
|
+
return;
|
|
10224
|
+
}
|
|
10225
|
+
inFlight.done = true;
|
|
10226
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
10227
|
+
}
|
|
9384
10228
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
9385
10229
|
/**
|
|
9386
10230
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -9419,11 +10263,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9419
10263
|
}
|
|
9420
10264
|
}
|
|
9421
10265
|
}
|
|
9422
|
-
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0) {
|
|
10266
|
+
if (this.untrackableAck.size > 0 || this.neverStartedAttempts.size > 0 || this.awaitingCorrelation.size > 0) {
|
|
9423
10267
|
const stillProcessing = new Set(rows.map((r) => r.id));
|
|
10268
|
+
for (const id of this.awaitingCorrelation.keys()) {
|
|
10269
|
+
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
10270
|
+
this.awaitingCorrelation.delete(id);
|
|
10271
|
+
this.log({
|
|
10272
|
+
level: "debug",
|
|
10273
|
+
message: `Re-adopt: message ${id.slice(0, 8)} left processing and pending \u2014 cleared awaiting-correlation fence`,
|
|
10274
|
+
message_id: id
|
|
10275
|
+
});
|
|
10276
|
+
}
|
|
10277
|
+
}
|
|
9424
10278
|
for (const id of [...this.untrackableAck.keys()]) {
|
|
9425
10279
|
if (!stillProcessing.has(id) && !this.pendingMessageIds.has(id)) {
|
|
9426
10280
|
this.untrackableAck.delete(id);
|
|
10281
|
+
this.untrackableAckSessionIds.delete(id);
|
|
9427
10282
|
this.untrackableAckSignalled.delete(id);
|
|
9428
10283
|
this.untrackableAckWarned.delete(id);
|
|
9429
10284
|
this.log({
|
|
@@ -9463,7 +10318,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9463
10318
|
bySession.set(row.opencode_session_id, list);
|
|
9464
10319
|
}
|
|
9465
10320
|
for (const [sessionId, sessionRows] of bySession) {
|
|
9466
|
-
let messages;
|
|
10321
|
+
let messages = null;
|
|
9467
10322
|
try {
|
|
9468
10323
|
if (this.isV2) {
|
|
9469
10324
|
const snapshot = await this.getSessionMessages(sessionId);
|
|
@@ -9472,9 +10327,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9472
10327
|
level: "warn",
|
|
9473
10328
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned an unreadable message snapshot \u2014 skipping this session this tick`
|
|
9474
10329
|
});
|
|
9475
|
-
|
|
10330
|
+
} else {
|
|
10331
|
+
messages = snapshot;
|
|
9476
10332
|
}
|
|
9477
|
-
messages = snapshot;
|
|
9478
10333
|
} else {
|
|
9479
10334
|
const polled = await pollSessionMessagesForRedrive(
|
|
9480
10335
|
this.port,
|
|
@@ -9487,15 +10342,22 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9487
10342
|
level: "warn",
|
|
9488
10343
|
message: `Re-adopt: polling session ${sessionId.slice(0, 8)} returned ${polled.malformed ? "a non-array message body" : `HTTP ${polled.status ?? "a network error"}${normalized ? `: ${normalized}` : ""}`} \u2014 skipping this session this tick`
|
|
9489
10344
|
});
|
|
9490
|
-
|
|
10345
|
+
} else {
|
|
10346
|
+
messages = polled.messages;
|
|
9491
10347
|
}
|
|
9492
|
-
messages = polled.messages;
|
|
9493
10348
|
}
|
|
9494
10349
|
} catch (err) {
|
|
9495
10350
|
this.log({
|
|
9496
10351
|
level: "warn",
|
|
9497
10352
|
message: `Re-adopt: failed to poll session ${sessionId.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`
|
|
9498
10353
|
});
|
|
10354
|
+
}
|
|
10355
|
+
if (messages === null) {
|
|
10356
|
+
for (const row of sessionRows) {
|
|
10357
|
+
if (this.awaitingCorrelation.has(row.id)) {
|
|
10358
|
+
await this.advanceReadoptCorrelation(sessionId, row, null);
|
|
10359
|
+
}
|
|
10360
|
+
}
|
|
9499
10361
|
continue;
|
|
9500
10362
|
}
|
|
9501
10363
|
const anyUntracked = sessionRows.some((row) => !this.isTracked(sessionId, row.id));
|
|
@@ -9505,6 +10367,59 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9505
10367
|
}
|
|
9506
10368
|
}
|
|
9507
10369
|
}
|
|
10370
|
+
/** Advance a re-adopted accepted prompt through the shared correlation fence. */
|
|
10371
|
+
async advanceReadoptCorrelation(sessionId, row, messages) {
|
|
10372
|
+
const correlation = this.awaitingCorrelation.get(row.id);
|
|
10373
|
+
if (!correlation) return false;
|
|
10374
|
+
correlation.attempts += 1;
|
|
10375
|
+
const correlatedId = messages === null ? null : this.findAwaitingCorrelationId(correlation, sessionId, messages);
|
|
10376
|
+
if (correlatedId !== null) {
|
|
10377
|
+
this.awaitingCorrelation.delete(row.id);
|
|
10378
|
+
const conv = this.convForRow(sessionId, row);
|
|
10379
|
+
const message = this.queuedMessageForRow(row);
|
|
10380
|
+
const processedAtMs = this.processedAtMs(row);
|
|
10381
|
+
this.registerReadopted(conv, sessionId, message, correlatedId, processedAtMs, processedAtMs);
|
|
10382
|
+
this.dispatched.add(row.id);
|
|
10383
|
+
this.readopted.add(row.id);
|
|
10384
|
+
this.ensureWatcherRunning(sessionId);
|
|
10385
|
+
this.log({
|
|
10386
|
+
level: "info",
|
|
10387
|
+
message: `Re-adopt: matched accepted message ${row.id.slice(0, 8)} to OpenCode turn ${correlatedId.slice(0, 8)} without sending it again`,
|
|
10388
|
+
conversation_id: row.conversation_id,
|
|
10389
|
+
message_id: row.id
|
|
10390
|
+
});
|
|
10391
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_redispatched");
|
|
10392
|
+
void this.postSignal(row.conversation_id, row.id, "ack_correlated", {
|
|
10393
|
+
opencode_message_id: correlatedId
|
|
10394
|
+
});
|
|
10395
|
+
return true;
|
|
10396
|
+
}
|
|
10397
|
+
if (correlation.knownUserIds === null) {
|
|
10398
|
+
this.log({
|
|
10399
|
+
level: "debug",
|
|
10400
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} cannot be correlated because its pre-dispatch session list was unreadable`,
|
|
10401
|
+
conversation_id: row.conversation_id,
|
|
10402
|
+
message_id: row.id
|
|
10403
|
+
});
|
|
10404
|
+
} else if (correlation.sessionId !== sessionId) {
|
|
10405
|
+
this.log({
|
|
10406
|
+
level: "warn",
|
|
10407
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} moved from original session ${correlation.sessionId.slice(0, 8)} to ${sessionId.slice(0, 8)} while awaiting correlation \u2014 refusing to adopt a turn from the rebound session`,
|
|
10408
|
+
conversation_id: row.conversation_id,
|
|
10409
|
+
message_id: row.id
|
|
10410
|
+
});
|
|
10411
|
+
}
|
|
10412
|
+
if (correlation.attempts >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
10413
|
+
await this.handoffUntrackableCorrelation(
|
|
10414
|
+
row.conversation_id,
|
|
10415
|
+
row.id,
|
|
10416
|
+
correlation.sessionId,
|
|
10417
|
+
correlation.attempts,
|
|
10418
|
+
"readopt"
|
|
10419
|
+
);
|
|
10420
|
+
}
|
|
10421
|
+
return true;
|
|
10422
|
+
}
|
|
9508
10423
|
/**
|
|
9509
10424
|
* Re-adopt ONE `processing` row against the tick's session message snapshot
|
|
9510
10425
|
* (ADR-0046 Decision §1/§2). Idempotent: skips a row already being driven.
|
|
@@ -9512,8 +10427,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9512
10427
|
* Branches on `messageRunState(messages, row.opencode_message_id)` — the
|
|
9513
10428
|
* opencode-assigned user-message id persisted on the first `processing` PATCH
|
|
9514
10429
|
* (#218). A row with a NULL stored id (dispatched but the read-back never landed
|
|
9515
|
-
* before the restart)
|
|
9516
|
-
*
|
|
10430
|
+
* before the restart) is first offered to an awaiting-correlation fence recorded
|
|
10431
|
+
* by this process; without that fence, it has no id to correlate after restart
|
|
10432
|
+
* and is treated as an orphan and re-dispatched (at most once, see
|
|
10433
|
+
* `forceReadoptRun`):
|
|
9517
10434
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
9518
10435
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
9519
10436
|
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
@@ -9522,7 +10439,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9522
10439
|
* re-dispatched instead (issue #1310, see the branch below);
|
|
9523
10440
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
9524
10441
|
* tracking the stored id so the reply correlates by it;
|
|
9525
|
-
* - `unknown`/null id →
|
|
10442
|
+
* - `unknown`/null id → correlate through an awaiting fence when present;
|
|
10443
|
+
* otherwise re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
9526
10444
|
*
|
|
9527
10445
|
* Only `ChannelAuthError` propagates.
|
|
9528
10446
|
*/
|
|
@@ -9536,6 +10454,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9536
10454
|
});
|
|
9537
10455
|
return;
|
|
9538
10456
|
}
|
|
10457
|
+
if (await this.advanceReadoptCorrelation(sessionId, row, messages)) return;
|
|
9539
10458
|
const ocId = row.opencode_message_id;
|
|
9540
10459
|
const state = messageRunState(messages, ocId ?? "");
|
|
9541
10460
|
if (state === "done") {
|
|
@@ -9602,12 +10521,48 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9602
10521
|
await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
|
|
9603
10522
|
this.dontRedispatch.delete(row.id);
|
|
9604
10523
|
this.untrackableAck.delete(row.id);
|
|
10524
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9605
10525
|
this.untrackableAckSignalled.delete(row.id);
|
|
9606
10526
|
this.untrackableAckWarned.delete(row.id);
|
|
9607
10527
|
void this.postSignal(row.conversation_id, row.id, "readopt_failed");
|
|
9608
10528
|
return;
|
|
9609
10529
|
}
|
|
9610
|
-
if (this.
|
|
10530
|
+
if (this.untrackableAck.has(row.id)) {
|
|
10531
|
+
if (this.untrackableAckSignalled.has(row.id)) {
|
|
10532
|
+
const firstFenceWarning = !this.untrackableAckWarned.has(row.id);
|
|
10533
|
+
this.untrackableAckWarned.add(row.id);
|
|
10534
|
+
this.log({
|
|
10535
|
+
level: firstFenceWarning ? "warn" : "debug",
|
|
10536
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} is fenced after an untrackable OpenCode turn \u2014 skipping re-dispatch and retrying the terminal report`,
|
|
10537
|
+
conversation_id: row.conversation_id,
|
|
10538
|
+
message_id: row.id
|
|
10539
|
+
});
|
|
10540
|
+
const errorMessage3 = this.untrackableAck.get(row.id);
|
|
10541
|
+
const reportError = await this.reportUntrackableAck(
|
|
10542
|
+
row.conversation_id,
|
|
10543
|
+
row.id,
|
|
10544
|
+
this.untrackableAckSessionIds.get(row.id) ?? null,
|
|
10545
|
+
errorMessage3
|
|
10546
|
+
);
|
|
10547
|
+
if (reportError !== null) {
|
|
10548
|
+
this.log({
|
|
10549
|
+
level: "warn",
|
|
10550
|
+
message: `Re-adopt: failed to report untrackable message ${row.id.slice(0, 8)} (will retry next drain): ${reportError.message}`,
|
|
10551
|
+
conversation_id: row.conversation_id,
|
|
10552
|
+
message_id: row.id
|
|
10553
|
+
});
|
|
10554
|
+
}
|
|
10555
|
+
} else {
|
|
10556
|
+
this.log({
|
|
10557
|
+
level: "debug",
|
|
10558
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
10559
|
+
conversation_id: row.conversation_id,
|
|
10560
|
+
message_id: row.id
|
|
10561
|
+
});
|
|
10562
|
+
}
|
|
10563
|
+
return;
|
|
10564
|
+
}
|
|
10565
|
+
if (this.dontRedispatch.has(row.id)) {
|
|
9611
10566
|
this.log({
|
|
9612
10567
|
level: "debug",
|
|
9613
10568
|
message: `Re-adopt: message ${row.id.slice(0, 8)} already gave up \u2014 left to the cron; skipping until it leaves processing`,
|
|
@@ -9785,6 +10740,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9785
10740
|
}
|
|
9786
10741
|
this.dontRedispatch.delete(row.id);
|
|
9787
10742
|
this.untrackableAck.delete(row.id);
|
|
10743
|
+
this.untrackableAckSessionIds.delete(row.id);
|
|
9788
10744
|
this.untrackableAckSignalled.delete(row.id);
|
|
9789
10745
|
this.untrackableAckWarned.delete(row.id);
|
|
9790
10746
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
@@ -9804,8 +10760,9 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9804
10760
|
* latch makes a null-id row re-dispatched AT MOST ONCE per outstanding read-back:
|
|
9805
10761
|
* short-circuit while the row is latched; clear it on a successful dispatch (the
|
|
9806
10762
|
* row is then tracked in `dispatched`, so `readoptOne`'s early skip prevents
|
|
9807
|
-
* re-entry)
|
|
9808
|
-
*
|
|
10763
|
+
* re-entry) or when the dispatch itself failed before acceptance. An accepted V1
|
|
10764
|
+
* dispatch whose id is not readable enters the bounded correlation fence instead
|
|
10765
|
+
* of retrying; exhaustion hands it to `untrackableAck`.
|
|
9809
10766
|
*
|
|
9810
10767
|
* `evidentMessageId = row.id` addresses the SERVER row. Deadline anchored to
|
|
9811
10768
|
* `processed_at` (Invariant 1).
|
|
@@ -9868,6 +10825,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9868
10825
|
);
|
|
9869
10826
|
}
|
|
9870
10827
|
let ocId;
|
|
10828
|
+
let preDispatchUserIds = null;
|
|
9871
10829
|
try {
|
|
9872
10830
|
ocId = this.isV2 ? await sendV2Prompt(this.openCodeClient, sessionId, row.content) : await this.dispatchLocked(
|
|
9873
10831
|
sessionId,
|
|
@@ -9877,7 +10835,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9877
10835
|
row.content,
|
|
9878
10836
|
options,
|
|
9879
10837
|
sendAttachments,
|
|
9880
|
-
this.openCodeClient
|
|
10838
|
+
this.openCodeClient,
|
|
10839
|
+
{
|
|
10840
|
+
onKnownUserIds: (ids) => {
|
|
10841
|
+
preDispatchUserIds = ids;
|
|
10842
|
+
},
|
|
10843
|
+
onKnownUserIdsError: (error2) => {
|
|
10844
|
+
this.log({
|
|
10845
|
+
level: "warn",
|
|
10846
|
+
message: `Pre-dispatch snapshot callback failed for message ${row.id.slice(0, 8)}: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
10847
|
+
conversation_id: row.conversation_id,
|
|
10848
|
+
message_id: row.id
|
|
10849
|
+
});
|
|
10850
|
+
}
|
|
10851
|
+
}
|
|
9881
10852
|
)
|
|
9882
10853
|
);
|
|
9883
10854
|
} catch (err) {
|
|
@@ -9910,6 +10881,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9910
10881
|
}
|
|
9911
10882
|
} else {
|
|
9912
10883
|
this.untrackableAck.set(row.id, errorMessage3);
|
|
10884
|
+
this.untrackableAckSessionIds.set(row.id, sessionId);
|
|
9913
10885
|
}
|
|
9914
10886
|
this.log({
|
|
9915
10887
|
level: "error",
|
|
@@ -9958,39 +10930,20 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
9958
10930
|
}
|
|
9959
10931
|
if (ocId === null) {
|
|
9960
10932
|
this.awaitingReadopt.delete(row.id);
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
this.log({
|
|
9968
|
-
level: "error",
|
|
9969
|
-
message: errorMessage3,
|
|
9970
|
-
conversation_id: row.conversation_id,
|
|
9971
|
-
message_id: row.id
|
|
9972
|
-
});
|
|
9973
|
-
await this.markFailed(row.conversation_id, row.id, null, errorMessage3).catch((markErr) => {
|
|
9974
|
-
this.log({
|
|
9975
|
-
level: "warn",
|
|
9976
|
-
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
9977
|
-
conversation_id: row.conversation_id,
|
|
9978
|
-
message_id: row.id
|
|
9979
|
-
});
|
|
9980
|
-
});
|
|
9981
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9982
|
-
return;
|
|
9983
|
-
}
|
|
10933
|
+
this.awaitingCorrelation.set(row.id, {
|
|
10934
|
+
sessionId,
|
|
10935
|
+
content: row.content,
|
|
10936
|
+
attempts: 0,
|
|
10937
|
+
knownUserIds: preDispatchUserIds
|
|
10938
|
+
});
|
|
9984
10939
|
this.log({
|
|
9985
10940
|
level: "warn",
|
|
9986
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)}
|
|
10941
|
+
message: `Re-adopt: OpenCode accepted message ${row.id.slice(0, 8)}, but its assigned id could not be read back \u2014 the message will not be sent again while the runner matches it to the turn already running`,
|
|
9987
10942
|
conversation_id: row.conversation_id,
|
|
9988
10943
|
message_id: row.id
|
|
9989
10944
|
});
|
|
9990
|
-
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
9991
10945
|
return;
|
|
9992
10946
|
}
|
|
9993
|
-
this.unconfirmedDispatchFailures.delete(row.id);
|
|
9994
10947
|
const processedAtMs = this.processedAtMs(row);
|
|
9995
10948
|
this.registerReadopted(
|
|
9996
10949
|
readoptConv,
|
|
@@ -12112,6 +13065,7 @@ function createCredentialSync({
|
|
|
12112
13065
|
|
|
12113
13066
|
// src/commands/run.ts
|
|
12114
13067
|
var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
13068
|
+
var MAX_DISCONNECT_REASON_LENGTH = 200;
|
|
12115
13069
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
12116
13070
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
12117
13071
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
@@ -12267,7 +13221,13 @@ function logActivity(state, entry) {
|
|
|
12267
13221
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
12268
13222
|
if (!meetsThreshold(state, level)) return;
|
|
12269
13223
|
forwardRunnerActivity(
|
|
12270
|
-
{
|
|
13224
|
+
{
|
|
13225
|
+
level,
|
|
13226
|
+
message: entry.message,
|
|
13227
|
+
error: entry.error,
|
|
13228
|
+
metadata: entry.metadata,
|
|
13229
|
+
...entry.source ? { source: entry.source } : {}
|
|
13230
|
+
},
|
|
12271
13231
|
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
12272
13232
|
);
|
|
12273
13233
|
const fullEntry = {
|
|
@@ -12289,7 +13249,7 @@ function logActivity(state, entry) {
|
|
|
12289
13249
|
}
|
|
12290
13250
|
function reportSessionDbRecovery(state) {
|
|
12291
13251
|
try {
|
|
12292
|
-
const report = drainSessionDbRecoveryReport({ homeDir:
|
|
13252
|
+
const report = drainSessionDbRecoveryReport({ homeDir: homedir7(), env: process.env });
|
|
12293
13253
|
for (const record of report.records) {
|
|
12294
13254
|
const activity = buildSessionDbRecoveryActivity(record);
|
|
12295
13255
|
if (!activity) throw new Error("could not map session-DB recovery record");
|
|
@@ -12324,7 +13284,9 @@ function displayStatus(state) {
|
|
|
12324
13284
|
const opencode = state.opencodeConnected ? chalk7.green(`opencode: :${state.port}`) : chalk7.red(`opencode: :${state.port} (down)`);
|
|
12325
13285
|
const messages = state.messageCount > 0 ? chalk7.dim(` \xB7 ${state.messageCount} processed`) : "";
|
|
12326
13286
|
const last = state.activityLog[state.activityLog.length - 1];
|
|
12327
|
-
const detail = last ? chalk7.dim(
|
|
13287
|
+
const detail = last ? chalk7.dim(
|
|
13288
|
+
` \xB7 ${last.timestamp.toISOString()} ${last.type === "error" ? last.error ?? "" : last.message ?? ""}`
|
|
13289
|
+
) : "";
|
|
12328
13290
|
const agent = state.agentName ?? state.agentId;
|
|
12329
13291
|
console.log(
|
|
12330
13292
|
`${chalk7.bold("Evident")} ${chalk7.dim(agent)} ${tunnel} ${opencode}${messages}${detail}`
|
|
@@ -12343,7 +13305,7 @@ function reportOpenCodeV2CredentialImport(state, result) {
|
|
|
12343
13305
|
logActivity(state, {
|
|
12344
13306
|
type: "info",
|
|
12345
13307
|
level: "warn",
|
|
12346
|
-
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them.
|
|
13308
|
+
message: `OpenCode V2 saved login(s) for ${integrationIds} on this machine, but OpenCode V2 will not use them. Connect the provider again from the runner's page in Evident; no shell on this machine is needed.`
|
|
12347
13309
|
});
|
|
12348
13310
|
}
|
|
12349
13311
|
if (result.imported.length === 0 && result.unsupported.length === 0 && result.failed.length === 0) {
|
|
@@ -12368,7 +13330,7 @@ async function importOpenCodeV2CredentialsForState(state) {
|
|
|
12368
13330
|
}
|
|
12369
13331
|
const result = await importOpenCodeV2Credentials({
|
|
12370
13332
|
request: client.request,
|
|
12371
|
-
homeDir:
|
|
13333
|
+
homeDir: homedir7(),
|
|
12372
13334
|
directory,
|
|
12373
13335
|
guard: state.opencodeV2CredentialImportGuard,
|
|
12374
13336
|
log: (message, level = "info") => logActivity(state, { type: "info", level, message })
|
|
@@ -12576,9 +13538,9 @@ async function driveChannels(state, driver) {
|
|
|
12576
13538
|
}
|
|
12577
13539
|
}
|
|
12578
13540
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
12579
|
-
var
|
|
13541
|
+
var SESSION_CLEANUP_MAX_DELETES_PER_SWEEP = 100;
|
|
12580
13542
|
function shouldWarnForReclaimSkip(reason) {
|
|
12581
|
-
if (reason !== "sqlite-unavailable") return false;
|
|
13543
|
+
if (reason !== "sqlite-unavailable" && reason !== "disk-exhaustion") return false;
|
|
12582
13544
|
const version2 = /^v(\d+)\.(\d+)\.(\d+)$/.exec(process.version);
|
|
12583
13545
|
if (!version2) return false;
|
|
12584
13546
|
const major = Number(version2[1]);
|
|
@@ -12587,7 +13549,14 @@ function shouldWarnForReclaimSkip(reason) {
|
|
|
12587
13549
|
return major > 22 || major === 22 && (minor > 13 || minor === 13 && patch >= 0);
|
|
12588
13550
|
}
|
|
12589
13551
|
function sessionDbPath() {
|
|
12590
|
-
return join11(
|
|
13552
|
+
return join11(homedir7(), ".local", "share", "opencode", "opencode.db");
|
|
13553
|
+
}
|
|
13554
|
+
function logSessionCleanupActivity(state, phase, entry) {
|
|
13555
|
+
logActivity(state, {
|
|
13556
|
+
...entry,
|
|
13557
|
+
source: "cli.session-cleanup",
|
|
13558
|
+
metadata: { ...entry.metadata, phase }
|
|
13559
|
+
});
|
|
12591
13560
|
}
|
|
12592
13561
|
function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBootMigrationCount) {
|
|
12593
13562
|
const record = {
|
|
@@ -12628,69 +13597,136 @@ function logSessionDbProvenanceMismatch(state, provenance, currentVersion, preBo
|
|
|
12628
13597
|
async function runSweep(state, driver, config) {
|
|
12629
13598
|
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
12630
13599
|
try {
|
|
12631
|
-
const
|
|
12632
|
-
|
|
13600
|
+
const enumeration = await listAllSessions(state.port, state.opencodeClient ?? void 0);
|
|
13601
|
+
const sessions = enumeration.sessions;
|
|
13602
|
+
const enumerationReason = sessions === null ? enumeration.reason : null;
|
|
13603
|
+
if (enumerationReason !== null) {
|
|
12633
13604
|
logActivity(state, {
|
|
12634
13605
|
type: "info",
|
|
12635
|
-
|
|
13606
|
+
level: "warn",
|
|
13607
|
+
message: `Session cleanup: could not enumerate sessions (${enumerationReason}); skipping deletes and continuing with space reclaim (${mode})`
|
|
12636
13608
|
});
|
|
12637
|
-
return;
|
|
12638
13609
|
}
|
|
12639
|
-
const toDelete = selectSessionsToDelete(
|
|
12640
|
-
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
12641
|
-
{
|
|
12642
|
-
maxAgeMs: config.maxAgeMs,
|
|
12643
|
-
maxCount: config.maxCount,
|
|
12644
|
-
nowMs: Date.now(),
|
|
12645
|
-
protectedIds: driver.protectedSessionIds()
|
|
12646
|
-
}
|
|
12647
|
-
);
|
|
12648
|
-
const protectedNow = driver.protectedSessionIds();
|
|
12649
13610
|
let deleted = 0;
|
|
12650
13611
|
let failed = 0;
|
|
13612
|
+
let failed4xx = 0;
|
|
13613
|
+
let failed5xx = 0;
|
|
13614
|
+
let failedOtherStatus = 0;
|
|
13615
|
+
let failedThrew = 0;
|
|
12651
13616
|
let skippedNewlyActive = 0;
|
|
12652
|
-
|
|
12653
|
-
|
|
12654
|
-
|
|
12655
|
-
|
|
12656
|
-
|
|
12657
|
-
|
|
12658
|
-
|
|
12659
|
-
|
|
13617
|
+
let remaining = 0;
|
|
13618
|
+
let protectedNow = driver.protectedSessionIds();
|
|
13619
|
+
if (sessions !== null) {
|
|
13620
|
+
const toDelete = selectSessionsToDelete(
|
|
13621
|
+
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
13622
|
+
{
|
|
13623
|
+
maxAgeMs: config.maxAgeMs,
|
|
13624
|
+
maxCount: config.maxCount,
|
|
13625
|
+
nowMs: Date.now(),
|
|
13626
|
+
protectedIds: protectedNow
|
|
13627
|
+
}
|
|
13628
|
+
);
|
|
13629
|
+
const deleteCandidates = toDelete.slice(0, SESSION_CLEANUP_MAX_DELETES_PER_SWEEP);
|
|
13630
|
+
remaining = toDelete.length - deleteCandidates.length;
|
|
13631
|
+
protectedNow = driver.protectedSessionIds();
|
|
13632
|
+
for (const id of deleteCandidates) {
|
|
13633
|
+
if (protectedNow.has(id)) {
|
|
13634
|
+
skippedNewlyActive++;
|
|
13635
|
+
logActivity(state, {
|
|
13636
|
+
type: "info",
|
|
13637
|
+
message: `Session cleanup: skipping ${id} \u2014 became active/bound after selection (${mode})`
|
|
13638
|
+
});
|
|
13639
|
+
continue;
|
|
13640
|
+
}
|
|
13641
|
+
const outcome = await deleteSession(state.port, id, state.opencodeClient ?? void 0);
|
|
13642
|
+
if (outcome.ok) {
|
|
13643
|
+
deleted++;
|
|
13644
|
+
} else if (outcome.status === "threw") {
|
|
13645
|
+
failed++;
|
|
13646
|
+
failedThrew++;
|
|
13647
|
+
} else {
|
|
13648
|
+
failed++;
|
|
13649
|
+
const statusClass = Math.floor(outcome.status / 100);
|
|
13650
|
+
if (statusClass === 4) failed4xx++;
|
|
13651
|
+
else if (statusClass === 5) failed5xx++;
|
|
13652
|
+
else failedOtherStatus++;
|
|
13653
|
+
}
|
|
12660
13654
|
}
|
|
12661
|
-
|
|
12662
|
-
|
|
13655
|
+
const failedNote = failed > 0 ? `, failed ${failed}` : "";
|
|
13656
|
+
const skippedNote = skippedNewlyActive > 0 ? `, skipped ${skippedNewlyActive} newly-active` : "";
|
|
13657
|
+
const remainingNote = remaining > 0 ? `, remaining ${remaining}` : "";
|
|
13658
|
+
logActivity(state, {
|
|
13659
|
+
type: "info",
|
|
13660
|
+
message: `Session cleanup: inspected ${sessions.length}, deleted ${deleted}${failedNote}${skippedNote}${remainingNote} (${mode})`
|
|
13661
|
+
});
|
|
12663
13662
|
}
|
|
12664
|
-
const
|
|
12665
|
-
|
|
12666
|
-
|
|
12667
|
-
|
|
12668
|
-
|
|
12669
|
-
|
|
13663
|
+
const sessionMetadata = {
|
|
13664
|
+
sessions_inspected: sessions?.length ?? null,
|
|
13665
|
+
sessions_deleted: deleted,
|
|
13666
|
+
sessions_failed: failed,
|
|
13667
|
+
sessions_failed_4xx: failed4xx,
|
|
13668
|
+
sessions_failed_5xx: failed5xx,
|
|
13669
|
+
sessions_failed_other: failedOtherStatus,
|
|
13670
|
+
sessions_failed_threw: failedThrew,
|
|
13671
|
+
sessions_skipped_active: skippedNewlyActive,
|
|
13672
|
+
sessions_remaining: remaining,
|
|
13673
|
+
...enumerationReason !== null ? { session_enumeration_reason: enumerationReason } : {}
|
|
13674
|
+
};
|
|
12670
13675
|
const reclaimResult = await reclaimSessionDbSpace({
|
|
12671
13676
|
dbPath: sessionDbPath(),
|
|
12672
|
-
|
|
13677
|
+
chunkPages: SESSION_DB_RECLAIM_CHUNK_PAGES,
|
|
13678
|
+
pageCeiling: SESSION_DB_RECLAIM_PAGE_CEILING,
|
|
13679
|
+
timeBudgetMs: SESSION_DB_RECLAIM_TIME_BUDGET_MS,
|
|
12673
13680
|
allowFullVacuum: protectedNow.size === 0 && !state.sessionDbProvenanceAnomaly
|
|
12674
13681
|
});
|
|
12675
13682
|
if (reclaimResult.ok) {
|
|
13683
|
+
state.sessionStoreHealth.observeWritable();
|
|
12676
13684
|
const beforeMib = (reclaimResult.beforeBytes / 1024 / 1024).toFixed(1);
|
|
12677
13685
|
const afterMib = (reclaimResult.afterBytes / 1024 / 1024).toFixed(1);
|
|
12678
13686
|
const checkpointNote = reclaimResult.checkpoint.busy ? ` (on-disk file truncation deferred: checkpoint busy, ${reclaimResult.checkpoint.log} WAL frames pending)` : "";
|
|
12679
|
-
|
|
13687
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12680
13688
|
type: "info",
|
|
12681
|
-
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB${
|
|
13689
|
+
message: `Session cleanup: reclaimed session-db space (${reclaimResult.mode}): ${beforeMib} MiB -> ${afterMib} MiB (stopped by ${reclaimResult.stoppedBy}; freelist ${reclaimResult.freelistBefore} -> ${reclaimResult.freelistAfter} pages)` + checkpointNote,
|
|
13690
|
+
metadata: {
|
|
13691
|
+
outcome: "reclaimed",
|
|
13692
|
+
mode: reclaimResult.mode,
|
|
13693
|
+
before_bytes: reclaimResult.beforeBytes,
|
|
13694
|
+
after_bytes: reclaimResult.afterBytes,
|
|
13695
|
+
checkpoint_busy: reclaimResult.checkpoint.busy,
|
|
13696
|
+
wal_frames_pending: reclaimResult.checkpoint.log,
|
|
13697
|
+
stopped_by: reclaimResult.stoppedBy,
|
|
13698
|
+
freelist_before: reclaimResult.freelistBefore,
|
|
13699
|
+
freelist_after: reclaimResult.freelistAfter,
|
|
13700
|
+
pages_reclaimed: reclaimResult.pagesReclaimed,
|
|
13701
|
+
reclaim_elapsed_ms: reclaimResult.elapsedMs,
|
|
13702
|
+
...sessionMetadata
|
|
13703
|
+
}
|
|
12682
13704
|
});
|
|
12683
13705
|
} else {
|
|
12684
|
-
|
|
13706
|
+
if (reclaimResult.skipped === "disk-exhaustion") {
|
|
13707
|
+
state.sessionStoreHealth.observeUnwritable();
|
|
13708
|
+
}
|
|
13709
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12685
13710
|
type: "info",
|
|
12686
|
-
|
|
13711
|
+
level: reclaimResult.skipped === "reclaim-error" || reclaimResult.skipped === "disk-exhaustion" || reclaimResult.skipped === "insufficient-disk-space" ? "warn" : "info",
|
|
13712
|
+
message: `Session cleanup: session-db space reclaim skipped (${reclaimResult.skipped})` + (reclaimResult.detail ? `: ${reclaimResult.detail}` : ""),
|
|
13713
|
+
metadata: {
|
|
13714
|
+
outcome: "reclaim-skipped",
|
|
13715
|
+
reason: reclaimResult.skipped,
|
|
13716
|
+
...reclaimResult.detail !== void 0 ? { detail: reclaimResult.detail } : {},
|
|
13717
|
+
...sessionMetadata
|
|
13718
|
+
}
|
|
12687
13719
|
});
|
|
12688
13720
|
}
|
|
12689
13721
|
} catch (error2) {
|
|
12690
13722
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
12691
|
-
|
|
13723
|
+
logSessionCleanupActivity(state, "sweep", {
|
|
12692
13724
|
type: "error",
|
|
12693
|
-
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}
|
|
13725
|
+
error: `Session cleanup sweep failed (non-fatal, ${mode}): ${message}`,
|
|
13726
|
+
metadata: {
|
|
13727
|
+
outcome: "sweep-failed",
|
|
13728
|
+
reason: "sweep-error"
|
|
13729
|
+
}
|
|
12694
13730
|
});
|
|
12695
13731
|
}
|
|
12696
13732
|
}
|
|
@@ -12706,14 +13742,19 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12706
13742
|
for (const warning2 of config.warnings) {
|
|
12707
13743
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
12708
13744
|
}
|
|
12709
|
-
const dbBytes = statSessionDbBytes(
|
|
13745
|
+
const dbBytes = statSessionDbBytes(homedir7());
|
|
12710
13746
|
void (async () => {
|
|
12711
13747
|
const reclaimAvailability = dbBytes !== null && config.enabled ? await probeReclaimAvailability({ dbPath: sessionDbPath(), requiredBytes: dbBytes }) : null;
|
|
12712
13748
|
if (reclaimAvailability !== null) {
|
|
12713
|
-
|
|
13749
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12714
13750
|
type: "info",
|
|
12715
13751
|
level: shouldWarnForReclaimSkip(reclaimAvailability.reason) ? "warn" : "info",
|
|
12716
|
-
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : "")
|
|
13752
|
+
message: `Session cleanup: reclaim preflight skipped (${reclaimAvailability.reason})` + (reclaimAvailability.detail ? `: ${reclaimAvailability.detail}` : ""),
|
|
13753
|
+
metadata: {
|
|
13754
|
+
outcome: "preflight-skipped",
|
|
13755
|
+
reason: reclaimAvailability.reason,
|
|
13756
|
+
...reclaimAvailability.detail !== void 0 ? { detail: reclaimAvailability.detail } : {}
|
|
13757
|
+
}
|
|
12717
13758
|
});
|
|
12718
13759
|
}
|
|
12719
13760
|
const sizeWarning = buildSessionStoreSizeWarning({
|
|
@@ -12730,9 +13771,15 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
12730
13771
|
);
|
|
12731
13772
|
});
|
|
12732
13773
|
if (!config.enabled) return;
|
|
12733
|
-
|
|
13774
|
+
logSessionCleanupActivity(state, "startup", {
|
|
12734
13775
|
type: "info",
|
|
12735
|
-
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)
|
|
13776
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`,
|
|
13777
|
+
metadata: {
|
|
13778
|
+
outcome: "armed",
|
|
13779
|
+
interval_ms: config.intervalMs,
|
|
13780
|
+
max_age_ms: config.maxAgeMs ?? null,
|
|
13781
|
+
max_count: config.maxCount ?? null
|
|
13782
|
+
}
|
|
12736
13783
|
});
|
|
12737
13784
|
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
12738
13785
|
const firstSweep = setTimeout(
|
|
@@ -12911,7 +13958,7 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12911
13958
|
});
|
|
12912
13959
|
return;
|
|
12913
13960
|
}
|
|
12914
|
-
const { collect, stop } = createResourceUsageCollector(
|
|
13961
|
+
const { collect, stop } = createResourceUsageCollector(homedir7());
|
|
12915
13962
|
state.stopResourceUsageSampling = stop;
|
|
12916
13963
|
let consecutiveFailures = 0;
|
|
12917
13964
|
const tick = async () => {
|
|
@@ -12924,6 +13971,19 @@ function scheduleResourceUsageReporting(state, options) {
|
|
|
12924
13971
|
message: `Resource usage collection: ${warning2}`
|
|
12925
13972
|
});
|
|
12926
13973
|
}
|
|
13974
|
+
const lowDiskDecision = evaluateLowDiskWarning({
|
|
13975
|
+
...usage,
|
|
13976
|
+
nowMs: Date.now(),
|
|
13977
|
+
state: state.lowDiskWarningState
|
|
13978
|
+
});
|
|
13979
|
+
state.lowDiskWarningState = lowDiskDecision.state;
|
|
13980
|
+
if (lowDiskDecision.warning !== null) {
|
|
13981
|
+
logActivity(state, {
|
|
13982
|
+
type: "info",
|
|
13983
|
+
level: "warn",
|
|
13984
|
+
message: lowDiskDecision.warning
|
|
13985
|
+
});
|
|
13986
|
+
}
|
|
12927
13987
|
const result = await reportResourceUsage(state.agentId, state.authHeader, usage);
|
|
12928
13988
|
if (result.ok) {
|
|
12929
13989
|
if (consecutiveFailures > 0) {
|
|
@@ -13136,7 +14196,7 @@ async function run(options) {
|
|
|
13136
14196
|
let fileSyncDirectories;
|
|
13137
14197
|
try {
|
|
13138
14198
|
logLevel = resolveLogLevel(options);
|
|
13139
|
-
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo,
|
|
14199
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir7());
|
|
13140
14200
|
if (options.restoreSessionDb && !options.sessionDbNoReplicateMarker) {
|
|
13141
14201
|
throw new Error(
|
|
13142
14202
|
"--restore-session-db requires --session-db-no-replicate-marker <path>; restore failures must block Litestream replication"
|
|
@@ -13170,6 +14230,7 @@ async function run(options) {
|
|
|
13170
14230
|
opencodeClient: null,
|
|
13171
14231
|
opencodeV2CredentialImportGuard: { inFlight: false },
|
|
13172
14232
|
sessionDbProvenanceAnomaly: false,
|
|
14233
|
+
sessionStoreHealth: createSessionStoreHealth(),
|
|
13173
14234
|
opencodeProcess: null,
|
|
13174
14235
|
stopOpenCodeLogTail: null,
|
|
13175
14236
|
litestreamProcess: null,
|
|
@@ -13187,10 +14248,12 @@ async function run(options) {
|
|
|
13187
14248
|
openaiUsageRearm: null,
|
|
13188
14249
|
resourceUsageTimer: null,
|
|
13189
14250
|
stopResourceUsageSampling: null,
|
|
14251
|
+
lowDiskWarningState: { worstBand: null, warnedAtMs: null },
|
|
13190
14252
|
credentialSync: null,
|
|
13191
14253
|
authHeader: ""
|
|
13192
14254
|
};
|
|
13193
14255
|
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
14256
|
+
setTelemetryDiagnosticsReporter((message) => log2(state, message, "warn"));
|
|
13194
14257
|
if (options.credentialSyncMarker) {
|
|
13195
14258
|
state.credentialSync = createCredentialSync({
|
|
13196
14259
|
markerPath: options.credentialSyncMarker,
|
|
@@ -13359,6 +14422,34 @@ async function run(options) {
|
|
|
13359
14422
|
} else {
|
|
13360
14423
|
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
13361
14424
|
}
|
|
14425
|
+
if (options.restoreRunnerCredentials || options.credentialSyncMarker) {
|
|
14426
|
+
const logCredentialVaultResolution = (message, level) => {
|
|
14427
|
+
log2(state, message, level);
|
|
14428
|
+
if (state.interactive && !state.json) {
|
|
14429
|
+
logActivity(state, { type: "info", level, message });
|
|
14430
|
+
}
|
|
14431
|
+
};
|
|
14432
|
+
const vaultResolution = await getRunnerCredentialVault(state.agentId, state.authHeader);
|
|
14433
|
+
if (vaultResolution.error) {
|
|
14434
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14435
|
+
logCredentialVaultResolution(
|
|
14436
|
+
`CREDENTIAL-VAULT-LOOKUP-FAILED: ${vaultResolution.error}; credentials will be read from and written to this runner's own prefix this boot`,
|
|
14437
|
+
"warn"
|
|
14438
|
+
);
|
|
14439
|
+
} else if (vaultResolution.prefix !== null && vaultResolution.vaultId !== null && vaultResolution.source !== null) {
|
|
14440
|
+
process.env.CREDENTIAL_VAULT_PREFIX = vaultResolution.prefix;
|
|
14441
|
+
logCredentialVaultResolution(
|
|
14442
|
+
`CREDENTIAL-VAULT: source=${vaultResolution.source} vault=${vaultResolution.vaultId} prefix=${vaultResolution.prefix}`,
|
|
14443
|
+
"info"
|
|
14444
|
+
);
|
|
14445
|
+
} else {
|
|
14446
|
+
delete process.env.CREDENTIAL_VAULT_PREFIX;
|
|
14447
|
+
logCredentialVaultResolution(
|
|
14448
|
+
"CREDENTIAL-VAULT-NONE: this runner is not linked to a credential vault; using its own prefix",
|
|
14449
|
+
"debug"
|
|
14450
|
+
);
|
|
14451
|
+
}
|
|
14452
|
+
}
|
|
13362
14453
|
if (options.restoreRunnerCredentials) {
|
|
13363
14454
|
log2(state, "Restoring runner credentials before starting OpenCode");
|
|
13364
14455
|
const credentialContext = {
|
|
@@ -13376,13 +14467,21 @@ async function run(options) {
|
|
|
13376
14467
|
if (githubTokenPopulated) await configureGitHubAccess(credentialContext);
|
|
13377
14468
|
}
|
|
13378
14469
|
state.credentialSync?.arm();
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
createOpenCodeActivityForwarder(() => ({
|
|
14470
|
+
const openCodeActivityForwarder = createOpenCodeActivityForwarder(
|
|
14471
|
+
() => ({
|
|
13382
14472
|
agentId: state.agentId,
|
|
13383
14473
|
authHeader: state.authHeader
|
|
13384
|
-
})
|
|
13385
|
-
|
|
14474
|
+
}),
|
|
14475
|
+
state.sessionStoreHealth.observeUnwritable
|
|
14476
|
+
);
|
|
14477
|
+
const openCodeLogTail = tailOpenCodeLogFile(
|
|
14478
|
+
resolveOpenCodeLogPath(homedir7(), process.env),
|
|
14479
|
+
openCodeActivityForwarder
|
|
14480
|
+
);
|
|
14481
|
+
state.stopOpenCodeLogTail = () => {
|
|
14482
|
+
openCodeActivityForwarder.flush();
|
|
14483
|
+
openCodeLogTail.stop();
|
|
14484
|
+
};
|
|
13386
14485
|
let sessionDbVerifyFatal = false;
|
|
13387
14486
|
if (!options.restoreSessionDb) {
|
|
13388
14487
|
log2(state, "Skipping session-DB restore: --restore-session-db was not passed", "debug");
|
|
@@ -13499,7 +14598,7 @@ async function run(options) {
|
|
|
13499
14598
|
const provenance = checkSessionDbProvenance({
|
|
13500
14599
|
dbPath: sessionDbPath(),
|
|
13501
14600
|
currentVersion: state.opencodeVersion,
|
|
13502
|
-
homeDir:
|
|
14601
|
+
homeDir: homedir7(),
|
|
13503
14602
|
env: process.env
|
|
13504
14603
|
});
|
|
13505
14604
|
if (provenance.anomaly) {
|
|
@@ -13666,10 +14765,11 @@ async function run(options) {
|
|
|
13666
14765
|
getAuthHeader: () => state.authHeader,
|
|
13667
14766
|
conversationFilter: state.conversationFilter,
|
|
13668
14767
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
14768
|
+
sessionStoreHealth: state.sessionStoreHealth,
|
|
13669
14769
|
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
13670
14770
|
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
13671
14771
|
fileSyncDirectories,
|
|
13672
|
-
homeDir:
|
|
14772
|
+
homeDir: homedir7(),
|
|
13673
14773
|
maxActiveSessions,
|
|
13674
14774
|
log: (entry) => (
|
|
13675
14775
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
@@ -13691,12 +14791,13 @@ async function run(options) {
|
|
|
13691
14791
|
openCodePassword: state.opencodePassword,
|
|
13692
14792
|
isRunning: () => state.running,
|
|
13693
14793
|
events: {
|
|
13694
|
-
onConnected: (agentId, isReconnect) => {
|
|
14794
|
+
onConnected: (agentId, isReconnect, transition) => {
|
|
13695
14795
|
state.connected = true;
|
|
13696
14796
|
state.agentId = agentId;
|
|
14797
|
+
const connectionMessage = isReconnect ? `Tunnel reconnected (runner: ${agentId}, generation: ${transition.generation}, attempt: ${transition.attempt}, offline for ${transition.downtimeMs}ms)` : `Tunnel connected (runner: ${agentId}, generation: ${transition.generation})`;
|
|
13697
14798
|
logActivity(state, {
|
|
13698
14799
|
type: "info",
|
|
13699
|
-
message:
|
|
14800
|
+
message: connectionMessage
|
|
13700
14801
|
});
|
|
13701
14802
|
if (options.tunnelReadyFile) {
|
|
13702
14803
|
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
@@ -13717,7 +14818,12 @@ async function run(options) {
|
|
|
13717
14818
|
version: state.opencodeVersion,
|
|
13718
14819
|
major: state.opencodeApiVersion,
|
|
13719
14820
|
connected: state.opencodeConnected
|
|
13720
|
-
})
|
|
14821
|
+
}),
|
|
14822
|
+
connection_generation: transition.generation,
|
|
14823
|
+
...isReconnect && transition.downtimeMs !== null ? {
|
|
14824
|
+
reconnect_attempt: transition.attempt,
|
|
14825
|
+
downtime_ms: transition.downtimeMs
|
|
14826
|
+
} : {}
|
|
13721
14827
|
});
|
|
13722
14828
|
if (!isReconnect) tunnelSpinner?.succeed("Tunnel connected");
|
|
13723
14829
|
if (state.interactive) displayStatus(state);
|
|
@@ -13739,13 +14845,18 @@ async function run(options) {
|
|
|
13739
14845
|
if (state.interactive) displayStatus(state);
|
|
13740
14846
|
});
|
|
13741
14847
|
},
|
|
13742
|
-
onDisconnected: (code, reason) => {
|
|
14848
|
+
onDisconnected: (code, reason, transition) => {
|
|
13743
14849
|
state.connected = false;
|
|
14850
|
+
const safeReason2 = redactDiagnosticText(reason).slice(0, MAX_DISCONNECT_REASON_LENGTH);
|
|
13744
14851
|
logActivity(state, {
|
|
13745
14852
|
type: "info",
|
|
13746
|
-
message: `Tunnel disconnected (code: ${code}, reason: ${
|
|
14853
|
+
message: `Tunnel disconnected (generation: ${transition.generation}, code: ${code}, reason: ${safeReason2})`
|
|
14854
|
+
});
|
|
14855
|
+
emitAgentDisconnected(state.agentId, {
|
|
14856
|
+
code,
|
|
14857
|
+
reason: safeReason2,
|
|
14858
|
+
connection_generation: transition.generation
|
|
13747
14859
|
});
|
|
13748
|
-
emitAgentDisconnected(state.agentId, { code, reason });
|
|
13749
14860
|
if (state.interactive) displayStatus(state);
|
|
13750
14861
|
},
|
|
13751
14862
|
onError: (error2) => {
|
|
@@ -13845,7 +14956,7 @@ async function run(options) {
|
|
|
13845
14956
|
},
|
|
13846
14957
|
report: (usage) => reportOpenAiUsage(state.agentId, state.authHeader, usage),
|
|
13847
14958
|
isLocalCredentialProblem: isLocalCredentialProblem2,
|
|
13848
|
-
forcedOnHint: "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
14959
|
+
forcedOnHint: state.opencodeApiVersion === "v2" ? "OpenCode V2 keeps its connected ChatGPT login in a store Evident cannot read" : "connect a ChatGPT account to this runner, or run `opencode auth login`",
|
|
13849
14960
|
firstDelayMs: firstReportDelayMs,
|
|
13850
14961
|
nextDelayMs: usageReportDelayMs,
|
|
13851
14962
|
failureLogLevel: usageReportFailureLogLevel
|
|
@@ -13888,7 +14999,7 @@ async function run(options) {
|
|
|
13888
14999
|
}
|
|
13889
15000
|
|
|
13890
15001
|
// src/index.ts
|
|
13891
|
-
var
|
|
15002
|
+
var version = getCliVersion();
|
|
13892
15003
|
var program = new Command();
|
|
13893
15004
|
program.name("evident").description("Run OpenCode locally and connect it to Evident").version(version).option(
|
|
13894
15005
|
"--endpoint <url>",
|
|
@@ -13906,6 +15017,7 @@ program.command("login").description("Authenticate with Evident").option("--toke
|
|
|
13906
15017
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
13907
15018
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
13908
15019
|
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
15020
|
+
program.command("credential-vault-env").description("Print the shell assignment a runner image evals before starting the synchroniser").action(credentialVaultEnv);
|
|
13909
15021
|
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
13910
15022
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
13911
15023
|
"-a, --agent [id]",
|