@evident-ai/cli 3.1.1-dev.f24491c → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -11
- package/dist/index.js +3096 -544
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -38,17 +38,39 @@ function getApiUrl() {
|
|
|
38
38
|
function getTunnelUrl() {
|
|
39
39
|
return tunnelOverride ?? process.env.EVIDENT_TUNNEL_URL ?? defaults.tunnelUrl;
|
|
40
40
|
}
|
|
41
|
-
var config = new Conf({
|
|
42
|
-
projectName: "evident",
|
|
43
|
-
projectSuffix: "",
|
|
44
|
-
defaults
|
|
45
|
-
});
|
|
46
41
|
var credentials = new Conf({
|
|
47
42
|
projectName: "evident",
|
|
48
43
|
projectSuffix: "",
|
|
49
44
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
45
|
+
defaults: {},
|
|
46
|
+
configFileMode: 384
|
|
51
47
|
});
|
|
48
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
49
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
50
|
+
var permissionWarningEmitted = false;
|
|
51
|
+
function hardenCredentialsPermissions() {
|
|
52
|
+
if (process.platform === "win32") {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const file = credentials.path;
|
|
56
|
+
for (const [path, mode] of [
|
|
57
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
58
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
59
|
+
]) {
|
|
60
|
+
try {
|
|
61
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
62
|
+
chmodSync(path, mode);
|
|
63
|
+
}
|
|
64
|
+
} catch (err) {
|
|
65
|
+
if (!permissionWarningEmitted) {
|
|
66
|
+
permissionWarningEmitted = true;
|
|
67
|
+
console.error(
|
|
68
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
52
74
|
function getApiUrlConfig() {
|
|
53
75
|
return getApiUrl();
|
|
54
76
|
}
|
|
@@ -59,6 +81,7 @@ function credentialsKey() {
|
|
|
59
81
|
return getApiUrl();
|
|
60
82
|
}
|
|
61
83
|
function getCredentials() {
|
|
84
|
+
hardenCredentialsPermissions();
|
|
62
85
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
86
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
87
|
}
|
|
@@ -70,14 +93,17 @@ function setCredentials(creds) {
|
|
|
70
93
|
expiresAt: creds.expiresAt
|
|
71
94
|
};
|
|
72
95
|
credentials.set("byEndpoint", byEndpoint);
|
|
96
|
+
hardenCredentialsPermissions();
|
|
73
97
|
}
|
|
74
98
|
function clearCredentials() {
|
|
75
99
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
100
|
delete byEndpoint[credentialsKey()];
|
|
77
101
|
credentials.set("byEndpoint", byEndpoint);
|
|
102
|
+
hardenCredentialsPermissions();
|
|
78
103
|
}
|
|
79
104
|
function clearAllCredentials() {
|
|
80
105
|
credentials.clear();
|
|
106
|
+
hardenCredentialsPermissions();
|
|
81
107
|
}
|
|
82
108
|
function getCliName() {
|
|
83
109
|
const argv1 = process.argv[1] || "";
|
|
@@ -236,16 +262,28 @@ async function getToken() {
|
|
|
236
262
|
}
|
|
237
263
|
return null;
|
|
238
264
|
}
|
|
265
|
+
function toError(err) {
|
|
266
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
267
|
+
}
|
|
239
268
|
async function deleteToken(options = {}) {
|
|
240
269
|
const keytar = await getKeytar();
|
|
270
|
+
const failures = [];
|
|
241
271
|
if (keytar) {
|
|
242
272
|
if (options.all) {
|
|
243
|
-
|
|
273
|
+
let accounts = [];
|
|
274
|
+
try {
|
|
275
|
+
accounts = await keytar.findCredentials(SERVICE_NAME);
|
|
276
|
+
} catch (err) {
|
|
277
|
+
failures.push({ type: "enumerate", error: toError(err) });
|
|
278
|
+
}
|
|
244
279
|
await Promise.all(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
280
|
+
accounts.map(async (entry) => {
|
|
281
|
+
try {
|
|
282
|
+
await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
283
|
+
} catch (err) {
|
|
284
|
+
failures.push({ type: "delete", account: entry.account, error: toError(err) });
|
|
285
|
+
}
|
|
286
|
+
})
|
|
249
287
|
);
|
|
250
288
|
} else {
|
|
251
289
|
await keytar.deletePassword(SERVICE_NAME, keychainAccount());
|
|
@@ -256,6 +294,7 @@ async function deleteToken(options = {}) {
|
|
|
256
294
|
} else {
|
|
257
295
|
clearCredentials();
|
|
258
296
|
}
|
|
297
|
+
return { failures };
|
|
259
298
|
}
|
|
260
299
|
|
|
261
300
|
// src/utils/ui.ts
|
|
@@ -285,14 +324,14 @@ function blank() {
|
|
|
285
324
|
console.log();
|
|
286
325
|
}
|
|
287
326
|
function waitForEnter(prompt = "Press Enter to continue...") {
|
|
288
|
-
return new Promise((
|
|
327
|
+
return new Promise((resolve3) => {
|
|
289
328
|
process.stdout.write(chalk.dim(prompt));
|
|
290
329
|
const handler = () => {
|
|
291
330
|
process.stdin.removeListener("data", handler);
|
|
292
331
|
process.stdin.setRawMode?.(false);
|
|
293
332
|
process.stdin.pause();
|
|
294
333
|
console.log();
|
|
295
|
-
|
|
334
|
+
resolve3();
|
|
296
335
|
};
|
|
297
336
|
if (process.stdin.isTTY) {
|
|
298
337
|
process.stdin.setRawMode?.(true);
|
|
@@ -302,7 +341,7 @@ function waitForEnter(prompt = "Press Enter to continue...") {
|
|
|
302
341
|
});
|
|
303
342
|
}
|
|
304
343
|
function sleep(ms) {
|
|
305
|
-
return new Promise((
|
|
344
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
306
345
|
}
|
|
307
346
|
|
|
308
347
|
// src/commands/login.ts
|
|
@@ -373,22 +412,25 @@ async function deviceFlowLogin(options) {
|
|
|
373
412
|
}
|
|
374
413
|
async function tokenLogin() {
|
|
375
414
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
415
|
+
console.log("Create a token under Settings \u2192 CLI tokens in the dashboard, then paste it below.");
|
|
416
|
+
console.log(
|
|
417
|
+
"(Alternatively, run `evident login` on a machine with a browser, or set EVIDENT_TOKEN for CI.)"
|
|
418
|
+
);
|
|
377
419
|
blank();
|
|
378
420
|
process.stdout.write("Paste token: ");
|
|
379
|
-
const token = await new Promise((
|
|
421
|
+
const token = await new Promise((resolve3) => {
|
|
380
422
|
let data = "";
|
|
381
423
|
process.stdin.setEncoding("utf8");
|
|
382
424
|
process.stdin.on("data", (chunk) => {
|
|
383
425
|
data += chunk;
|
|
384
426
|
});
|
|
385
427
|
process.stdin.on("end", () => {
|
|
386
|
-
|
|
428
|
+
resolve3(data.trim());
|
|
387
429
|
});
|
|
388
430
|
if (process.stdin.isTTY) {
|
|
389
431
|
process.stdin.once("data", (chunk) => {
|
|
390
432
|
process.stdin.pause();
|
|
391
|
-
|
|
433
|
+
resolve3(chunk.toString().trim());
|
|
392
434
|
});
|
|
393
435
|
process.stdin.resume();
|
|
394
436
|
}
|
|
@@ -397,13 +439,22 @@ async function tokenLogin() {
|
|
|
397
439
|
printError("No token provided.");
|
|
398
440
|
process.exit(1);
|
|
399
441
|
}
|
|
442
|
+
await validateAndStoreToken(token);
|
|
443
|
+
}
|
|
444
|
+
async function validateAndStoreToken(token) {
|
|
400
445
|
const spinner = ora("Validating token...").start();
|
|
401
446
|
try {
|
|
402
|
-
const result = await api.
|
|
447
|
+
const result = await api.get("/me", {
|
|
448
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
449
|
+
});
|
|
450
|
+
if (!result.user) {
|
|
451
|
+
throw new Error(
|
|
452
|
+
"This token is not a user login (e.g. a runner key). Paste a CLI token instead."
|
|
453
|
+
);
|
|
454
|
+
}
|
|
403
455
|
await storeToken({
|
|
404
456
|
token,
|
|
405
|
-
user: result.user
|
|
406
|
-
expiresAt: result.expires_at
|
|
457
|
+
user: { email: result.user.email }
|
|
407
458
|
});
|
|
408
459
|
spinner.stop();
|
|
409
460
|
printSuccess(`Logged in as ${chalk2.bold(result.user.email)}`);
|
|
@@ -423,9 +474,22 @@ async function login(options) {
|
|
|
423
474
|
}
|
|
424
475
|
|
|
425
476
|
// src/commands/logout.ts
|
|
477
|
+
function describeFailure(failure) {
|
|
478
|
+
if (failure.type === "enumerate") {
|
|
479
|
+
return `could not list stored keychain entries (${failure.error.message})`;
|
|
480
|
+
}
|
|
481
|
+
return `${failure.account} (${failure.error.message})`;
|
|
482
|
+
}
|
|
426
483
|
async function logout(options = {}) {
|
|
427
484
|
if (options.all) {
|
|
428
|
-
await deleteToken({ all: true });
|
|
485
|
+
const result = await deleteToken({ all: true });
|
|
486
|
+
if (result.failures.length > 0) {
|
|
487
|
+
printError(
|
|
488
|
+
`Failed to fully clear your keychain: ${result.failures.map(describeFailure).join("; ")}. Your local credentials file was cleared, but stale keychain entries may remain \u2014 run \`evident logout --all\` again, or remove them manually from your OS keychain / credential manager.`
|
|
489
|
+
);
|
|
490
|
+
process.exitCode = 1;
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
429
493
|
printSuccess("Logged out of all endpoints.");
|
|
430
494
|
return;
|
|
431
495
|
}
|
|
@@ -450,7 +514,9 @@ async function whoami() {
|
|
|
450
514
|
blank();
|
|
451
515
|
console.log(keyValue("Endpoint", apiUrl));
|
|
452
516
|
console.log(keyValue("User", chalk3.bold(credentials2.user.email)));
|
|
453
|
-
|
|
517
|
+
if (credentials2.user.id) {
|
|
518
|
+
console.log(keyValue("User ID", credentials2.user.id));
|
|
519
|
+
}
|
|
454
520
|
if (credentials2.expiresAt) {
|
|
455
521
|
const expiresAt = new Date(credentials2.expiresAt);
|
|
456
522
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -466,192 +532,6 @@ async function whoami() {
|
|
|
466
532
|
blank();
|
|
467
533
|
}
|
|
468
534
|
|
|
469
|
-
// src/commands/run.ts
|
|
470
|
-
import chalk6 from "chalk";
|
|
471
|
-
import ora3 from "ora";
|
|
472
|
-
import { select as select3 } from "@inquirer/prompts";
|
|
473
|
-
|
|
474
|
-
// ../../packages/types/src/telemetry/index.ts
|
|
475
|
-
var TelemetryEventTypes = {
|
|
476
|
-
// Agent activity events (shown in web UI activity log)
|
|
477
|
-
AGENT_CONNECTED: "agent.connected",
|
|
478
|
-
AGENT_DISCONNECTED: "agent.disconnected",
|
|
479
|
-
AGENT_MESSAGE_PROCESSING: "agent.message_processing",
|
|
480
|
-
AGENT_MESSAGE_DONE: "agent.message_done",
|
|
481
|
-
AGENT_MESSAGE_FAILED: "agent.message_failed"
|
|
482
|
-
};
|
|
483
|
-
|
|
484
|
-
// ../../packages/types/src/tunnel/index.ts
|
|
485
|
-
var MAX_FRAME_BYTES = 256 * 1024;
|
|
486
|
-
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
487
|
-
|
|
488
|
-
// ../../packages/types/src/logging/index.ts
|
|
489
|
-
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
490
|
-
function log(level, event, fields) {
|
|
491
|
-
const method = level === "debug" ? "log" : level;
|
|
492
|
-
try {
|
|
493
|
-
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
494
|
-
} catch (err) {
|
|
495
|
-
console.error(
|
|
496
|
-
"[evident] log_serialize_failed",
|
|
497
|
-
event,
|
|
498
|
-
err instanceof Error ? err.message : String(err)
|
|
499
|
-
);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
function stripQuery(url) {
|
|
503
|
-
try {
|
|
504
|
-
return new URL(url).pathname;
|
|
505
|
-
} catch {
|
|
506
|
-
const q = url.indexOf("?");
|
|
507
|
-
return q === -1 ? url : url.slice(0, q);
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
// src/lib/telemetry.ts
|
|
512
|
-
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
513
|
-
function getCliVersion() {
|
|
514
|
-
return CLI_VERSION;
|
|
515
|
-
}
|
|
516
|
-
var eventBuffer = [];
|
|
517
|
-
var flushTimeout = null;
|
|
518
|
-
var isShuttingDown = false;
|
|
519
|
-
var FLUSH_INTERVAL_MS = 5e3;
|
|
520
|
-
var MAX_BUFFER_SIZE = 50;
|
|
521
|
-
var FLUSH_TIMEOUT_MS = 3e3;
|
|
522
|
-
function logEvent(eventType, options = {}) {
|
|
523
|
-
const event = {
|
|
524
|
-
event_type: eventType,
|
|
525
|
-
severity: options.severity || "info",
|
|
526
|
-
message: options.message,
|
|
527
|
-
metadata: options.metadata,
|
|
528
|
-
agent_id: options.agentId,
|
|
529
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
530
|
-
};
|
|
531
|
-
eventBuffer.push(event);
|
|
532
|
-
if (options.severity === "error" || eventBuffer.length >= MAX_BUFFER_SIZE) {
|
|
533
|
-
void flushEvents();
|
|
534
|
-
} else if (!flushTimeout && !isShuttingDown) {
|
|
535
|
-
flushTimeout = setTimeout(() => {
|
|
536
|
-
flushTimeout = null;
|
|
537
|
-
void flushEvents();
|
|
538
|
-
}, FLUSH_INTERVAL_MS);
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
var telemetry = {
|
|
542
|
-
debug: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "debug", message, metadata, agentId }),
|
|
543
|
-
info: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "info", message, metadata, agentId }),
|
|
544
|
-
warn: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "warning", message, metadata, agentId }),
|
|
545
|
-
error: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "error", message, metadata, agentId })
|
|
546
|
-
};
|
|
547
|
-
async function flushEvents() {
|
|
548
|
-
if (eventBuffer.length === 0) return;
|
|
549
|
-
const events = eventBuffer;
|
|
550
|
-
eventBuffer = [];
|
|
551
|
-
if (flushTimeout) {
|
|
552
|
-
clearTimeout(flushTimeout);
|
|
553
|
-
flushTimeout = null;
|
|
554
|
-
}
|
|
555
|
-
try {
|
|
556
|
-
const credentials2 = await getToken();
|
|
557
|
-
if (!credentials2) {
|
|
558
|
-
return;
|
|
559
|
-
}
|
|
560
|
-
const apiUrl = getApiUrlConfig();
|
|
561
|
-
const controller = new AbortController();
|
|
562
|
-
const timeout = setTimeout(() => controller.abort(), FLUSH_TIMEOUT_MS);
|
|
563
|
-
try {
|
|
564
|
-
const request = {
|
|
565
|
-
events,
|
|
566
|
-
client_type: "cli",
|
|
567
|
-
client_version: CLI_VERSION
|
|
568
|
-
};
|
|
569
|
-
const response = await fetch(`${apiUrl}/telemetry/events`, {
|
|
570
|
-
method: "POST",
|
|
571
|
-
headers: {
|
|
572
|
-
"Content-Type": "application/json",
|
|
573
|
-
Authorization: `Bearer ${credentials2.token}`
|
|
574
|
-
},
|
|
575
|
-
body: JSON.stringify(request),
|
|
576
|
-
signal: controller.signal
|
|
577
|
-
});
|
|
578
|
-
if (!response.ok) {
|
|
579
|
-
console.error(`Telemetry flush failed: ${response.status}`);
|
|
580
|
-
}
|
|
581
|
-
} finally {
|
|
582
|
-
clearTimeout(timeout);
|
|
583
|
-
}
|
|
584
|
-
} catch (error2) {
|
|
585
|
-
if (process.env.DEBUG) {
|
|
586
|
-
console.error("Telemetry error:", error2);
|
|
587
|
-
}
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
async function shutdownTelemetry() {
|
|
591
|
-
isShuttingDown = true;
|
|
592
|
-
if (flushTimeout) {
|
|
593
|
-
clearTimeout(flushTimeout);
|
|
594
|
-
flushTimeout = null;
|
|
595
|
-
}
|
|
596
|
-
await flushEvents();
|
|
597
|
-
}
|
|
598
|
-
function emitEvent(event) {
|
|
599
|
-
logEvent(event.event_type, {
|
|
600
|
-
severity: event.severity,
|
|
601
|
-
message: event.message,
|
|
602
|
-
metadata: event.metadata,
|
|
603
|
-
agentId: event.agent_id
|
|
604
|
-
});
|
|
605
|
-
}
|
|
606
|
-
function emitAgentConnected(agentId, metadata) {
|
|
607
|
-
emitEvent({
|
|
608
|
-
event_type: TelemetryEventTypes.AGENT_CONNECTED,
|
|
609
|
-
severity: "info",
|
|
610
|
-
message: "Agent CLI connected",
|
|
611
|
-
metadata,
|
|
612
|
-
agent_id: agentId
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
function emitAgentDisconnected(agentId, metadata) {
|
|
616
|
-
emitEvent({
|
|
617
|
-
event_type: TelemetryEventTypes.AGENT_DISCONNECTED,
|
|
618
|
-
severity: "info",
|
|
619
|
-
message: `Agent CLI disconnected (code: ${metadata.code})`,
|
|
620
|
-
metadata,
|
|
621
|
-
agent_id: agentId
|
|
622
|
-
});
|
|
623
|
-
}
|
|
624
|
-
var EventTypes = {
|
|
625
|
-
// Tunnel lifecycle
|
|
626
|
-
TUNNEL_STARTING: "tunnel.starting",
|
|
627
|
-
TUNNEL_CONNECTED: "tunnel.connected",
|
|
628
|
-
TUNNEL_DISCONNECTED: "tunnel.disconnected",
|
|
629
|
-
TUNNEL_RECONNECTING: "tunnel.reconnecting",
|
|
630
|
-
TUNNEL_ERROR: "tunnel.error",
|
|
631
|
-
// OpenCode communication
|
|
632
|
-
OPENCODE_HEALTH_CHECK: "opencode.health_check",
|
|
633
|
-
OPENCODE_HEALTH_OK: "opencode.health_ok",
|
|
634
|
-
OPENCODE_HEALTH_FAILED: "opencode.health_failed",
|
|
635
|
-
OPENCODE_REQUEST_RECEIVED: "opencode.request_received",
|
|
636
|
-
OPENCODE_REQUEST_FORWARDED: "opencode.request_forwarded",
|
|
637
|
-
OPENCODE_RESPONSE_SENT: "opencode.response_sent",
|
|
638
|
-
OPENCODE_UNREACHABLE: "opencode.unreachable",
|
|
639
|
-
OPENCODE_ERROR: "opencode.error",
|
|
640
|
-
// Authentication
|
|
641
|
-
AUTH_LOGIN_STARTED: "auth.login_started",
|
|
642
|
-
AUTH_LOGIN_SUCCESS: "auth.login_success",
|
|
643
|
-
AUTH_LOGIN_FAILED: "auth.login_failed",
|
|
644
|
-
AUTH_LOGOUT: "auth.logout",
|
|
645
|
-
// CLI lifecycle
|
|
646
|
-
CLI_STARTED: "cli.started",
|
|
647
|
-
CLI_COMMAND: "cli.command",
|
|
648
|
-
CLI_ERROR: "cli.error",
|
|
649
|
-
// Deprecation telemetry (#412) — usage of the old `--agent`/`EVIDENT_AGENT_KEY`
|
|
650
|
-
// names instead of the preferred `--runner`/`EVIDENT_RUNNER_KEY` (#409).
|
|
651
|
-
DEPRECATED_AGENT_FLAG_USED: "cli.deprecated_agent_flag_used",
|
|
652
|
-
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
653
|
-
};
|
|
654
|
-
|
|
655
535
|
// src/lib/auth.ts
|
|
656
536
|
async function getAuthCredentials() {
|
|
657
537
|
const runnerKey = process.env.EVIDENT_RUNNER_KEY;
|
|
@@ -695,38 +575,755 @@ function isInteractive(jsonOutput) {
|
|
|
695
575
|
return true;
|
|
696
576
|
}
|
|
697
577
|
|
|
698
|
-
// src/
|
|
699
|
-
async function
|
|
578
|
+
// src/commands/agent-lookup.ts
|
|
579
|
+
async function readErrorMessage(response) {
|
|
580
|
+
const text = await response.text().catch(() => "");
|
|
581
|
+
if (!text) return response.statusText || void 0;
|
|
700
582
|
try {
|
|
701
|
-
const
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
if (!response.ok) {
|
|
706
|
-
return { healthy: false, error: `HTTP ${response.status}` };
|
|
583
|
+
const data = JSON.parse(text);
|
|
584
|
+
const message = data.message ?? data.error;
|
|
585
|
+
if (typeof message === "string" && message.trim()) {
|
|
586
|
+
return message;
|
|
707
587
|
}
|
|
708
|
-
|
|
709
|
-
return { healthy: true, version: data.version };
|
|
710
|
-
} catch (error2) {
|
|
711
|
-
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
712
|
-
return { healthy: false, error: message };
|
|
588
|
+
} catch {
|
|
713
589
|
}
|
|
590
|
+
return text.trim() || response.statusText || void 0;
|
|
714
591
|
}
|
|
715
|
-
|
|
716
|
-
const
|
|
717
|
-
|
|
718
|
-
const health = await checkOpenCodeHealth(port);
|
|
719
|
-
if (health.healthy) {
|
|
720
|
-
return health;
|
|
721
|
-
}
|
|
722
|
-
await new Promise((resolve2) => setTimeout(resolve2, 1e3));
|
|
723
|
-
}
|
|
724
|
-
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
592
|
+
function authFailureHint(apiUrl, serverMessage) {
|
|
593
|
+
const reason = serverMessage ? `: ${serverMessage}` : "";
|
|
594
|
+
return `Authentication failed${reason}. Your credentials were rejected by ${apiUrl}. This usually means you logged in against a different environment, or your session expired \u2014 log in again pointing at this endpoint and retry.`;
|
|
725
595
|
}
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
596
|
+
async function resolveAgentIdFromKey(authHeader) {
|
|
597
|
+
const apiUrl = getApiUrlConfig();
|
|
598
|
+
try {
|
|
599
|
+
const response = await fetch(`${apiUrl}/me`, {
|
|
600
|
+
headers: { Authorization: authHeader }
|
|
601
|
+
});
|
|
602
|
+
if (response.status === 401) {
|
|
603
|
+
const serverMessage = await readErrorMessage(response);
|
|
604
|
+
return { error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
605
|
+
}
|
|
606
|
+
if (!response.ok) {
|
|
607
|
+
const serverMessage = await readErrorMessage(response);
|
|
608
|
+
return {
|
|
609
|
+
error: `Failed to resolve runner from key (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
const data = await response.json();
|
|
613
|
+
if (data.auth_type === "agent_key" && data.agent_id) {
|
|
614
|
+
return { agent_id: data.agent_id };
|
|
615
|
+
}
|
|
616
|
+
return {
|
|
617
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
618
|
+
};
|
|
619
|
+
} catch (error2) {
|
|
620
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
621
|
+
return { error: `Failed to resolve runner from key: ${message}` };
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
625
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
626
|
+
const apiUrl = getApiUrlConfig();
|
|
627
|
+
try {
|
|
628
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
629
|
+
method: "POST",
|
|
630
|
+
headers: { Authorization: authHeader },
|
|
631
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
632
|
+
});
|
|
633
|
+
if (!response.ok) {
|
|
634
|
+
const serverMessage = await readErrorMessage(response);
|
|
635
|
+
return {
|
|
636
|
+
ok: false,
|
|
637
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
return { ok: true };
|
|
641
|
+
} catch (error2) {
|
|
642
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
function describeBestEffortError(error2) {
|
|
646
|
+
const name = error2?.name;
|
|
647
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
648
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
649
|
+
}
|
|
650
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
651
|
+
}
|
|
652
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
653
|
+
try {
|
|
654
|
+
const apiUrl = getApiUrlConfig();
|
|
655
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
656
|
+
method: "POST",
|
|
657
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
658
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
659
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
660
|
+
});
|
|
661
|
+
if (!response.ok) {
|
|
662
|
+
const serverMessage = await readErrorMessage(response);
|
|
663
|
+
return {
|
|
664
|
+
ok: false,
|
|
665
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
return { ok: true };
|
|
669
|
+
} catch (error2) {
|
|
670
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
function toReportedWindow(window) {
|
|
674
|
+
if (!window) return null;
|
|
675
|
+
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
676
|
+
}
|
|
677
|
+
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
678
|
+
try {
|
|
679
|
+
const apiUrl = getApiUrlConfig();
|
|
680
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/claude-usage`, {
|
|
681
|
+
method: "POST",
|
|
682
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
683
|
+
body: JSON.stringify({
|
|
684
|
+
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
685
|
+
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
686
|
+
}),
|
|
687
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
688
|
+
});
|
|
689
|
+
if (!response.ok) {
|
|
690
|
+
const serverMessage = await readErrorMessage(response);
|
|
691
|
+
return {
|
|
692
|
+
ok: false,
|
|
693
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
return { ok: true };
|
|
697
|
+
} catch (error2) {
|
|
698
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
async function getAgentInfo(agentId, authHeader) {
|
|
702
|
+
const apiUrl = getApiUrlConfig();
|
|
703
|
+
try {
|
|
704
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}`, {
|
|
705
|
+
headers: { Authorization: authHeader }
|
|
706
|
+
});
|
|
707
|
+
if (response.status === 401) {
|
|
708
|
+
const serverMessage = await readErrorMessage(response);
|
|
709
|
+
return { valid: false, error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
710
|
+
}
|
|
711
|
+
if (response.status === 403) {
|
|
712
|
+
const serverMessage = await readErrorMessage(response);
|
|
713
|
+
return {
|
|
714
|
+
valid: false,
|
|
715
|
+
error: serverMessage ?? "You do not have access to this runner (it may belong to a different team or organization)."
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
if (response.status === 404) {
|
|
719
|
+
const serverMessage = await readErrorMessage(response);
|
|
720
|
+
return { valid: false, error: serverMessage ?? `Runner ${agentId} not found` };
|
|
721
|
+
}
|
|
722
|
+
if (!response.ok) {
|
|
723
|
+
const serverMessage = await readErrorMessage(response);
|
|
724
|
+
return {
|
|
725
|
+
valid: false,
|
|
726
|
+
error: `API error (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
const agent = await response.json();
|
|
730
|
+
if (agent.agent_type !== "local") {
|
|
731
|
+
return {
|
|
732
|
+
valid: false,
|
|
733
|
+
error: `Runner is type '${agent.agent_type}', must be 'local' for CLI connection`
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
return { valid: true, agent };
|
|
737
|
+
} catch (error2) {
|
|
738
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
739
|
+
return { valid: false, error: `Failed to validate runner: ${message}` };
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
// src/commands/status.ts
|
|
744
|
+
var STATUS_TIMEOUT_MS = 1e4;
|
|
745
|
+
function authLabelFor(credentials2) {
|
|
746
|
+
if (credentials2.authType === "agent_key") {
|
|
747
|
+
return credentials2.keySource === "agent_key" ? "runner key (EVIDENT_AGENT_KEY)" : "runner key (EVIDENT_RUNNER_KEY)";
|
|
748
|
+
}
|
|
749
|
+
return "user token";
|
|
750
|
+
}
|
|
751
|
+
function describeFetchError(error2) {
|
|
752
|
+
const name = error2?.name;
|
|
753
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
754
|
+
return `timed out after ${STATUS_TIMEOUT_MS}ms waiting for a response`;
|
|
755
|
+
}
|
|
756
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
757
|
+
}
|
|
758
|
+
async function checkStatus(jsonMode) {
|
|
759
|
+
const apiUrl = getApiUrlConfig();
|
|
760
|
+
const credentials2 = await getAuthCredentials();
|
|
761
|
+
if (!credentials2) {
|
|
762
|
+
return {
|
|
763
|
+
ok: false,
|
|
764
|
+
endpoint: apiUrl,
|
|
765
|
+
reason: "no_credentials",
|
|
766
|
+
error: "No credentials configured. Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY), or EVIDENT_TOKEN, or run `evident login`.",
|
|
767
|
+
exitCode: 1
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
if (credentials2.notice && !jsonMode) {
|
|
771
|
+
printWarning(credentials2.notice);
|
|
772
|
+
}
|
|
773
|
+
let response;
|
|
774
|
+
try {
|
|
775
|
+
response = await fetch(`${apiUrl}/me`, {
|
|
776
|
+
headers: { Authorization: getAuthHeader(credentials2) },
|
|
777
|
+
signal: AbortSignal.timeout(STATUS_TIMEOUT_MS)
|
|
778
|
+
});
|
|
779
|
+
} catch (error2) {
|
|
780
|
+
return {
|
|
781
|
+
ok: false,
|
|
782
|
+
endpoint: apiUrl,
|
|
783
|
+
authLabel: authLabelFor(credentials2),
|
|
784
|
+
reason: "unreachable",
|
|
785
|
+
error: `Could not reach ${apiUrl}: ${describeFetchError(error2)}. The credentials were NOT validated.`,
|
|
786
|
+
exitCode: 75
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
if (response.status === 401) {
|
|
790
|
+
const serverMessage = await readErrorMessage(response);
|
|
791
|
+
return {
|
|
792
|
+
ok: false,
|
|
793
|
+
endpoint: apiUrl,
|
|
794
|
+
authLabel: authLabelFor(credentials2),
|
|
795
|
+
reason: "unauthorized",
|
|
796
|
+
error: authFailureHint(apiUrl, serverMessage),
|
|
797
|
+
exitCode: 1
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
if (response.status === 404) {
|
|
801
|
+
return {
|
|
802
|
+
ok: false,
|
|
803
|
+
endpoint: apiUrl,
|
|
804
|
+
authLabel: authLabelFor(credentials2),
|
|
805
|
+
reason: "endpoint_not_found",
|
|
806
|
+
error: `${apiUrl}/me returned HTTP 404 \u2014 that endpoint has no /me route, so it is probably missing the /v1 prefix. The credentials were NOT validated.`,
|
|
807
|
+
exitCode: 75
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
if (response.status >= 500) {
|
|
811
|
+
const serverMessage = await readErrorMessage(response);
|
|
812
|
+
return {
|
|
813
|
+
ok: false,
|
|
814
|
+
endpoint: apiUrl,
|
|
815
|
+
authLabel: authLabelFor(credentials2),
|
|
816
|
+
reason: "unreachable",
|
|
817
|
+
error: `${apiUrl} returned HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}. The credentials were NOT validated.`,
|
|
818
|
+
exitCode: 75
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
if (!response.ok) {
|
|
822
|
+
const serverMessage = await readErrorMessage(response);
|
|
823
|
+
return {
|
|
824
|
+
ok: false,
|
|
825
|
+
endpoint: apiUrl,
|
|
826
|
+
authLabel: authLabelFor(credentials2),
|
|
827
|
+
reason: "http_error",
|
|
828
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`,
|
|
829
|
+
exitCode: 1
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
const data = await response.json();
|
|
833
|
+
return {
|
|
834
|
+
ok: true,
|
|
835
|
+
endpoint: apiUrl,
|
|
836
|
+
authType: data.auth_type,
|
|
837
|
+
authLabel: authLabelFor(credentials2),
|
|
838
|
+
runnerId: data.auth_type === "agent_key" ? data.agent_id : void 0,
|
|
839
|
+
reason: "ok",
|
|
840
|
+
exitCode: 0
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
function printJson(result) {
|
|
844
|
+
const payload = {
|
|
845
|
+
ok: result.ok,
|
|
846
|
+
endpoint: result.endpoint
|
|
847
|
+
};
|
|
848
|
+
if (result.authType) payload.auth_type = result.authType;
|
|
849
|
+
if (result.runnerId) payload.runner_id = result.runnerId;
|
|
850
|
+
if (result.reason) payload.reason = result.reason;
|
|
851
|
+
if (result.error) payload.error = result.error;
|
|
852
|
+
console.log(JSON.stringify(payload));
|
|
853
|
+
}
|
|
854
|
+
function printHuman(result) {
|
|
855
|
+
blank();
|
|
856
|
+
console.log(keyValue("Endpoint", result.endpoint));
|
|
857
|
+
if (result.ok) {
|
|
858
|
+
console.log(keyValue("Auth", result.authLabel ?? "\u2014"));
|
|
859
|
+
if (result.runnerId) {
|
|
860
|
+
console.log(keyValue("Runner", result.runnerId));
|
|
861
|
+
}
|
|
862
|
+
console.log(keyValue("Status", "OK \u2014 credentials accepted"));
|
|
863
|
+
blank();
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
if (result.authLabel) {
|
|
867
|
+
console.log(keyValue("Auth", result.authLabel));
|
|
868
|
+
}
|
|
869
|
+
blank();
|
|
870
|
+
printError(result.error ?? "Unknown error");
|
|
871
|
+
}
|
|
872
|
+
async function status(options = {}) {
|
|
873
|
+
const result = await checkStatus(Boolean(options.json));
|
|
874
|
+
if (options.json) {
|
|
875
|
+
printJson(result);
|
|
876
|
+
} else {
|
|
877
|
+
printHuman(result);
|
|
878
|
+
}
|
|
879
|
+
process.exit(result.exitCode);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// src/lib/claude-usage.ts
|
|
883
|
+
import { execFileSync } from "child_process";
|
|
884
|
+
import { readFileSync } from "fs";
|
|
885
|
+
import { homedir } from "os";
|
|
886
|
+
import { join } from "path";
|
|
887
|
+
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
888
|
+
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
889
|
+
function parseClaudeCliCredentials(raw) {
|
|
890
|
+
let parsed;
|
|
891
|
+
try {
|
|
892
|
+
parsed = JSON.parse(raw);
|
|
893
|
+
} catch {
|
|
894
|
+
return null;
|
|
895
|
+
}
|
|
896
|
+
const data = parsed.claudeAiOauth ?? parsed;
|
|
897
|
+
const creds = data;
|
|
898
|
+
if (typeof creds.accessToken !== "string" || typeof creds.expiresAt !== "number") {
|
|
899
|
+
return null;
|
|
900
|
+
}
|
|
901
|
+
return { accessToken: creds.accessToken, expiresAt: creds.expiresAt };
|
|
902
|
+
}
|
|
903
|
+
function readClaudeCliCredentials() {
|
|
904
|
+
if (process.platform === "darwin") {
|
|
905
|
+
try {
|
|
906
|
+
const raw = execFileSync(
|
|
907
|
+
"/usr/bin/security",
|
|
908
|
+
["find-generic-password", "-s", KEYCHAIN_SERVICE, "-w"],
|
|
909
|
+
{ encoding: "utf-8", timeout: 2e3, stdio: ["pipe", "pipe", "ignore"] }
|
|
910
|
+
);
|
|
911
|
+
return parseClaudeCliCredentials(raw);
|
|
912
|
+
} catch {
|
|
913
|
+
return null;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
try {
|
|
917
|
+
const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
|
|
918
|
+
return parseClaudeCliCredentials(raw);
|
|
919
|
+
} catch {
|
|
920
|
+
return null;
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
var ClaudeUsageError = class extends Error {
|
|
924
|
+
constructor(message, reason) {
|
|
925
|
+
super(message);
|
|
926
|
+
this.reason = reason;
|
|
927
|
+
}
|
|
928
|
+
};
|
|
929
|
+
function isLocalCredentialProblem(err) {
|
|
930
|
+
return err instanceof ClaudeUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
931
|
+
}
|
|
932
|
+
function normalizeResetsAt(value) {
|
|
933
|
+
const ms = Date.parse(value);
|
|
934
|
+
return Number.isNaN(ms) ? null : new Date(ms).toISOString();
|
|
935
|
+
}
|
|
936
|
+
function toWindow(value) {
|
|
937
|
+
if (!value || typeof value !== "object") {
|
|
938
|
+
return null;
|
|
939
|
+
}
|
|
940
|
+
const window = value;
|
|
941
|
+
if (typeof window.utilization !== "number" || typeof window.resets_at !== "string") {
|
|
942
|
+
return null;
|
|
943
|
+
}
|
|
944
|
+
const resetsAt = normalizeResetsAt(window.resets_at);
|
|
945
|
+
if (resetsAt === null) {
|
|
946
|
+
return null;
|
|
947
|
+
}
|
|
948
|
+
return { utilization: window.utilization, resetsAt };
|
|
949
|
+
}
|
|
950
|
+
async function getClaudeUsage() {
|
|
951
|
+
const credentials2 = readClaudeCliCredentials();
|
|
952
|
+
if (!credentials2) {
|
|
953
|
+
throw new ClaudeUsageError(
|
|
954
|
+
"No local Claude Code login found. Run `claude` once to sign in with your Claude subscription.",
|
|
955
|
+
"no_credentials"
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
959
|
+
throw new ClaudeUsageError(
|
|
960
|
+
"Claude Code credentials have expired. Run `claude` to refresh them.",
|
|
961
|
+
"credentials_expired"
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
const res = await fetch(CLAUDE_USAGE_URL, {
|
|
965
|
+
headers: {
|
|
966
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
967
|
+
"Content-Type": "application/json",
|
|
968
|
+
"anthropic-version": "2023-06-01"
|
|
969
|
+
}
|
|
970
|
+
});
|
|
971
|
+
if (!res.ok) {
|
|
972
|
+
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
973
|
+
}
|
|
974
|
+
const body = await res.json();
|
|
975
|
+
return {
|
|
976
|
+
fiveHour: toWindow(body.five_hour),
|
|
977
|
+
sevenDay: toWindow(body.seven_day)
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
// src/commands/claude-usage.ts
|
|
982
|
+
function formatWindow(label, window) {
|
|
983
|
+
if (!window) {
|
|
984
|
+
return keyValue(label, "not available for this plan");
|
|
985
|
+
}
|
|
986
|
+
const resetsAt = new Date(window.resetsAt);
|
|
987
|
+
return keyValue(label, `${window.utilization}% used, resets ${resetsAt.toLocaleString()}`);
|
|
988
|
+
}
|
|
989
|
+
async function claudeUsage() {
|
|
990
|
+
try {
|
|
991
|
+
const usage = await getClaudeUsage();
|
|
992
|
+
blank();
|
|
993
|
+
console.log(formatWindow("5-hour session", usage.fiveHour));
|
|
994
|
+
console.log(formatWindow("7-day", usage.sevenDay));
|
|
995
|
+
blank();
|
|
996
|
+
} catch (err) {
|
|
997
|
+
if (err instanceof ClaudeUsageError) {
|
|
998
|
+
printError(err.message);
|
|
999
|
+
process.exit(1);
|
|
1000
|
+
}
|
|
1001
|
+
throw err;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// src/commands/run.ts
|
|
1006
|
+
import { homedir as homedir3 } from "os";
|
|
1007
|
+
import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
|
|
1008
|
+
import chalk6 from "chalk";
|
|
1009
|
+
|
|
1010
|
+
// ../../packages/types/src/agents/index.ts
|
|
1011
|
+
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1012
|
+
|
|
1013
|
+
// ../../packages/types/src/telemetry/index.ts
|
|
1014
|
+
var TelemetryEventTypes = {
|
|
1015
|
+
// Agent activity events (shown in web UI activity log)
|
|
1016
|
+
AGENT_CONNECTED: "agent.connected",
|
|
1017
|
+
AGENT_DISCONNECTED: "agent.disconnected",
|
|
1018
|
+
AGENT_MESSAGE_PROCESSING: "agent.message_processing",
|
|
1019
|
+
AGENT_MESSAGE_DONE: "agent.message_done",
|
|
1020
|
+
AGENT_MESSAGE_FAILED: "agent.message_failed",
|
|
1021
|
+
// A `warn`/`error` runner-side log line forwarded server-side for
|
|
1022
|
+
// observability (issue #916) — see `apps/cli/src/lib/runner-activity-telemetry.ts`.
|
|
1023
|
+
RUNNER_ACTIVITY: "runner.activity"
|
|
1024
|
+
};
|
|
1025
|
+
|
|
1026
|
+
// ../../packages/types/src/tunnel/index.ts
|
|
1027
|
+
var MAX_FRAME_BYTES = 256 * 1024;
|
|
1028
|
+
var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
|
|
1029
|
+
|
|
1030
|
+
// ../../packages/types/src/runner-files.ts
|
|
1031
|
+
var MAX_FILE_PUSH_BYTES = 64 * 1024;
|
|
1032
|
+
var MAX_FILE_SYNC_DIRECTORIES = 16;
|
|
1033
|
+
|
|
1034
|
+
// ../../packages/types/src/logging/index.ts
|
|
1035
|
+
var CORRELATION_ID_HEADER = "x-evident-correlation-id";
|
|
1036
|
+
function log(level, event, fields) {
|
|
1037
|
+
const method = level === "debug" ? "log" : level;
|
|
1038
|
+
try {
|
|
1039
|
+
console[method]("[evident]", JSON.stringify({ level, event, ...fields }));
|
|
1040
|
+
} catch (err) {
|
|
1041
|
+
console.error(
|
|
1042
|
+
"[evident] log_serialize_failed",
|
|
1043
|
+
event,
|
|
1044
|
+
err instanceof Error ? err.message : String(err)
|
|
1045
|
+
);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
function errorFields(err) {
|
|
1049
|
+
if (err instanceof Error) {
|
|
1050
|
+
return { error: err.message, error_name: err.name };
|
|
1051
|
+
}
|
|
1052
|
+
return { error: String(err) };
|
|
1053
|
+
}
|
|
1054
|
+
function stripQuery(url) {
|
|
1055
|
+
try {
|
|
1056
|
+
return new URL(url).pathname;
|
|
1057
|
+
} catch {
|
|
1058
|
+
const q = url.indexOf("?");
|
|
1059
|
+
return q === -1 ? url : url.slice(0, q);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
// src/commands/run.ts
|
|
1064
|
+
import ora3 from "ora";
|
|
1065
|
+
import { select as select3 } from "@inquirer/prompts";
|
|
1066
|
+
|
|
1067
|
+
// src/lib/telemetry.ts
|
|
1068
|
+
var CLI_VERSION = (true ? "3.0.0" : void 0) ?? process.env.npm_package_version ?? "unknown";
|
|
1069
|
+
function getCliVersion() {
|
|
1070
|
+
return CLI_VERSION;
|
|
1071
|
+
}
|
|
1072
|
+
var eventBuffer = [];
|
|
1073
|
+
var flushTimeout = null;
|
|
1074
|
+
var isShuttingDown = false;
|
|
1075
|
+
var FLUSH_INTERVAL_MS = 5e3;
|
|
1076
|
+
var MAX_BUFFER_SIZE = 50;
|
|
1077
|
+
var FLUSH_TIMEOUT_MS = 3e3;
|
|
1078
|
+
var authProvider = null;
|
|
1079
|
+
function setTelemetryAuthProvider(provider) {
|
|
1080
|
+
authProvider = provider;
|
|
1081
|
+
}
|
|
1082
|
+
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1083
|
+
var lastFlushFailureLoggedAt = 0;
|
|
1084
|
+
var suppressedFlushFailureCount = 0;
|
|
1085
|
+
function logEvent(eventType, options = {}) {
|
|
1086
|
+
const event = {
|
|
1087
|
+
event_type: eventType,
|
|
1088
|
+
severity: options.severity || "info",
|
|
1089
|
+
message: options.message,
|
|
1090
|
+
metadata: options.metadata,
|
|
1091
|
+
agent_id: options.agentId,
|
|
1092
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1093
|
+
};
|
|
1094
|
+
eventBuffer.push(event);
|
|
1095
|
+
if (options.severity === "error" || eventBuffer.length >= MAX_BUFFER_SIZE) {
|
|
1096
|
+
void flushEvents();
|
|
1097
|
+
} else if (!flushTimeout && !isShuttingDown) {
|
|
1098
|
+
flushTimeout = setTimeout(() => {
|
|
1099
|
+
flushTimeout = null;
|
|
1100
|
+
void flushEvents();
|
|
1101
|
+
}, FLUSH_INTERVAL_MS);
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
var telemetry = {
|
|
1105
|
+
debug: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "debug", message, metadata, agentId }),
|
|
1106
|
+
info: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "info", message, metadata, agentId }),
|
|
1107
|
+
warn: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "warning", message, metadata, agentId }),
|
|
1108
|
+
error: (eventType, message, metadata, agentId) => logEvent(eventType, { severity: "error", message, metadata, agentId })
|
|
1109
|
+
};
|
|
1110
|
+
async function flushEvents() {
|
|
1111
|
+
if (eventBuffer.length === 0) return;
|
|
1112
|
+
const events = eventBuffer;
|
|
1113
|
+
eventBuffer = [];
|
|
1114
|
+
if (flushTimeout) {
|
|
1115
|
+
clearTimeout(flushTimeout);
|
|
1116
|
+
flushTimeout = null;
|
|
1117
|
+
}
|
|
1118
|
+
try {
|
|
1119
|
+
const providerContext = authProvider?.();
|
|
1120
|
+
let authHeader;
|
|
1121
|
+
if (providerContext?.authHeader) {
|
|
1122
|
+
authHeader = providerContext.authHeader;
|
|
1123
|
+
} else {
|
|
1124
|
+
const credentials2 = await getToken();
|
|
1125
|
+
if (!credentials2) {
|
|
1126
|
+
return;
|
|
1127
|
+
}
|
|
1128
|
+
authHeader = `Bearer ${credentials2.token}`;
|
|
1129
|
+
}
|
|
1130
|
+
const apiUrl = getApiUrlConfig();
|
|
1131
|
+
const controller = new AbortController();
|
|
1132
|
+
const timeout = setTimeout(() => controller.abort(), FLUSH_TIMEOUT_MS);
|
|
1133
|
+
try {
|
|
1134
|
+
const request = {
|
|
1135
|
+
events,
|
|
1136
|
+
client_type: "cli",
|
|
1137
|
+
client_version: CLI_VERSION
|
|
1138
|
+
};
|
|
1139
|
+
const response = await fetch(`${apiUrl}/telemetry/events`, {
|
|
1140
|
+
method: "POST",
|
|
1141
|
+
headers: {
|
|
1142
|
+
"Content-Type": "application/json",
|
|
1143
|
+
Authorization: authHeader
|
|
1144
|
+
},
|
|
1145
|
+
body: JSON.stringify(request),
|
|
1146
|
+
signal: controller.signal
|
|
1147
|
+
});
|
|
1148
|
+
if (!response.ok) {
|
|
1149
|
+
console.error(`Telemetry flush failed: ${response.status}`);
|
|
1150
|
+
}
|
|
1151
|
+
} finally {
|
|
1152
|
+
clearTimeout(timeout);
|
|
1153
|
+
}
|
|
1154
|
+
} catch (error2) {
|
|
1155
|
+
const now = Date.now();
|
|
1156
|
+
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1157
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1158
|
+
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1159
|
+
console.error(`Telemetry flush error: ${message}${suffix}`);
|
|
1160
|
+
lastFlushFailureLoggedAt = now;
|
|
1161
|
+
suppressedFlushFailureCount = 0;
|
|
1162
|
+
} else {
|
|
1163
|
+
suppressedFlushFailureCount++;
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
async function shutdownTelemetry() {
|
|
1168
|
+
isShuttingDown = true;
|
|
1169
|
+
if (flushTimeout) {
|
|
1170
|
+
clearTimeout(flushTimeout);
|
|
1171
|
+
flushTimeout = null;
|
|
1172
|
+
}
|
|
1173
|
+
await flushEvents();
|
|
1174
|
+
}
|
|
1175
|
+
function emitEvent(event) {
|
|
1176
|
+
logEvent(event.event_type, {
|
|
1177
|
+
severity: event.severity,
|
|
1178
|
+
message: event.message,
|
|
1179
|
+
metadata: event.metadata,
|
|
1180
|
+
agentId: event.agent_id
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
function emitAgentConnected(agentId, metadata) {
|
|
1184
|
+
emitEvent({
|
|
1185
|
+
event_type: TelemetryEventTypes.AGENT_CONNECTED,
|
|
1186
|
+
severity: "info",
|
|
1187
|
+
message: "Agent CLI connected",
|
|
1188
|
+
metadata,
|
|
1189
|
+
agent_id: agentId
|
|
1190
|
+
});
|
|
1191
|
+
}
|
|
1192
|
+
function emitAgentDisconnected(agentId, metadata) {
|
|
1193
|
+
emitEvent({
|
|
1194
|
+
event_type: TelemetryEventTypes.AGENT_DISCONNECTED,
|
|
1195
|
+
severity: "info",
|
|
1196
|
+
message: `Agent CLI disconnected (code: ${metadata.code})`,
|
|
1197
|
+
metadata,
|
|
1198
|
+
agent_id: agentId
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
var EventTypes = {
|
|
1202
|
+
// Tunnel lifecycle
|
|
1203
|
+
TUNNEL_STARTING: "tunnel.starting",
|
|
1204
|
+
TUNNEL_CONNECTED: "tunnel.connected",
|
|
1205
|
+
TUNNEL_DISCONNECTED: "tunnel.disconnected",
|
|
1206
|
+
TUNNEL_RECONNECTING: "tunnel.reconnecting",
|
|
1207
|
+
TUNNEL_ERROR: "tunnel.error",
|
|
1208
|
+
// OpenCode communication
|
|
1209
|
+
OPENCODE_HEALTH_CHECK: "opencode.health_check",
|
|
1210
|
+
OPENCODE_HEALTH_OK: "opencode.health_ok",
|
|
1211
|
+
OPENCODE_HEALTH_FAILED: "opencode.health_failed",
|
|
1212
|
+
OPENCODE_REQUEST_RECEIVED: "opencode.request_received",
|
|
1213
|
+
OPENCODE_REQUEST_FORWARDED: "opencode.request_forwarded",
|
|
1214
|
+
OPENCODE_RESPONSE_SENT: "opencode.response_sent",
|
|
1215
|
+
OPENCODE_UNREACHABLE: "opencode.unreachable",
|
|
1216
|
+
OPENCODE_ERROR: "opencode.error",
|
|
1217
|
+
// Authentication
|
|
1218
|
+
AUTH_LOGIN_STARTED: "auth.login_started",
|
|
1219
|
+
AUTH_LOGIN_SUCCESS: "auth.login_success",
|
|
1220
|
+
AUTH_LOGIN_FAILED: "auth.login_failed",
|
|
1221
|
+
AUTH_LOGOUT: "auth.logout",
|
|
1222
|
+
// CLI lifecycle
|
|
1223
|
+
CLI_STARTED: "cli.started",
|
|
1224
|
+
CLI_COMMAND: "cli.command",
|
|
1225
|
+
CLI_ERROR: "cli.error",
|
|
1226
|
+
// Deprecation telemetry (#412) — usage of the old `--agent`/`EVIDENT_AGENT_KEY`
|
|
1227
|
+
// names instead of the preferred `--runner`/`EVIDENT_RUNNER_KEY` (#409).
|
|
1228
|
+
DEPRECATED_AGENT_FLAG_USED: "cli.deprecated_agent_flag_used",
|
|
1229
|
+
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
1230
|
+
};
|
|
1231
|
+
|
|
1232
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1233
|
+
var FORWARDED_LEVELS = /* @__PURE__ */ new Set(["warn", "error"]);
|
|
1234
|
+
var SEVERITY_BY_LEVEL = {
|
|
1235
|
+
warn: "warning",
|
|
1236
|
+
error: "error"
|
|
1237
|
+
};
|
|
1238
|
+
var MAX_MESSAGE_LENGTH = 500;
|
|
1239
|
+
var TRUNCATION_MARKER = "\u2026";
|
|
1240
|
+
function redact(message) {
|
|
1241
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1242
|
+
}
|
|
1243
|
+
function truncate(message) {
|
|
1244
|
+
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1245
|
+
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1246
|
+
}
|
|
1247
|
+
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1248
|
+
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1249
|
+
var windowStartedAt = 0;
|
|
1250
|
+
var windowCount = 0;
|
|
1251
|
+
var windowDroppedCount = 0;
|
|
1252
|
+
function admitUnderRateLimit(now) {
|
|
1253
|
+
if (now - windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1254
|
+
if (windowDroppedCount > 0) {
|
|
1255
|
+
console.error(
|
|
1256
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
windowStartedAt = now;
|
|
1260
|
+
windowCount = 0;
|
|
1261
|
+
windowDroppedCount = 0;
|
|
1262
|
+
}
|
|
1263
|
+
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1264
|
+
windowDroppedCount++;
|
|
1265
|
+
if (windowDroppedCount === 1) {
|
|
1266
|
+
console.error(
|
|
1267
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
1268
|
+
);
|
|
1269
|
+
}
|
|
1270
|
+
return false;
|
|
1271
|
+
}
|
|
1272
|
+
windowCount++;
|
|
1273
|
+
return true;
|
|
1274
|
+
}
|
|
1275
|
+
function forwardRunnerActivity(entry, context) {
|
|
1276
|
+
try {
|
|
1277
|
+
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1278
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1279
|
+
if (!admitUnderRateLimit(Date.now())) return;
|
|
1280
|
+
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1281
|
+
const message = truncate(redact(rawMessage));
|
|
1282
|
+
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1283
|
+
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1284
|
+
message,
|
|
1285
|
+
metadata: { source: "cli.run" },
|
|
1286
|
+
agentId: context.agentId
|
|
1287
|
+
});
|
|
1288
|
+
} catch (err) {
|
|
1289
|
+
console.error(
|
|
1290
|
+
`[runner-activity-telemetry] failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1291
|
+
);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
// src/lib/opencode/health.ts
|
|
1296
|
+
async function checkOpenCodeHealth(port) {
|
|
1297
|
+
try {
|
|
1298
|
+
const response = await fetch(`http://127.0.0.1:${port}/global/health`, {
|
|
1299
|
+
signal: AbortSignal.timeout(2e3)
|
|
1300
|
+
// 2 second timeout
|
|
1301
|
+
});
|
|
1302
|
+
if (!response.ok) {
|
|
1303
|
+
return { healthy: false, error: `HTTP ${response.status}` };
|
|
1304
|
+
}
|
|
1305
|
+
const data = await response.json().catch(() => ({}));
|
|
1306
|
+
return { healthy: true, version: data.version };
|
|
1307
|
+
} catch (error2) {
|
|
1308
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
1309
|
+
return { healthy: false, error: message };
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
async function waitForOpenCodeHealth(port, timeoutMs = 3e4) {
|
|
1313
|
+
const startTime = Date.now();
|
|
1314
|
+
while (Date.now() - startTime < timeoutMs) {
|
|
1315
|
+
const health = await checkOpenCodeHealth(port);
|
|
1316
|
+
if (health.healthy) {
|
|
1317
|
+
return health;
|
|
1318
|
+
}
|
|
1319
|
+
await new Promise((resolve3) => setTimeout(resolve3, 1e3));
|
|
1320
|
+
}
|
|
1321
|
+
return { healthy: false, error: "Timeout waiting for OpenCode to be healthy" };
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// src/lib/opencode/opencode-version-gate.ts
|
|
1325
|
+
var QUEUE_VALIDATED_OPENCODE_VERSIONS = ["1.17.11", "1.18.3"];
|
|
1326
|
+
function isQueueValidatedVersion(version2) {
|
|
730
1327
|
if (!version2) return false;
|
|
731
1328
|
return QUEUE_VALIDATED_OPENCODE_VERSIONS.includes(version2);
|
|
732
1329
|
}
|
|
@@ -1358,7 +1955,7 @@ async function sendPromptAsync(port, sessionId, content, options, attachments) {
|
|
|
1358
1955
|
}
|
|
1359
1956
|
}
|
|
1360
1957
|
if (attempt < READ_BACK_ATTEMPTS - 1) {
|
|
1361
|
-
await new Promise((
|
|
1958
|
+
await new Promise((resolve3) => setTimeout(resolve3, READ_BACK_DELAY_MS));
|
|
1362
1959
|
}
|
|
1363
1960
|
}
|
|
1364
1961
|
return null;
|
|
@@ -1486,6 +2083,9 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
1486
2083
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1487
2084
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1488
2085
|
}
|
|
2086
|
+
function isB2AbandonmentConfirmed(params) {
|
|
2087
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
2088
|
+
}
|
|
1489
2089
|
function messageError(messages, userMessageId) {
|
|
1490
2090
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1491
2091
|
const error2 = errorOf(reply);
|
|
@@ -1499,6 +2099,57 @@ function messageError(messages, userMessageId) {
|
|
|
1499
2099
|
}
|
|
1500
2100
|
return "The agent run failed.";
|
|
1501
2101
|
}
|
|
2102
|
+
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2103
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2104
|
+
const error2 = errorOf(reply);
|
|
2105
|
+
if (error2 == null) return false;
|
|
2106
|
+
if (typeof error2 === "string") return error2.trim() === "Aborted";
|
|
2107
|
+
if (typeof error2 === "object") {
|
|
2108
|
+
const e = error2;
|
|
2109
|
+
if (e.name === "MessageAbortedError") return true;
|
|
2110
|
+
if (e.name === "AbortError") return true;
|
|
2111
|
+
const dataMessage = e.data?.message;
|
|
2112
|
+
const rendered = typeof dataMessage === "string" ? dataMessage : typeof e.message === "string" ? e.message : null;
|
|
2113
|
+
return rendered != null && rendered.trim() === "Aborted";
|
|
2114
|
+
}
|
|
2115
|
+
return false;
|
|
2116
|
+
}
|
|
2117
|
+
function messageFailure(messages, userMessageId) {
|
|
2118
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2119
|
+
const error2 = errorOf(reply);
|
|
2120
|
+
if (error2 == null || typeof error2 !== "object") return null;
|
|
2121
|
+
const e = error2;
|
|
2122
|
+
const replyProviderId = reply?.info?.providerID ?? null;
|
|
2123
|
+
const replyModelId = reply?.info?.modelID ?? null;
|
|
2124
|
+
if (e.name === "ProviderAuthError") {
|
|
2125
|
+
const data = e.data;
|
|
2126
|
+
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
2127
|
+
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
2128
|
+
}
|
|
2129
|
+
if (e.name === "APIError") {
|
|
2130
|
+
const data = e.data;
|
|
2131
|
+
const statusCode = data?.statusCode;
|
|
2132
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
2133
|
+
return {
|
|
2134
|
+
kind: "model_auth",
|
|
2135
|
+
providerId: replyProviderId,
|
|
2136
|
+
modelId: replyModelId,
|
|
2137
|
+
reason: "rejected"
|
|
2138
|
+
};
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2141
|
+
return null;
|
|
2142
|
+
}
|
|
2143
|
+
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2144
|
+
if (classified != null) return classified;
|
|
2145
|
+
if (hasConfiguredProvider !== false) return null;
|
|
2146
|
+
return {
|
|
2147
|
+
kind: "model_auth",
|
|
2148
|
+
providerId: replyProviderId,
|
|
2149
|
+
modelId: replyModelId,
|
|
2150
|
+
reason: "missing"
|
|
2151
|
+
};
|
|
2152
|
+
}
|
|
1502
2153
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1503
2154
|
if (!messages || messages.length === 0) return false;
|
|
1504
2155
|
return messages.some(
|
|
@@ -1694,10 +2345,11 @@ var StreamForwarder = class {
|
|
|
1694
2345
|
* Abort every in-flight stream (e.g. on WebSocket close).
|
|
1695
2346
|
*/
|
|
1696
2347
|
abortAll() {
|
|
1697
|
-
for (const stream of this.inflight.
|
|
2348
|
+
for (const [sid, stream] of this.inflight.entries()) {
|
|
1698
2349
|
try {
|
|
1699
2350
|
stream.abort();
|
|
1700
|
-
} catch {
|
|
2351
|
+
} catch (err) {
|
|
2352
|
+
log("error", "forwarder_abort_failed", { sid, ...errorFields(err) });
|
|
1701
2353
|
}
|
|
1702
2354
|
}
|
|
1703
2355
|
this.inflight.clear();
|
|
@@ -1731,12 +2383,12 @@ var StreamForwarder = class {
|
|
|
1731
2383
|
let endBody;
|
|
1732
2384
|
if (has_body) {
|
|
1733
2385
|
const chunks = [];
|
|
1734
|
-
bodyPromise = new Promise((
|
|
2386
|
+
bodyPromise = new Promise((resolve3) => {
|
|
1735
2387
|
pushBody = (buf) => {
|
|
1736
2388
|
chunks.push(buf);
|
|
1737
2389
|
};
|
|
1738
2390
|
endBody = () => {
|
|
1739
|
-
|
|
2391
|
+
resolve3(Buffer.concat(chunks));
|
|
1740
2392
|
};
|
|
1741
2393
|
});
|
|
1742
2394
|
}
|
|
@@ -1853,7 +2505,7 @@ function connectTunnel(options) {
|
|
|
1853
2505
|
} = options;
|
|
1854
2506
|
const tunnelUrl = getTunnelUrlConfig();
|
|
1855
2507
|
const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
|
|
1856
|
-
return new Promise((
|
|
2508
|
+
return new Promise((resolve3, reject) => {
|
|
1857
2509
|
const ws = new WebSocket2(url, {
|
|
1858
2510
|
headers: {
|
|
1859
2511
|
Authorization: authHeader
|
|
@@ -1908,7 +2560,7 @@ function connectTunnel(options) {
|
|
|
1908
2560
|
clearTimeout(connectionTimeout);
|
|
1909
2561
|
const connectedAgentId = message.agent_id ?? agentId;
|
|
1910
2562
|
onConnected?.(connectedAgentId);
|
|
1911
|
-
|
|
2563
|
+
resolve3({
|
|
1912
2564
|
ws,
|
|
1913
2565
|
close: () => ws.close(1e3, "CLI shutdown")
|
|
1914
2566
|
});
|
|
@@ -1970,7 +2622,11 @@ var RunnerConnection = class {
|
|
|
1970
2622
|
if (this.connection) {
|
|
1971
2623
|
try {
|
|
1972
2624
|
this.connection.close();
|
|
1973
|
-
} catch {
|
|
2625
|
+
} catch (err) {
|
|
2626
|
+
log("error", "runner_connection_close_failed", {
|
|
2627
|
+
agent_id: this.resolvedAgentId,
|
|
2628
|
+
...errorFields(err)
|
|
2629
|
+
});
|
|
1974
2630
|
}
|
|
1975
2631
|
this.connection = null;
|
|
1976
2632
|
}
|
|
@@ -2022,6 +2678,447 @@ var RunnerConnection = class {
|
|
|
2022
2678
|
}
|
|
2023
2679
|
};
|
|
2024
2680
|
|
|
2681
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2682
|
+
import { writeFileSync } from "fs";
|
|
2683
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2684
|
+
try {
|
|
2685
|
+
writeFileSync(path, `${agentId}
|
|
2686
|
+
`);
|
|
2687
|
+
return { ok: true };
|
|
2688
|
+
} catch (error2) {
|
|
2689
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2690
|
+
}
|
|
2691
|
+
}
|
|
2692
|
+
|
|
2693
|
+
// src/lib/claude-usage-reporting.ts
|
|
2694
|
+
var VALID_MODES = ["auto", "on", "off"];
|
|
2695
|
+
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
2696
|
+
const raw = flagValue ?? env.EVIDENT_CLAUDE_USAGE_REPORTING;
|
|
2697
|
+
if (raw === void 0 || raw === "") {
|
|
2698
|
+
return { mode: "auto", warnings: [] };
|
|
2699
|
+
}
|
|
2700
|
+
const normalized = raw.trim().toLowerCase();
|
|
2701
|
+
if (VALID_MODES.includes(normalized)) {
|
|
2702
|
+
return { mode: normalized, warnings: [] };
|
|
2703
|
+
}
|
|
2704
|
+
const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
|
|
2705
|
+
return {
|
|
2706
|
+
mode: "auto",
|
|
2707
|
+
warnings: [
|
|
2708
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
2709
|
+
]
|
|
2710
|
+
};
|
|
2711
|
+
}
|
|
2712
|
+
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2713
|
+
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2714
|
+
function nextReportDelayMs(random = Math.random) {
|
|
2715
|
+
const jitterRangeMs = BASE_REPORT_DELAY_MS * REPORT_DELAY_JITTER_FRACTION;
|
|
2716
|
+
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2717
|
+
}
|
|
2718
|
+
var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
|
|
2719
|
+
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
2720
|
+
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2721
|
+
return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
// src/lib/channels/driver.ts
|
|
2725
|
+
import { homedir as homedir2 } from "os";
|
|
2726
|
+
|
|
2727
|
+
// src/lib/file-push.ts
|
|
2728
|
+
import { randomUUID } from "crypto";
|
|
2729
|
+
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2730
|
+
import { basename, dirname as dirname2, isAbsolute, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
2731
|
+
var FILE_MODE = 384;
|
|
2732
|
+
var DIRECTORY_MODE = 448;
|
|
2733
|
+
async function writePushedFile(request) {
|
|
2734
|
+
const { requestedPath, content, allowedDirectories, homeDir } = request;
|
|
2735
|
+
const bytes = content.byteLength;
|
|
2736
|
+
if (allowedDirectories.length === 0) {
|
|
2737
|
+
return refuse("file_sync_disabled", "File sync is not enabled on this runner.", {
|
|
2738
|
+
path: requestedPath,
|
|
2739
|
+
bytes
|
|
2740
|
+
});
|
|
2741
|
+
}
|
|
2742
|
+
if (bytes > MAX_FILE_PUSH_BYTES) {
|
|
2743
|
+
return refuse(
|
|
2744
|
+
"file_too_large",
|
|
2745
|
+
`File is ${bytes} bytes; the limit is ${MAX_FILE_PUSH_BYTES}.`,
|
|
2746
|
+
{
|
|
2747
|
+
path: requestedPath,
|
|
2748
|
+
bytes
|
|
2749
|
+
}
|
|
2750
|
+
);
|
|
2751
|
+
}
|
|
2752
|
+
const candidate = expandAndValidate(requestedPath, homeDir);
|
|
2753
|
+
if (candidate === null) {
|
|
2754
|
+
return refuse("invalid_path", "The requested path is not a valid absolute file path.", {
|
|
2755
|
+
path: requestedPath,
|
|
2756
|
+
bytes
|
|
2757
|
+
});
|
|
2758
|
+
}
|
|
2759
|
+
try {
|
|
2760
|
+
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2761
|
+
dirname2(candidate)
|
|
2762
|
+
);
|
|
2763
|
+
const realTarget = join2(existingAncestor, ...missingSegments, basename(candidate));
|
|
2764
|
+
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
2765
|
+
if (allowedDirectory === null) {
|
|
2766
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2767
|
+
path: realTarget,
|
|
2768
|
+
bytes
|
|
2769
|
+
});
|
|
2770
|
+
}
|
|
2771
|
+
if (missingSegments.length > 0) {
|
|
2772
|
+
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2773
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2774
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2775
|
+
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2776
|
+
path: realTarget,
|
|
2777
|
+
bytes,
|
|
2778
|
+
reason: "parent_changed_after_create"
|
|
2779
|
+
});
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
await writeAtomically(realTarget, content);
|
|
2783
|
+
log("info", "file_push_written", { path: realTarget, bytes });
|
|
2784
|
+
return { ok: true, path: realTarget };
|
|
2785
|
+
} catch (err) {
|
|
2786
|
+
const errno = err.code ?? "UNKNOWN";
|
|
2787
|
+
return refuse("write_failed", `The runner could not write the file (${errno}).`, {
|
|
2788
|
+
path: candidate,
|
|
2789
|
+
bytes,
|
|
2790
|
+
errno,
|
|
2791
|
+
...errorFields(err)
|
|
2792
|
+
});
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
function expandAndValidate(requestedPath, homeDir) {
|
|
2796
|
+
if (requestedPath.trim() === "" || requestedPath.includes("\0")) {
|
|
2797
|
+
return null;
|
|
2798
|
+
}
|
|
2799
|
+
const expanded = requestedPath === "~" ? homeDir : requestedPath.startsWith("~/") ? join2(homeDir, requestedPath.slice(2)) : requestedPath;
|
|
2800
|
+
if (expanded.split(/[/\\]/).includes("..")) {
|
|
2801
|
+
return null;
|
|
2802
|
+
}
|
|
2803
|
+
if (!isAbsolute(expanded)) {
|
|
2804
|
+
return null;
|
|
2805
|
+
}
|
|
2806
|
+
const candidate = resolve2(expanded);
|
|
2807
|
+
const name = basename(candidate);
|
|
2808
|
+
return name === "" || name === "." || name === ".." ? null : candidate;
|
|
2809
|
+
}
|
|
2810
|
+
async function resolveNearestExistingAncestor(directory) {
|
|
2811
|
+
const missingSegments = [];
|
|
2812
|
+
let current = directory;
|
|
2813
|
+
for (; ; ) {
|
|
2814
|
+
try {
|
|
2815
|
+
return { existingAncestor: await realpath(current), missingSegments };
|
|
2816
|
+
} catch (err) {
|
|
2817
|
+
const parent = dirname2(current);
|
|
2818
|
+
if (err.code !== "ENOENT" || parent === current) {
|
|
2819
|
+
throw err;
|
|
2820
|
+
}
|
|
2821
|
+
missingSegments.unshift(basename(current));
|
|
2822
|
+
current = parent;
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
async function findContainingAllowedDirectory(allowedDirectories, realTarget) {
|
|
2827
|
+
for (const directory of allowedDirectories) {
|
|
2828
|
+
if (!isAbsolute(directory)) {
|
|
2829
|
+
log("warn", "file_push_allowed_directory_skipped", { directory, reason: "not_absolute" });
|
|
2830
|
+
continue;
|
|
2831
|
+
}
|
|
2832
|
+
const realDirectory = await realpathCreatingIfMissing(directory);
|
|
2833
|
+
if (realDirectory !== null && contains(realDirectory, realTarget)) {
|
|
2834
|
+
return realDirectory;
|
|
2835
|
+
}
|
|
2836
|
+
}
|
|
2837
|
+
return null;
|
|
2838
|
+
}
|
|
2839
|
+
async function realpathCreatingIfMissing(directory) {
|
|
2840
|
+
try {
|
|
2841
|
+
return await realpath(directory);
|
|
2842
|
+
} catch (err) {
|
|
2843
|
+
if (err.code !== "ENOENT") {
|
|
2844
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2845
|
+
directory,
|
|
2846
|
+
reason: "unresolvable",
|
|
2847
|
+
...errorFields(err)
|
|
2848
|
+
});
|
|
2849
|
+
return null;
|
|
2850
|
+
}
|
|
2851
|
+
}
|
|
2852
|
+
try {
|
|
2853
|
+
await mkdir(directory, { recursive: true, mode: DIRECTORY_MODE });
|
|
2854
|
+
await chmod(directory, DIRECTORY_MODE);
|
|
2855
|
+
return await realpath(directory);
|
|
2856
|
+
} catch (err) {
|
|
2857
|
+
log("warn", "file_push_allowed_directory_skipped", {
|
|
2858
|
+
directory,
|
|
2859
|
+
reason: "create_failed",
|
|
2860
|
+
...errorFields(err)
|
|
2861
|
+
});
|
|
2862
|
+
return null;
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
function contains(realDirectory, realTarget) {
|
|
2866
|
+
const rel = relative(realDirectory, realTarget);
|
|
2867
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
2868
|
+
}
|
|
2869
|
+
async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
2870
|
+
let current = existingAncestor;
|
|
2871
|
+
for (const segment of missingSegments) {
|
|
2872
|
+
current = join2(current, segment);
|
|
2873
|
+
await mkdir(current, { recursive: true, mode: DIRECTORY_MODE });
|
|
2874
|
+
await chmod(current, DIRECTORY_MODE);
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2877
|
+
async function writeAtomically(realTarget, content) {
|
|
2878
|
+
const temporaryPath = join2(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2879
|
+
let handle;
|
|
2880
|
+
try {
|
|
2881
|
+
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
2882
|
+
await handle.writeFile(content);
|
|
2883
|
+
await handle.chmod(FILE_MODE);
|
|
2884
|
+
await handle.close();
|
|
2885
|
+
handle = void 0;
|
|
2886
|
+
await rename(temporaryPath, realTarget);
|
|
2887
|
+
} catch (err) {
|
|
2888
|
+
await discardTemporaryFile(temporaryPath, handle);
|
|
2889
|
+
throw err;
|
|
2890
|
+
}
|
|
2891
|
+
}
|
|
2892
|
+
async function discardTemporaryFile(temporaryPath, handle) {
|
|
2893
|
+
try {
|
|
2894
|
+
await handle?.close();
|
|
2895
|
+
} catch (err) {
|
|
2896
|
+
log("warn", "file_push_temp_close_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2897
|
+
}
|
|
2898
|
+
try {
|
|
2899
|
+
await unlink(temporaryPath);
|
|
2900
|
+
} catch (err) {
|
|
2901
|
+
const errno = err.code;
|
|
2902
|
+
if (errno !== "ENOENT" && errno !== "ENOTDIR") {
|
|
2903
|
+
log("warn", "file_push_temp_cleanup_failed", { path: temporaryPath, ...errorFields(err) });
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
function refuse(code, message, fields) {
|
|
2908
|
+
log(code === "write_failed" ? "error" : "warn", "file_push_refused", { code, ...fields });
|
|
2909
|
+
return { ok: false, code, message };
|
|
2910
|
+
}
|
|
2911
|
+
|
|
2912
|
+
// src/lib/runner-file-sync.ts
|
|
2913
|
+
var MAX_ACK_ATTEMPTS = 5;
|
|
2914
|
+
async function syncPendingRunnerFiles(options) {
|
|
2915
|
+
const pending = await listPendingFiles(options);
|
|
2916
|
+
const pendingIds = new Set(pending.map((file) => file.id));
|
|
2917
|
+
for (const id of options.ackFailures.keys()) {
|
|
2918
|
+
if (!pendingIds.has(id)) options.ackFailures.delete(id);
|
|
2919
|
+
}
|
|
2920
|
+
if (pending.length === 0) return 0;
|
|
2921
|
+
options.log({
|
|
2922
|
+
level: "info",
|
|
2923
|
+
message: `Runner file sync: ${pending.length} file(s) queued for this runner`
|
|
2924
|
+
});
|
|
2925
|
+
let applied = 0;
|
|
2926
|
+
for (const file of pending) {
|
|
2927
|
+
if ((options.ackFailures.get(file.id) ?? 0) >= MAX_ACK_ATTEMPTS) continue;
|
|
2928
|
+
if (await applyOne(options, file)) applied += 1;
|
|
2929
|
+
}
|
|
2930
|
+
return applied;
|
|
2931
|
+
}
|
|
2932
|
+
async function listPendingFiles(options) {
|
|
2933
|
+
let res;
|
|
2934
|
+
try {
|
|
2935
|
+
res = await options.fetchImpl(`${options.apiUrl}/runners/${options.agentId}/files/pending`, {
|
|
2936
|
+
headers: { Authorization: options.getAuthHeader() }
|
|
2937
|
+
});
|
|
2938
|
+
} catch (err) {
|
|
2939
|
+
options.log({
|
|
2940
|
+
level: "warn",
|
|
2941
|
+
message: `Could not list pending runner files \u2014 retrying on the next drain: ${describe(err)}`
|
|
2942
|
+
});
|
|
2943
|
+
return [];
|
|
2944
|
+
}
|
|
2945
|
+
if (!res.ok) {
|
|
2946
|
+
options.log({
|
|
2947
|
+
level: res.status === 404 ? "debug" : "warn",
|
|
2948
|
+
message: `Listing pending runner files returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
2949
|
+
});
|
|
2950
|
+
return [];
|
|
2951
|
+
}
|
|
2952
|
+
let body;
|
|
2953
|
+
try {
|
|
2954
|
+
body = await res.json();
|
|
2955
|
+
} catch (err) {
|
|
2956
|
+
options.log({
|
|
2957
|
+
level: "warn",
|
|
2958
|
+
message: `Pending runner file list was not readable JSON \u2014 retrying on the next drain: ${describe(err)}`
|
|
2959
|
+
});
|
|
2960
|
+
return [];
|
|
2961
|
+
}
|
|
2962
|
+
if (!Array.isArray(body)) {
|
|
2963
|
+
options.log({
|
|
2964
|
+
level: "warn",
|
|
2965
|
+
message: "Pending runner file list was not an array \u2014 ignoring it for this drain"
|
|
2966
|
+
});
|
|
2967
|
+
return [];
|
|
2968
|
+
}
|
|
2969
|
+
const files = [];
|
|
2970
|
+
for (const entry of body) {
|
|
2971
|
+
const file = asPendingFile(entry);
|
|
2972
|
+
if (file === null) {
|
|
2973
|
+
options.log({
|
|
2974
|
+
level: "warn",
|
|
2975
|
+
message: "Ignoring a malformed pending runner file entry (expected id, path and size)"
|
|
2976
|
+
});
|
|
2977
|
+
continue;
|
|
2978
|
+
}
|
|
2979
|
+
files.push(file);
|
|
2980
|
+
}
|
|
2981
|
+
return files;
|
|
2982
|
+
}
|
|
2983
|
+
function asPendingFile(entry) {
|
|
2984
|
+
if (entry === null || typeof entry !== "object") return null;
|
|
2985
|
+
const { id, path, size } = entry;
|
|
2986
|
+
if (typeof id !== "string" || id === "") return null;
|
|
2987
|
+
if (typeof path !== "string" || path === "") return null;
|
|
2988
|
+
if (typeof size !== "number" || !Number.isFinite(size) || size < 0) return null;
|
|
2989
|
+
return { id, path, size };
|
|
2990
|
+
}
|
|
2991
|
+
async function applyOne(options, file) {
|
|
2992
|
+
const label = `${file.id.slice(0, 8)} (${file.path})`;
|
|
2993
|
+
if (options.allowedDirectories.length === 0) {
|
|
2994
|
+
options.log({
|
|
2995
|
+
level: "warn",
|
|
2996
|
+
message: `Runner file ${label} rejected: file sync is not enabled on this runner (start it with --enable-file-sync-to)`
|
|
2997
|
+
});
|
|
2998
|
+
await ack(options, file, "rejected", "file_sync_disabled");
|
|
2999
|
+
return false;
|
|
3000
|
+
}
|
|
3001
|
+
if (file.size > MAX_FILE_PUSH_BYTES) {
|
|
3002
|
+
options.log({
|
|
3003
|
+
level: "warn",
|
|
3004
|
+
message: `Runner file ${label} rejected: declared ${file.size} bytes, the limit is ${MAX_FILE_PUSH_BYTES}`
|
|
3005
|
+
});
|
|
3006
|
+
await ack(options, file, "rejected", "file_too_large");
|
|
3007
|
+
return false;
|
|
3008
|
+
}
|
|
3009
|
+
const download = await downloadContent(options, file, label);
|
|
3010
|
+
if (!download.ok) {
|
|
3011
|
+
if (download.terminal) await ack(options, file, "rejected", download.code);
|
|
3012
|
+
return false;
|
|
3013
|
+
}
|
|
3014
|
+
let outcome;
|
|
3015
|
+
try {
|
|
3016
|
+
outcome = await writePushedFile({
|
|
3017
|
+
requestedPath: file.path,
|
|
3018
|
+
content: download.content,
|
|
3019
|
+
allowedDirectories: options.allowedDirectories,
|
|
3020
|
+
homeDir: options.homeDir
|
|
3021
|
+
});
|
|
3022
|
+
} catch (err) {
|
|
3023
|
+
options.log({
|
|
3024
|
+
level: "error",
|
|
3025
|
+
message: `Runner file ${label} could not be written: ${describe(err)}`
|
|
3026
|
+
});
|
|
3027
|
+
await ack(options, file, "rejected", "write_failed");
|
|
3028
|
+
return false;
|
|
3029
|
+
}
|
|
3030
|
+
if (!outcome.ok) {
|
|
3031
|
+
options.log({
|
|
3032
|
+
level: "warn",
|
|
3033
|
+
message: `Runner file ${label} rejected (${outcome.code}): ${outcome.message}`
|
|
3034
|
+
});
|
|
3035
|
+
await ack(options, file, "rejected", outcome.code);
|
|
3036
|
+
return false;
|
|
3037
|
+
}
|
|
3038
|
+
options.log({
|
|
3039
|
+
level: "info",
|
|
3040
|
+
message: `Runner file ${label} applied (${download.content.byteLength} bytes)`
|
|
3041
|
+
});
|
|
3042
|
+
await ack(options, file, "applied");
|
|
3043
|
+
return true;
|
|
3044
|
+
}
|
|
3045
|
+
function durableDownloadCode(status2) {
|
|
3046
|
+
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
3047
|
+
}
|
|
3048
|
+
async function downloadContent(options, file, label) {
|
|
3049
|
+
try {
|
|
3050
|
+
const res = await options.fetchImpl(
|
|
3051
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/content`,
|
|
3052
|
+
{ headers: { Authorization: options.getAuthHeader() } }
|
|
3053
|
+
);
|
|
3054
|
+
if (!res.ok) {
|
|
3055
|
+
const terminal = res.status >= 400 && res.status < 500 && res.status !== 401 && res.status !== 403 && res.status !== 408 && res.status !== 429;
|
|
3056
|
+
if (!terminal) {
|
|
3057
|
+
options.log({
|
|
3058
|
+
level: "warn",
|
|
3059
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 retrying on the next drain`
|
|
3060
|
+
});
|
|
3061
|
+
return { ok: false, terminal: false };
|
|
3062
|
+
}
|
|
3063
|
+
const code = durableDownloadCode(res.status);
|
|
3064
|
+
options.log({
|
|
3065
|
+
level: "error",
|
|
3066
|
+
message: `Downloading runner file ${label} returned HTTP ${res.status} \u2014 rejecting it as ${code} (the bytes never reached the writer)`
|
|
3067
|
+
});
|
|
3068
|
+
return { ok: false, terminal: true, code };
|
|
3069
|
+
}
|
|
3070
|
+
return { ok: true, content: Buffer.from(await res.arrayBuffer()) };
|
|
3071
|
+
} catch (err) {
|
|
3072
|
+
options.log({
|
|
3073
|
+
level: "warn",
|
|
3074
|
+
message: `Downloading runner file ${label} failed \u2014 retrying on the next drain: ${describe(err)}`
|
|
3075
|
+
});
|
|
3076
|
+
return { ok: false, terminal: false };
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
async function ack(options, file, status2, reason) {
|
|
3080
|
+
const outcome = `${status2}${reason ? ` (${reason})` : ""}`;
|
|
3081
|
+
try {
|
|
3082
|
+
const res = await options.fetchImpl(
|
|
3083
|
+
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/ack`,
|
|
3084
|
+
{
|
|
3085
|
+
method: "POST",
|
|
3086
|
+
headers: {
|
|
3087
|
+
Authorization: options.getAuthHeader(),
|
|
3088
|
+
"Content-Type": "application/json"
|
|
3089
|
+
},
|
|
3090
|
+
body: JSON.stringify(reason ? { status: status2, reason } : { status: status2 })
|
|
3091
|
+
}
|
|
3092
|
+
);
|
|
3093
|
+
if (!res.ok) {
|
|
3094
|
+
recordAckFailure(
|
|
3095
|
+
options,
|
|
3096
|
+
file,
|
|
3097
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} returned HTTP ${res.status}`
|
|
3098
|
+
);
|
|
3099
|
+
return;
|
|
3100
|
+
}
|
|
3101
|
+
options.ackFailures.delete(file.id);
|
|
3102
|
+
} catch (err) {
|
|
3103
|
+
recordAckFailure(
|
|
3104
|
+
options,
|
|
3105
|
+
file,
|
|
3106
|
+
`Acking runner file ${file.id.slice(0, 8)} as ${outcome} failed: ${describe(err)}`
|
|
3107
|
+
);
|
|
3108
|
+
}
|
|
3109
|
+
}
|
|
3110
|
+
function recordAckFailure(options, file, what) {
|
|
3111
|
+
const attempts = (options.ackFailures.get(file.id) ?? 0) + 1;
|
|
3112
|
+
options.ackFailures.set(file.id, attempts);
|
|
3113
|
+
options.log({
|
|
3114
|
+
level: "error",
|
|
3115
|
+
message: attempts >= MAX_ACK_ATTEMPTS ? `${what} \u2014 giving up after ${attempts} attempts. It stays pending until the server expires it; restart the runner to retry.` : `${what} \u2014 it stays pending until a later drain re-acks it (attempt ${attempts} of ${MAX_ACK_ATTEMPTS})`
|
|
3116
|
+
});
|
|
3117
|
+
}
|
|
3118
|
+
function describe(err) {
|
|
3119
|
+
return err instanceof Error ? err.message : String(err);
|
|
3120
|
+
}
|
|
3121
|
+
|
|
2025
3122
|
// src/lib/channels/driver.ts
|
|
2026
3123
|
function messageIdOf(m) {
|
|
2027
3124
|
if (!m || typeof m !== "object") return void 0;
|
|
@@ -2050,7 +3147,11 @@ var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
|
2050
3147
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2051
3148
|
var HEARTBEAT_MS = 6e4;
|
|
2052
3149
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
3150
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
3151
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2053
3152
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
3153
|
+
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3154
|
+
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
2054
3155
|
var ChannelAuthError = class extends Error {
|
|
2055
3156
|
constructor(message) {
|
|
2056
3157
|
super(message);
|
|
@@ -2059,10 +3160,10 @@ var ChannelAuthError = class extends Error {
|
|
|
2059
3160
|
};
|
|
2060
3161
|
var ChannelTerminalError = class extends Error {
|
|
2061
3162
|
status;
|
|
2062
|
-
constructor(message,
|
|
3163
|
+
constructor(message, status2) {
|
|
2063
3164
|
super(message);
|
|
2064
3165
|
this.name = "ChannelTerminalError";
|
|
2065
|
-
this.status =
|
|
3166
|
+
this.status = status2;
|
|
2066
3167
|
}
|
|
2067
3168
|
};
|
|
2068
3169
|
function backoffDelay(attempt, policy) {
|
|
@@ -2070,8 +3171,12 @@ function backoffDelay(attempt, policy) {
|
|
|
2070
3171
|
const capped = Math.min(policy.maxDelayMs, exp);
|
|
2071
3172
|
return Math.floor(Math.random() * capped);
|
|
2072
3173
|
}
|
|
2073
|
-
function isRetryableStatus(
|
|
2074
|
-
return
|
|
3174
|
+
function isRetryableStatus(status2) {
|
|
3175
|
+
return status2 === 429 || status2 >= 500 && status2 <= 599;
|
|
3176
|
+
}
|
|
3177
|
+
var VOLATILE_BODY_FIELD_PATTERN = /("(?:ref|requestId|request_id|traceId|trace_id)"\s*:\s*)"[^"]*"/gi;
|
|
3178
|
+
function normalizeRedrivePollFailureBody(body) {
|
|
3179
|
+
return body.replace(VOLATILE_BODY_FIELD_PATTERN, '$1"<redacted>"').replace(/\s+/g, " ").trim().slice(0, 200);
|
|
2075
3180
|
}
|
|
2076
3181
|
var ChannelDriver = class _ChannelDriver {
|
|
2077
3182
|
agentId;
|
|
@@ -2087,8 +3192,39 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2087
3192
|
pausedMaxWaitMs;
|
|
2088
3193
|
stuckQueuedMs;
|
|
2089
3194
|
now;
|
|
3195
|
+
fileSyncDirectories;
|
|
3196
|
+
homeDir;
|
|
3197
|
+
maxActiveSessions;
|
|
2090
3198
|
/** Cache of conversationId → opencode sessionId. */
|
|
2091
3199
|
sessions = /* @__PURE__ */ new Map();
|
|
3200
|
+
/**
|
|
3201
|
+
* conversationId → the opencode session this runner has ABANDONED as that
|
|
3202
|
+
* conversation's binding (#553), after a genuine (`sessionExists === true`)
|
|
3203
|
+
* dispatch failure: the session still exists but is wedged, so #485's self-heal
|
|
3204
|
+
* must bind a fresh one.
|
|
3205
|
+
*
|
|
3206
|
+
* Dropping the local binding + clearing the server row is not enough on its own:
|
|
3207
|
+
* a SIBLING message dispatched earlier in the same drain is still in-flight under
|
|
3208
|
+
* the same session, and its watcher's routine status writes carry
|
|
3209
|
+
* `opencode_session_id`, RESURRECTING the wedged id server-side after the clear —
|
|
3210
|
+
* and `ensureSession`'s persisted-id fallback then reuses it, defeating the
|
|
3211
|
+
* self-heal. This map makes the runner authoritative instead of racing those
|
|
3212
|
+
* writes: *`ensureSession` never reuses an abandoned id for that conversation,
|
|
3213
|
+
* whatever the server row says* — which holds even when the resurrecting write
|
|
3214
|
+
* is one we deliberately keep (see `markDone`).
|
|
3215
|
+
*
|
|
3216
|
+
* Bounded by construction, on both axes: keyed by CONVERSATION, so N failures on
|
|
3217
|
+
* one conversation hold ONE entry (the newest abandonment replaces the older), and
|
|
3218
|
+
* hard-capped at `MAX_SUPERSEDED_CONVERSATIONS` with FIFO eviction. Only the
|
|
3219
|
+
* NEWEST abandoned id per conversation is guarded: after a second abandonment a
|
|
3220
|
+
* late sibling of the FIRST session can write that id back and `ensureSession`
|
|
3221
|
+
* will reuse it — costing ONE repeat failure, which re-supersedes it. Deliberately
|
|
3222
|
+
* NOT dropped when the session's watcher tears down: `markDone` still writes the
|
|
3223
|
+
* abandoned id back (it must, or the reply is lost), so the guard has to outlive
|
|
3224
|
+
* the turn that resurrects it. In-memory only — a restart forgets it, at the same
|
|
3225
|
+
* bounded cost.
|
|
3226
|
+
*/
|
|
3227
|
+
supersededSessions = /* @__PURE__ */ new Map();
|
|
2092
3228
|
/**
|
|
2093
3229
|
* Per-opencode-session dispatch lock (Task 2.1a). `sendPromptAsync` is no
|
|
2094
3230
|
* longer idempotent (no caller-supplied `messageID`), and its read-back picks
|
|
@@ -2156,6 +3292,84 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2156
3292
|
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
2157
3293
|
*/
|
|
2158
3294
|
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
3295
|
+
/**
|
|
3296
|
+
* "Already emitted `redrive_unresolved` for this row" (#965). Mirrors
|
|
3297
|
+
* `readoptPollUnresolvedSignalled`: `resolveRedrive`'s `unresolved` leaf recurs
|
|
3298
|
+
* every ~2s drain until opencode's status becomes readable, but the
|
|
3299
|
+
* server-visible signal is an OUTCOME, so it fires at most once per row. Cleared
|
|
3300
|
+
* on any non-`unresolved` outcome so the set cannot grow beyond the currently
|
|
3301
|
+
* unresolvable rows.
|
|
3302
|
+
*/
|
|
3303
|
+
redriveUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
3304
|
+
/**
|
|
3305
|
+
* First `now()` a `pending` row's re-drive was observed `unresolved` (#965). A
|
|
3306
|
+
* `pending` row is invisible to every cron arm (all require `status =
|
|
3307
|
+
* 'processing'`), so an indefinitely-`unresolved` row would be stranded with
|
|
3308
|
+
* nothing driving it. Once `now - since >= pausedMaxWaitMs`, `resolveRedrive`
|
|
3309
|
+
* takes `dispatch` instead of `unresolved` (reusing the existing knob — see
|
|
3310
|
+
* ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
|
|
3311
|
+
*/
|
|
3312
|
+
redriveUnresolvedSince = /* @__PURE__ */ new Map();
|
|
3313
|
+
/**
|
|
3314
|
+
* Consecutive-identical-poll-failure streak for the re-drive fence (#1348),
|
|
3315
|
+
* keyed by Evident **message id** (not session) so `clearRedriveUnresolved`
|
|
3316
|
+
* can drop it with the other two trackers and it cannot leak. `sessionId` is
|
|
3317
|
+
* carried inside the entry, not the key: a session change is a different
|
|
3318
|
+
* situation and resets the streak, which gives the `(sessionId, message.id)`
|
|
3319
|
+
* pairing #1348 asks for without a composite map key.
|
|
3320
|
+
*/
|
|
3321
|
+
redrivePollFailures = /* @__PURE__ */ new Map();
|
|
3322
|
+
/**
|
|
3323
|
+
* "Already emitted `redrive_outcome_unreported` for THIS (message, outcome)
|
|
3324
|
+
* streak" (Class B, #1340: the runner DECIDED reattach/settle/fail_permanent
|
|
3325
|
+
* but its own PATCH to record it failed — distinct from Class A's
|
|
3326
|
+
* `redrive_poll_failed`, where opencode itself can't be observed). Keyed by
|
|
3327
|
+
* message id, valued by the outcome currently failing to report, so a
|
|
3328
|
+
* change of outcome starts a fresh signal. Cleared by
|
|
3329
|
+
* `clearRedriveUnresolved` the instant either PATCH succeeds.
|
|
3330
|
+
*/
|
|
3331
|
+
redriveOutcomeUnreportedSignalled = /* @__PURE__ */ new Map();
|
|
3332
|
+
/**
|
|
3333
|
+
* First `now()` a Class B outcome PATCH (reattach/settle/fail_permanent) was
|
|
3334
|
+
* observed to fail for this message (#1366's failure-window trip arm,
|
|
3335
|
+
* `boundRedriveOutcome`). Duration, not a tick count — bounded by the
|
|
3336
|
+
* existing `pausedMaxWaitMs` window (reusing the knob, not a new constant).
|
|
3337
|
+
* Cleared by `clearRedriveUnresolved` the instant the original PATCH
|
|
3338
|
+
* succeeds.
|
|
3339
|
+
*/
|
|
3340
|
+
redriveOutcomeFailingSince = /* @__PURE__ */ new Map();
|
|
3341
|
+
/**
|
|
3342
|
+
* "Already posted `redrive_outcome_abandoned` with `reported: false` for this
|
|
3343
|
+
* row" (#1366) — the bound tripped but the terminal `markFailed` fallback ALSO
|
|
3344
|
+
* failed (the route-level fault of G2), so every following tick re-attempts
|
|
3345
|
+
* the same terminal PATCH. Guards that quiet retry from re-signalling on
|
|
3346
|
+
* every tick. Cleared by `clearRedriveUnresolved`.
|
|
3347
|
+
*/
|
|
3348
|
+
redriveOutcomeAbandonedSignalled = /* @__PURE__ */ new Set();
|
|
3349
|
+
/**
|
|
3350
|
+
* "Already emitted `dispatch_not_started` for THIS (message, branch) streak"
|
|
3351
|
+
* (#1340). Valued by the branch currently firing, so a row that moves between
|
|
3352
|
+
* exits re-signals — the move IS the finding. Cleared only on a CONFIRMED
|
|
3353
|
+
* dispatch, never on the fence's decision to dispatch: `clearRedriveUnresolved`
|
|
3354
|
+
* runs on that decision (`resolveRedriveUnresolved`), so clearing there would
|
|
3355
|
+
* re-signal on every one of the 15h of re-dispatch attempts #1110 made.
|
|
3356
|
+
*/
|
|
3357
|
+
dispatchNotStartedSignalled = /* @__PURE__ */ new Map();
|
|
3358
|
+
/**
|
|
3359
|
+
* Consecutive-UNCONFIRMED-dispatch streak for a `pending` row with NO stored
|
|
3360
|
+
* `opencode_message_id` yet — i.e. one that has never even reached the
|
|
3361
|
+
* re-drive fence above. `sendPromptAsync`'s POST may 2xx, but its own
|
|
3362
|
+
* read-back retries can never confirm the assigned id when the session's
|
|
3363
|
+
* message list is PERMANENTLY unreadable (e.g. a corrupted local opencode
|
|
3364
|
+
* SQLite DB, #1345/#1348's exact fault, just hit BEFORE the row is ever
|
|
3365
|
+
* dispatched instead of after). Unlike an already-dispatched row, THIS row has
|
|
3366
|
+
* no other safety net at all: the lifecycle cron only reclaims `status =
|
|
3367
|
+
* 'processing'` rows, and a row stuck here never reaches `processing`. Keyed
|
|
3368
|
+
* by message id, carrying `sessionId` so a session change (a fresh one bound
|
|
3369
|
+
* after abandonment) starts a new streak rather than inheriting the old
|
|
3370
|
+
* session's count — same shape as `redrivePollFailures` above.
|
|
3371
|
+
*/
|
|
3372
|
+
unconfirmedDispatchFailures = /* @__PURE__ */ new Map();
|
|
2159
3373
|
/**
|
|
2160
3374
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
2161
3375
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -2211,6 +3425,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2211
3425
|
sessionTitles = /* @__PURE__ */ new Map();
|
|
2212
3426
|
/** Serialises drains so a reconnect during a drain doesn't double-process. */
|
|
2213
3427
|
draining = false;
|
|
3428
|
+
/**
|
|
3429
|
+
* Serialises runner-file syncs (#559) so the ~2s poll tick and a concurrent
|
|
3430
|
+
* drain ping don't download, write and ack the same file twice.
|
|
3431
|
+
*/
|
|
3432
|
+
syncingFiles = false;
|
|
3433
|
+
/**
|
|
3434
|
+
* Consecutive failed acks per pending file (#559). Lives on the driver so it
|
|
3435
|
+
* survives across drains — without it, a file whose ack keeps failing is
|
|
3436
|
+
* re-downloaded and re-written every ~2s until the server expires it.
|
|
3437
|
+
*/
|
|
3438
|
+
fileAckFailures = /* @__PURE__ */ new Map();
|
|
3439
|
+
/**
|
|
3440
|
+
* Monotonic count of files this runner has pulled and written (#559). Only
|
|
3441
|
+
* ever increases, so `run.ts` detects work by comparing it against the value
|
|
3442
|
+
* it saw on the previous cycle — including work that landed mid-sleep, the
|
|
3443
|
+
* same trick `lastProxiedActivityAt` uses.
|
|
3444
|
+
*/
|
|
3445
|
+
appliedFileCount = 0;
|
|
2214
3446
|
/**
|
|
2215
3447
|
* The currently-executing `drainPending()` promise, or null when idle. Lets a
|
|
2216
3448
|
* graceful shutdown (`waitForInFlight`) await an in-progress drain so a turn it
|
|
@@ -2226,21 +3458,24 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2226
3458
|
* and stops opencode.
|
|
2227
3459
|
*/
|
|
2228
3460
|
stopped = false;
|
|
2229
|
-
constructor(
|
|
2230
|
-
this.agentId =
|
|
2231
|
-
this.port =
|
|
2232
|
-
this.apiUrl =
|
|
2233
|
-
this.getAuthHeader =
|
|
2234
|
-
this.conversationFilter =
|
|
2235
|
-
this.retry = { ...DEFAULT_RETRY_POLICY, ...
|
|
2236
|
-
this.log =
|
|
3461
|
+
constructor(config) {
|
|
3462
|
+
this.agentId = config.agentId;
|
|
3463
|
+
this.port = config.port;
|
|
3464
|
+
this.apiUrl = config.apiUrl.replace(/\/$/, "");
|
|
3465
|
+
this.getAuthHeader = config.getAuthHeader;
|
|
3466
|
+
this.conversationFilter = config.conversationFilter ?? null;
|
|
3467
|
+
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3468
|
+
this.log = config.log ?? (() => {
|
|
2237
3469
|
});
|
|
2238
|
-
this.fetchImpl =
|
|
2239
|
-
this.sleep =
|
|
2240
|
-
this.pausedPollIntervalMs =
|
|
2241
|
-
this.pausedMaxWaitMs =
|
|
2242
|
-
this.stuckQueuedMs =
|
|
2243
|
-
this.now =
|
|
3470
|
+
this.fetchImpl = config.fetchImpl ?? fetch;
|
|
3471
|
+
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3472
|
+
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3473
|
+
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
3474
|
+
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
3475
|
+
this.now = config.now ?? (() => Date.now());
|
|
3476
|
+
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3477
|
+
this.homeDir = config.homeDir ?? homedir2();
|
|
3478
|
+
this.maxActiveSessions = config.maxActiveSessions;
|
|
2244
3479
|
}
|
|
2245
3480
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
2246
3481
|
get opencodeBase() {
|
|
@@ -2268,6 +3503,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2268
3503
|
);
|
|
2269
3504
|
return run2;
|
|
2270
3505
|
}
|
|
3506
|
+
/**
|
|
3507
|
+
* Pull-and-apply any files Evident has queued for this runner (#559), riding
|
|
3508
|
+
* the EXISTING drain cycle — `run.ts` calls it from the same ~2s channel poll
|
|
3509
|
+
* and drain ping that call `drainPending()`. There is deliberately no channel,
|
|
3510
|
+
* control frame or poll loop of its own: worst-case latency is one poll tick.
|
|
3511
|
+
*
|
|
3512
|
+
* NEVER throws and never surfaces a `ChannelAuthError`: a file failure must not
|
|
3513
|
+
* cost a conversation turn. Failures are logged and either acked as a terminal
|
|
3514
|
+
* outcome or left pending for the next drain (see `runner-file-sync.ts`).
|
|
3515
|
+
*
|
|
3516
|
+
* Re-entrant calls are skipped (the poll tick and a drain ping can overlap).
|
|
3517
|
+
*
|
|
3518
|
+
* @returns the number of files written to disk.
|
|
3519
|
+
*/
|
|
3520
|
+
async syncPendingFiles() {
|
|
3521
|
+
if (this.stopped) return 0;
|
|
3522
|
+
if (this.syncingFiles) return 0;
|
|
3523
|
+
this.syncingFiles = true;
|
|
3524
|
+
try {
|
|
3525
|
+
const applied = await syncPendingRunnerFiles({
|
|
3526
|
+
agentId: this.agentId,
|
|
3527
|
+
apiUrl: this.apiUrl,
|
|
3528
|
+
getAuthHeader: this.getAuthHeader,
|
|
3529
|
+
fetchImpl: this.fetchImpl,
|
|
3530
|
+
allowedDirectories: this.fileSyncDirectories,
|
|
3531
|
+
homeDir: this.homeDir,
|
|
3532
|
+
ackFailures: this.fileAckFailures,
|
|
3533
|
+
log: this.log
|
|
3534
|
+
});
|
|
3535
|
+
this.appliedFileCount += applied;
|
|
3536
|
+
return applied;
|
|
3537
|
+
} catch (err) {
|
|
3538
|
+
this.log({
|
|
3539
|
+
level: "error",
|
|
3540
|
+
message: `Runner file sync failed unexpectedly (message processing is unaffected): ${err instanceof Error ? err.message : String(err)}`
|
|
3541
|
+
});
|
|
3542
|
+
return 0;
|
|
3543
|
+
} finally {
|
|
3544
|
+
this.syncingFiles = false;
|
|
3545
|
+
}
|
|
3546
|
+
}
|
|
2271
3547
|
async runDrain() {
|
|
2272
3548
|
let dispatched = 0;
|
|
2273
3549
|
try {
|
|
@@ -2279,10 +3555,26 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2279
3555
|
message: `Found ${total} pending message(s) across ${conversations.length} conversation(s) \u2014 draining`
|
|
2280
3556
|
});
|
|
2281
3557
|
}
|
|
3558
|
+
let cappedSkips = 0;
|
|
2282
3559
|
for (const conv of conversations) {
|
|
2283
3560
|
if (this.stopped) break;
|
|
3561
|
+
if (this.maxActiveSessions !== void 0) {
|
|
3562
|
+
const activeSessionIds = this.activeSessionIdsForCap();
|
|
3563
|
+
const resolvedSessionId = this.sessions.get(conv.id) ?? conv.opencode_session_id;
|
|
3564
|
+
const alreadyActive = resolvedSessionId != null && activeSessionIds.has(resolvedSessionId);
|
|
3565
|
+
if (activeSessionIds.size >= this.maxActiveSessions && !alreadyActive) {
|
|
3566
|
+
cappedSkips++;
|
|
3567
|
+
continue;
|
|
3568
|
+
}
|
|
3569
|
+
}
|
|
2284
3570
|
dispatched += await this.processConversation(conv);
|
|
2285
3571
|
}
|
|
3572
|
+
if (cappedSkips > 0) {
|
|
3573
|
+
this.log({
|
|
3574
|
+
level: "warn",
|
|
3575
|
+
message: `max-active-sessions cap (${this.maxActiveSessions}) reached \u2014 skipped ${cappedSkips} pending conversation(s) this tick`
|
|
3576
|
+
});
|
|
3577
|
+
}
|
|
2286
3578
|
await this.readoptProcessing();
|
|
2287
3579
|
} finally {
|
|
2288
3580
|
this.draining = false;
|
|
@@ -2301,6 +3593,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2301
3593
|
}
|
|
2302
3594
|
return false;
|
|
2303
3595
|
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Session ids active *for the `--max-active-sessions` cap*: in-flight work AND
|
|
3598
|
+
* a live watcher loop. Unlike `hasInFlightWatchers()` / `protectedSessionIds()`,
|
|
3599
|
+
* a ZOMBIE watcher (in-flight but `loop === null`, left by a non-auth failure
|
|
3600
|
+
* inside `runWatcherLoop`) does not count here — under a cap it would
|
|
3601
|
+
* permanently consume a slot, whereas cleanup/idle-exit should still treat it
|
|
3602
|
+
* as protected. One call per drain iteration serves both the cap check
|
|
3603
|
+
* (`.size`) and the already-active exemption (`.has`).
|
|
3604
|
+
*/
|
|
3605
|
+
activeSessionIdsForCap() {
|
|
3606
|
+
const ids = /* @__PURE__ */ new Set();
|
|
3607
|
+
for (const [sessionId, watcher] of this.watchers) {
|
|
3608
|
+
if (watcher.inFlight.size > 0 && watcher.loop !== null) ids.add(sessionId);
|
|
3609
|
+
}
|
|
3610
|
+
return ids;
|
|
3611
|
+
}
|
|
3612
|
+
/**
|
|
3613
|
+
* File-pull work, for `run.ts`'s idle accounting (#559).
|
|
3614
|
+
*
|
|
3615
|
+
* Pulling a file is real work that `drainPending()` knows nothing about, so
|
|
3616
|
+
* without this a near-idle runner counts a credential pull as an empty tick
|
|
3617
|
+
* and `--idle-timeout` can `process.exit` mid-pull — leaving a
|
|
3618
|
+
* `.evident-push-*.tmp` behind — or immediately after the write, before the
|
|
3619
|
+
* browser has run the authorize/callback that activates it (the user then sees
|
|
3620
|
+
* `saved_not_activated` for a runner that was fine).
|
|
3621
|
+
*
|
|
3622
|
+
* Two signals because one cannot cover both cases: `inFlight` is the pull
|
|
3623
|
+
* happening RIGHT NOW (it may outlive the tick that started it), and
|
|
3624
|
+
* `appliedFiles` is monotonic so a pull that started AND finished between two
|
|
3625
|
+
* idle checks still shows up as an advance.
|
|
3626
|
+
*
|
|
3627
|
+
* CALLER CONTRACT: sample `inFlight` BEFORE calling `syncPendingFiles()` for
|
|
3628
|
+
* the cycle. `syncPendingFiles` sets the flag synchronously, so a caller that
|
|
3629
|
+
* samples afterwards reads `true` every single cycle and can never idle out.
|
|
3630
|
+
*/
|
|
3631
|
+
fileSyncActivity() {
|
|
3632
|
+
return { appliedFiles: this.appliedFileCount, inFlight: this.syncingFiles };
|
|
3633
|
+
}
|
|
2304
3634
|
/**
|
|
2305
3635
|
* OpenCode session ids the session-cleanup sweep (issue #190) must NOT delete:
|
|
2306
3636
|
* exactly those with a live (dispatched-but-not-done / paused) turn, i.e. a
|
|
@@ -2362,7 +3692,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2362
3692
|
await this.sleep(step);
|
|
2363
3693
|
}
|
|
2364
3694
|
}
|
|
2365
|
-
while (this.hasInFlightWatchers()) {
|
|
3695
|
+
while (this.hasInFlightWatchers() || this.syncingFiles) {
|
|
2366
3696
|
if (this.now() >= deadline) return false;
|
|
2367
3697
|
await this.sleep(step);
|
|
2368
3698
|
}
|
|
@@ -2397,16 +3727,30 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2397
3727
|
* @returns the count of messages NEWLY dispatched (not already in-flight).
|
|
2398
3728
|
*/
|
|
2399
3729
|
async processConversation(conv) {
|
|
2400
|
-
const sessionId = await this.ensureSession(conv);
|
|
3730
|
+
const { sessionId, refusedSessionId, created: sessionCreated } = await this.ensureSession(conv);
|
|
2401
3731
|
const messages = await this.getPendingMessages(conv.id);
|
|
2402
3732
|
let dispatched = 0;
|
|
2403
3733
|
let skippedAlreadyDispatched = 0;
|
|
3734
|
+
if (refusedSessionId && messages.length > 0) {
|
|
3735
|
+
void this.postSignal(conv.id, messages[0].id, "session_superseded", {
|
|
3736
|
+
superseded_session_id: refusedSessionId
|
|
3737
|
+
});
|
|
3738
|
+
}
|
|
2404
3739
|
for (const message of messages) {
|
|
2405
3740
|
if (this.stopped) break;
|
|
2406
3741
|
if (this.dispatched.has(message.id)) {
|
|
2407
3742
|
skippedAlreadyDispatched += 1;
|
|
2408
3743
|
continue;
|
|
2409
3744
|
}
|
|
3745
|
+
if (message.opencode_message_id) {
|
|
3746
|
+
const outcome = await this.resolveRedrive(conv, sessionId, message, sessionCreated);
|
|
3747
|
+
if (outcome === "abandoned") {
|
|
3748
|
+
continue;
|
|
3749
|
+
}
|
|
3750
|
+
if (outcome !== "dispatch") {
|
|
3751
|
+
break;
|
|
3752
|
+
}
|
|
3753
|
+
}
|
|
2410
3754
|
const options = {
|
|
2411
3755
|
agent: message.opencode_agent ?? void 0,
|
|
2412
3756
|
model: message.opencode_model ?? void 0
|
|
@@ -2427,7 +3771,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2427
3771
|
} catch (err) {
|
|
2428
3772
|
if (err instanceof ChannelAuthError) throw err;
|
|
2429
3773
|
this.dispatched.delete(message.id);
|
|
2430
|
-
|
|
3774
|
+
const exists = await sessionExists(this.port, sessionId);
|
|
3775
|
+
if (exists === false) {
|
|
2431
3776
|
this.sessions.delete(conv.id);
|
|
2432
3777
|
this.log({
|
|
2433
3778
|
level: "warn",
|
|
@@ -2435,27 +3780,80 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2435
3780
|
conversation_id: conv.id,
|
|
2436
3781
|
message_id: message.id
|
|
2437
3782
|
});
|
|
3783
|
+
this.signalDispatchNotStarted(conv, message, "session_deleted_race");
|
|
2438
3784
|
break;
|
|
2439
3785
|
}
|
|
2440
|
-
|
|
3786
|
+
if (exists === null) {
|
|
3787
|
+
this.log({
|
|
3788
|
+
level: "warn",
|
|
3789
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed and session (${sessionId.slice(0, 8)}) existence could not be confirmed (opencode momentarily unreachable) \u2014 deferring this and later messages for conversation ${conv.id.slice(0, 8)} to the next tick rather than treating it as a genuine failure.`,
|
|
3790
|
+
conversation_id: conv.id,
|
|
3791
|
+
message_id: message.id
|
|
3792
|
+
});
|
|
3793
|
+
this.signalDispatchNotStarted(conv, message, "session_existence_unknown");
|
|
3794
|
+
break;
|
|
3795
|
+
}
|
|
3796
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
3797
|
+
this.sessions.delete(conv.id);
|
|
3798
|
+
this.supersede(conv.id, sessionId);
|
|
3799
|
+
this.log({
|
|
3800
|
+
level: "warn",
|
|
3801
|
+
message: `Abandoning OpenCode session ${sessionId.slice(0, 8)} as the binding for conversation ${conv.id.slice(0, 8)} (it exists but failed to run a turn) \u2014 a fresh session is created on the next tick, whatever the persisted binding says by then.`,
|
|
3802
|
+
conversation_id: conv.id,
|
|
3803
|
+
message_id: message.id
|
|
3804
|
+
});
|
|
3805
|
+
await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
|
|
3806
|
+
this.log({
|
|
3807
|
+
level: "warn",
|
|
3808
|
+
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
3809
|
+
conversation_id: conv.id,
|
|
3810
|
+
message_id: message.id
|
|
3811
|
+
});
|
|
3812
|
+
this.signalDispatchNotStarted(conv, message, "failure_unreported");
|
|
2441
3813
|
});
|
|
2442
3814
|
this.log({
|
|
2443
3815
|
level: "error",
|
|
2444
|
-
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${
|
|
3816
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage}`,
|
|
2445
3817
|
conversation_id: conv.id,
|
|
2446
3818
|
message_id: message.id
|
|
2447
3819
|
});
|
|
2448
|
-
|
|
3820
|
+
break;
|
|
2449
3821
|
}
|
|
2450
3822
|
if (opencodeMessageId === null) {
|
|
3823
|
+
const streak = this.recordUnconfirmedDispatch(message.id, sessionId);
|
|
3824
|
+
if (streak < MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
3825
|
+
this.log({
|
|
3826
|
+
level: "warn",
|
|
3827
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next tick`,
|
|
3828
|
+
conversation_id: conv.id,
|
|
3829
|
+
message_id: message.id
|
|
3830
|
+
});
|
|
3831
|
+
this.signalDispatchNotStarted(conv, message, "readback_unconfirmed");
|
|
3832
|
+
continue;
|
|
3833
|
+
}
|
|
3834
|
+
this.unconfirmedDispatchFailures.delete(message.id);
|
|
3835
|
+
this.sessions.delete(conv.id);
|
|
3836
|
+
this.supersede(conv.id, sessionId);
|
|
3837
|
+
const errorMessage = `OpenCode accepted this message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
2451
3838
|
this.log({
|
|
2452
|
-
level: "
|
|
2453
|
-
message:
|
|
3839
|
+
level: "error",
|
|
3840
|
+
message: errorMessage,
|
|
2454
3841
|
conversation_id: conv.id,
|
|
2455
3842
|
message_id: message.id
|
|
2456
3843
|
});
|
|
2457
|
-
|
|
3844
|
+
await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
|
|
3845
|
+
this.log({
|
|
3846
|
+
level: "warn",
|
|
3847
|
+
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
3848
|
+
conversation_id: conv.id,
|
|
3849
|
+
message_id: message.id
|
|
3850
|
+
});
|
|
3851
|
+
this.signalDispatchNotStarted(conv, message, "abandon_unreported");
|
|
3852
|
+
});
|
|
3853
|
+
break;
|
|
2458
3854
|
}
|
|
3855
|
+
this.unconfirmedDispatchFailures.delete(message.id);
|
|
3856
|
+
this.dispatchNotStartedSignalled.delete(message.id);
|
|
2459
3857
|
this.dispatched.add(message.id);
|
|
2460
3858
|
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
2461
3859
|
dispatched += 1;
|
|
@@ -2464,15 +3862,505 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2464
3862
|
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
2465
3863
|
this.log({
|
|
2466
3864
|
level: "warn",
|
|
2467
|
-
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
2468
|
-
conversation_id: conv.id
|
|
3865
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
3866
|
+
conversation_id: conv.id
|
|
3867
|
+
});
|
|
3868
|
+
}
|
|
3869
|
+
this.ensureWatcherRunning(sessionId);
|
|
3870
|
+
return dispatched;
|
|
3871
|
+
}
|
|
3872
|
+
/**
|
|
3873
|
+
* Poll a session's message list for the re-drive fence (#965), via the
|
|
3874
|
+
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
3875
|
+
* hits the global `fetch` and would bypass the same override every other
|
|
3876
|
+
* opencode poll in this file respects. Mirrors `readoptProcessing`'s own
|
|
3877
|
+
* snapshot fetch (`:3081-3111`).
|
|
3878
|
+
*
|
|
3879
|
+
* Returns `{ ok: true, messages }` on a readable snapshot, or
|
|
3880
|
+
* `{ ok: false, signature }` on failure — `signature` is a string that
|
|
3881
|
+
* repeats across attempts for the SAME underlying fault (used by the
|
|
3882
|
+
* consecutive-identical-failure bound, #1348), or `null` for a thrown
|
|
3883
|
+
* exception, which is NOT countable toward that bound (a network blip / an
|
|
3884
|
+
* opencode restart also throws identically every tick, and must keep
|
|
3885
|
+
* retrying unbounded rather than ever being treated as permanent).
|
|
3886
|
+
*/
|
|
3887
|
+
async pollSessionMessagesForRedrive(conv, message, sessionId) {
|
|
3888
|
+
try {
|
|
3889
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
3890
|
+
if (!res.ok) {
|
|
3891
|
+
const rawBody = await res.text();
|
|
3892
|
+
const normalized = normalizeRedrivePollFailureBody(rawBody);
|
|
3893
|
+
this.log({
|
|
3894
|
+
level: "warn",
|
|
3895
|
+
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status}${normalized ? `: ${normalized}` : ""} \u2014 treating as unreadable this tick`,
|
|
3896
|
+
conversation_id: conv.id,
|
|
3897
|
+
message_id: message.id
|
|
3898
|
+
});
|
|
3899
|
+
return { ok: false, signature: `HTTP ${res.status}${normalized ? `: ${normalized}` : ""}` };
|
|
3900
|
+
}
|
|
3901
|
+
const body = await res.json();
|
|
3902
|
+
if (!Array.isArray(body)) {
|
|
3903
|
+
this.log({
|
|
3904
|
+
level: "warn",
|
|
3905
|
+
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned a non-array message body \u2014 treating as unreadable this tick`,
|
|
3906
|
+
conversation_id: conv.id,
|
|
3907
|
+
message_id: message.id
|
|
3908
|
+
});
|
|
3909
|
+
return { ok: false, signature: "non-array message body" };
|
|
3910
|
+
}
|
|
3911
|
+
return { ok: true, messages: body };
|
|
3912
|
+
} catch (err) {
|
|
3913
|
+
this.log({
|
|
3914
|
+
level: "warn",
|
|
3915
|
+
message: `Re-drive: failed to poll session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3916
|
+
conversation_id: conv.id,
|
|
3917
|
+
message_id: message.id
|
|
3918
|
+
});
|
|
3919
|
+
return { ok: false, signature: null };
|
|
3920
|
+
}
|
|
3921
|
+
}
|
|
3922
|
+
/**
|
|
3923
|
+
* The re-drive fence for a `pending` row that already carries a stored
|
|
3924
|
+
* `opencode_message_id` (#965) — i.e. it has already been handed to opencode at
|
|
3925
|
+
* least once (see the invariant at `QueuedMessage.opencode_message_id`'s doc).
|
|
3926
|
+
* The lifecycle cron can falsely reclaim a `processing` row back to `pending`
|
|
3927
|
+
* mid-turn (a 5-minute liveness-staleness check racing a still-running turn);
|
|
3928
|
+
* without this fence the drain loop would re-`prompt_async` the SAME turn a
|
|
3929
|
+
* second time against live GitHub state. Mirrors `readoptOne`'s job for the
|
|
3930
|
+
* `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
|
|
3931
|
+
* needed here because `sessionCreated` already handles the cases (a #553
|
|
3932
|
+
* abandoned session, a #190 vanished one) that path exists for.
|
|
3933
|
+
*
|
|
3934
|
+
* Only `ChannelAuthError` propagates. A poll that fails identically
|
|
3935
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row reports the message
|
|
3936
|
+
* failed instead of retrying it (#1348) — SEPARATE from, not a replacement
|
|
3937
|
+
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
3938
|
+
* other failure resolves to `unresolved` and is retried whole on the next
|
|
3939
|
+
* ~2s drain tick.
|
|
3940
|
+
*/
|
|
3941
|
+
async resolveRedrive(conv, sessionId, message, sessionCreated) {
|
|
3942
|
+
const ocId = message.opencode_message_id ?? null;
|
|
3943
|
+
if (sessionCreated) {
|
|
3944
|
+
this.clearRedriveUnresolved(message.id);
|
|
3945
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3946
|
+
return "dispatch";
|
|
3947
|
+
}
|
|
3948
|
+
const polled = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
|
|
3949
|
+
if (!polled.ok) {
|
|
3950
|
+
const streak = this.recordRedrivePollFailure(message.id, sessionId, polled.signature);
|
|
3951
|
+
if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES && polled.signature !== null) {
|
|
3952
|
+
return this.failRedrivePollPermanent(conv, sessionId, message, polled.signature, streak);
|
|
3953
|
+
}
|
|
3954
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3955
|
+
}
|
|
3956
|
+
this.redrivePollFailures.delete(message.id);
|
|
3957
|
+
const messages = polled.messages;
|
|
3958
|
+
if (messages.length === 0) {
|
|
3959
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3960
|
+
}
|
|
3961
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
3962
|
+
if (state === "failed" && isAbortedTerminalReply(messages, ocId ?? "")) {
|
|
3963
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
3964
|
+
if (ongoing === false) {
|
|
3965
|
+
this.log({
|
|
3966
|
+
level: "info",
|
|
3967
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
3968
|
+
conversation_id: conv.id,
|
|
3969
|
+
message_id: message.id
|
|
3970
|
+
});
|
|
3971
|
+
this.clearRedriveUnresolved(message.id);
|
|
3972
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3973
|
+
return "dispatch";
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
if (state === "done" || state === "failed") {
|
|
3977
|
+
return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
|
|
3978
|
+
}
|
|
3979
|
+
if (state === "running" || state === "queued") {
|
|
3980
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
3981
|
+
if (ongoing === true) {
|
|
3982
|
+
return this.reattachRedrive(conv, sessionId, message, ocId);
|
|
3983
|
+
}
|
|
3984
|
+
if (ongoing === false) {
|
|
3985
|
+
this.clearRedriveUnresolved(message.id);
|
|
3986
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3987
|
+
return "dispatch";
|
|
3988
|
+
}
|
|
3989
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3990
|
+
}
|
|
3991
|
+
this.clearRedriveUnresolved(message.id);
|
|
3992
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3993
|
+
return "dispatch";
|
|
3994
|
+
}
|
|
3995
|
+
/**
|
|
3996
|
+
* The `reattached` outcome (Task 3.3): the prior turn is STILL ONGOING per
|
|
3997
|
+
* opencode's own status map — undo the false reclaim instead of starting a
|
|
3998
|
+
* second turn.
|
|
3999
|
+
*/
|
|
4000
|
+
async reattachRedrive(conv, sessionId, message, ocId) {
|
|
4001
|
+
let anchorMs;
|
|
4002
|
+
const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
|
|
4003
|
+
if (!Number.isNaN(parsed)) {
|
|
4004
|
+
anchorMs = parsed;
|
|
4005
|
+
} else {
|
|
4006
|
+
anchorMs = this.now();
|
|
4007
|
+
this.log({
|
|
4008
|
+
level: "error",
|
|
4009
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} has null/unparseable processing_started_at (${String(message.processing_started_at)}) \u2014 anchoring the watcher's absolute-age ceiling to now (defensive)`,
|
|
4010
|
+
conversation_id: conv.id,
|
|
4011
|
+
message_id: message.id
|
|
4012
|
+
});
|
|
4013
|
+
}
|
|
4014
|
+
const title = await this.resolveSessionTitle(sessionId, conv.id);
|
|
4015
|
+
try {
|
|
4016
|
+
await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
|
|
4017
|
+
} catch (err) {
|
|
4018
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4019
|
+
if (err instanceof ChannelTerminalError) {
|
|
4020
|
+
this.log({
|
|
4021
|
+
level: "error",
|
|
4022
|
+
message: `Re-drive: the server definitively refused to restore message ${message.id.slice(0, 8)} to processing (terminal HTTP ${err.status} \u2014 the row is gone or the update was rejected); NOT reporting a re-attach`,
|
|
4023
|
+
conversation_id: conv.id,
|
|
4024
|
+
message_id: message.id
|
|
4025
|
+
});
|
|
4026
|
+
} else {
|
|
4027
|
+
this.log({
|
|
4028
|
+
level: "warn",
|
|
4029
|
+
message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4030
|
+
conversation_id: conv.id,
|
|
4031
|
+
message_id: message.id
|
|
4032
|
+
});
|
|
4033
|
+
}
|
|
4034
|
+
const bound = await this.boundRedriveOutcome(conv, message, "reattach");
|
|
4035
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
4036
|
+
}
|
|
4037
|
+
this.clearRedriveUnresolved(message.id);
|
|
4038
|
+
this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
|
|
4039
|
+
this.dispatched.add(message.id);
|
|
4040
|
+
this.readopted.add(message.id);
|
|
4041
|
+
this.ensureWatcherRunning(sessionId);
|
|
4042
|
+
const watchedForMs = this.now() - anchorMs;
|
|
4043
|
+
void this.postSignal(conv.id, message.id, "redrive_reattached", {
|
|
4044
|
+
watched_for_ms: watchedForMs
|
|
4045
|
+
});
|
|
4046
|
+
this.log({
|
|
4047
|
+
level: "warn",
|
|
4048
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) was wrongly reclaimed to pending while its turn was still running (watched ${watchedForMs}ms) \u2014 restored to processing instead of re-dispatching`,
|
|
4049
|
+
conversation_id: conv.id,
|
|
4050
|
+
message_id: message.id
|
|
4051
|
+
});
|
|
4052
|
+
return "reattached";
|
|
4053
|
+
}
|
|
4054
|
+
/**
|
|
4055
|
+
* The `settled` outcome (Task 3.2): the prior turn already finished (or
|
|
4056
|
+
* errored) while nobody was watching — deliver/report it instead of re-running.
|
|
4057
|
+
* Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
|
|
4058
|
+
* (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
|
|
4059
|
+
* drain, same as any other non-auth failure). The restart-abort carve-out that
|
|
4060
|
+
* keeps the two in step for `failed` lives in the caller (`resolveRedrive`, #1310),
|
|
4061
|
+
* so a row reaching this `failed` branch is a GENUINE failure.
|
|
4062
|
+
*/
|
|
4063
|
+
async settleRedrive(conv, sessionId, message, ocId, messages, state) {
|
|
4064
|
+
try {
|
|
4065
|
+
if (state === "done") {
|
|
4066
|
+
const title = await this.resolveSessionTitle(sessionId, conv.id);
|
|
4067
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
4068
|
+
this.log({
|
|
4069
|
+
level: "info",
|
|
4070
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
|
|
4071
|
+
conversation_id: conv.id,
|
|
4072
|
+
message_id: message.id
|
|
4073
|
+
});
|
|
4074
|
+
await this.markDone(conv.id, message.id, sessionId, ocId, title, usage);
|
|
4075
|
+
} else {
|
|
4076
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
4077
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
4078
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
4079
|
+
this.log({
|
|
4080
|
+
level: "error",
|
|
4081
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} errored while its row was wrongly reclaimed to pending \u2014 marking failed instead of re-dispatching: ${error2 ?? "(no error text)"}`,
|
|
4082
|
+
conversation_id: conv.id,
|
|
4083
|
+
message_id: message.id
|
|
4084
|
+
});
|
|
4085
|
+
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
4086
|
+
}
|
|
4087
|
+
} catch (err) {
|
|
4088
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4089
|
+
this.log({
|
|
4090
|
+
level: "warn",
|
|
4091
|
+
message: `Re-drive: failed to report message ${message.id.slice(0, 8)} ${state} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4092
|
+
conversation_id: conv.id,
|
|
4093
|
+
message_id: message.id
|
|
4094
|
+
});
|
|
4095
|
+
const bound = await this.boundRedriveOutcome(conv, message, "settle");
|
|
4096
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
4097
|
+
}
|
|
4098
|
+
this.clearRedriveUnresolved(message.id);
|
|
4099
|
+
void this.postSignal(conv.id, message.id, "redrive_settled");
|
|
4100
|
+
return "settled";
|
|
4101
|
+
}
|
|
4102
|
+
/**
|
|
4103
|
+
* The permanent-failure outcome (#1348): the fence's own poll of this session
|
|
4104
|
+
* failed with the SAME opencode-answered signature
|
|
4105
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row — a transient blip
|
|
4106
|
+
* would have varied or eventually cleared (see `pollSessionMessagesForRedrive`
|
|
4107
|
+
* and `recordRedrivePollFailure`), so this is a durable fault (e.g. #1345's
|
|
4108
|
+
* corrupted opencode session) rather than something worth retrying forever.
|
|
4109
|
+
* Mirrors `settleRedrive`'s error discipline: no `usage`/`failure` args to
|
|
4110
|
+
* `markFailed` (no opencode snapshot to extract them from — this poll never
|
|
4111
|
+
* got a readable one).
|
|
4112
|
+
*/
|
|
4113
|
+
async failRedrivePollPermanent(conv, sessionId, message, signature, streak) {
|
|
4114
|
+
this.log({
|
|
4115
|
+
level: "error",
|
|
4116
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) failed to poll with the identical signature "${signature}" ${streak} times in a row \u2014 reporting the message failed instead of retrying forever`,
|
|
4117
|
+
conversation_id: conv.id,
|
|
4118
|
+
message_id: message.id
|
|
4119
|
+
});
|
|
4120
|
+
try {
|
|
4121
|
+
await this.markFailed(
|
|
4122
|
+
conv.id,
|
|
4123
|
+
message.id,
|
|
4124
|
+
sessionId,
|
|
4125
|
+
`The runner could not read this conversation's state from OpenCode (${signature}). The same failure repeated ${streak} times in a row, so the message was not retried further.`
|
|
4126
|
+
);
|
|
4127
|
+
} catch (err) {
|
|
4128
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4129
|
+
this.log({
|
|
4130
|
+
level: "warn",
|
|
4131
|
+
message: `Re-drive: failed to report message ${message.id.slice(0, 8)} permanently failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4132
|
+
conversation_id: conv.id,
|
|
4133
|
+
message_id: message.id
|
|
4134
|
+
});
|
|
4135
|
+
const bound = await this.boundRedriveOutcome(conv, message, "fail_permanent");
|
|
4136
|
+
return bound === "abandoned" ? "abandoned" : "unresolved";
|
|
4137
|
+
}
|
|
4138
|
+
this.clearRedriveUnresolved(message.id);
|
|
4139
|
+
void this.postSignal(conv.id, message.id, "redrive_poll_failed");
|
|
4140
|
+
return "settled";
|
|
4141
|
+
}
|
|
4142
|
+
/**
|
|
4143
|
+
* The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
|
|
4144
|
+
* observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
|
|
4145
|
+
* A `pending` row is swept by the server's own `PENDING_MAX_AGE_MS` (24h,
|
|
4146
|
+
* #1368) cron arm, but that is a day-scale backstop — this local bound acts
|
|
4147
|
+
* in minutes so the row (and the conversation it starves, per the ordering
|
|
4148
|
+
* invariant below) isn't left stranded for that long. Bound to the existing
|
|
4149
|
+
* `pausedMaxWaitMs` window (reusing the knob, not a new constant); takes
|
|
4150
|
+
* `dispatch` once elapsed.
|
|
4151
|
+
*/
|
|
4152
|
+
resolveRedriveUnresolved(conv, message) {
|
|
4153
|
+
const now = this.now();
|
|
4154
|
+
const since = this.redriveUnresolvedSince.get(message.id);
|
|
4155
|
+
if (since !== void 0 && now - since >= this.pausedMaxWaitMs) {
|
|
4156
|
+
this.clearRedriveUnresolved(message.id);
|
|
4157
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
4158
|
+
return "dispatch";
|
|
4159
|
+
}
|
|
4160
|
+
if (since === void 0) {
|
|
4161
|
+
this.redriveUnresolvedSince.set(message.id, now);
|
|
4162
|
+
}
|
|
4163
|
+
if (!this.redriveUnresolvedSignalled.has(message.id)) {
|
|
4164
|
+
this.redriveUnresolvedSignalled.add(message.id);
|
|
4165
|
+
void this.postSignal(conv.id, message.id, "redrive_unresolved");
|
|
4166
|
+
}
|
|
4167
|
+
return "unresolved";
|
|
4168
|
+
}
|
|
4169
|
+
/** Clear all `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
|
|
4170
|
+
clearRedriveUnresolved(messageId) {
|
|
4171
|
+
this.redriveUnresolvedSince.delete(messageId);
|
|
4172
|
+
this.redriveUnresolvedSignalled.delete(messageId);
|
|
4173
|
+
this.redrivePollFailures.delete(messageId);
|
|
4174
|
+
this.redriveOutcomeUnreportedSignalled.delete(messageId);
|
|
4175
|
+
this.redriveOutcomeFailingSince.delete(messageId);
|
|
4176
|
+
this.redriveOutcomeAbandonedSignalled.delete(messageId);
|
|
4177
|
+
}
|
|
4178
|
+
/**
|
|
4179
|
+
* #1340: the dispatch loop reached a message and did NOT start a turn. Fires at
|
|
4180
|
+
* most once per (message, branch) streak — a wedged row is re-tried every tick,
|
|
4181
|
+
* and the per-tick count is already carried by the co-occurring
|
|
4182
|
+
* `redrive_unresolved`/`redrive_redispatched` signals.
|
|
4183
|
+
*/
|
|
4184
|
+
signalDispatchNotStarted(conv, message, branch) {
|
|
4185
|
+
if (this.dispatchNotStartedSignalled.get(message.id) === branch) return;
|
|
4186
|
+
this.dispatchNotStartedSignalled.set(message.id, branch);
|
|
4187
|
+
void this.postSignal(conv.id, message.id, "dispatch_not_started", { branch });
|
|
4188
|
+
}
|
|
4189
|
+
/**
|
|
4190
|
+
* Class B (#1340): the runner DECIDED an outcome (reattach/settle/fail_permanent)
|
|
4191
|
+
* but its own PATCH to record it failed. Fires at most once per (message,
|
|
4192
|
+
* outcome) streak, and only while `boundRedriveOutcome` has not yet tripped —
|
|
4193
|
+
* once it trips, `redrive_outcome_abandoned` takes over reporting for the row
|
|
4194
|
+
* (#1366).
|
|
4195
|
+
*/
|
|
4196
|
+
signalRedriveOutcomeUnreported(conv, message, outcome) {
|
|
4197
|
+
if (this.redriveOutcomeUnreportedSignalled.get(message.id) === outcome) return;
|
|
4198
|
+
this.redriveOutcomeUnreportedSignalled.set(message.id, outcome);
|
|
4199
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_unreported", {
|
|
4200
|
+
attempted_outcome: outcome
|
|
4201
|
+
});
|
|
4202
|
+
}
|
|
4203
|
+
/**
|
|
4204
|
+
* The runner-authored, honest error text for the terminal fallback a tripped
|
|
4205
|
+
* `boundRedriveOutcome` sends. Distinguishable per outcome and truthful about
|
|
4206
|
+
* what actually happened — the `settle`/done case must say the turn finished
|
|
4207
|
+
* but its result could not be recorded, never that the runner stopped
|
|
4208
|
+
* responding (that would be a lie for this shape, see #1366's "why this ships").
|
|
4209
|
+
*/
|
|
4210
|
+
static REDRIVE_ABANDON_ERROR = {
|
|
4211
|
+
reattach: "your runner could not record that this message had started, so it was given up on",
|
|
4212
|
+
settle: "your runner finished this message but could not record the result, so the reply could not be delivered",
|
|
4213
|
+
fail_permanent: "the runner could not read this conversation's state from OpenCode, and could not record that failure either, so the message was given up on"
|
|
4214
|
+
};
|
|
4215
|
+
/**
|
|
4216
|
+
* Bound for Class B (#1340, #1366): the runner DECIDED an outcome but its own
|
|
4217
|
+
* PATCH to record it failed. Two independent trip arms (either sufficient):
|
|
4218
|
+
* (1) this failure streak has lasted `pausedMaxWaitMs` — DURATION, not a tick
|
|
4219
|
+
* count, reusing the knob `resolveRedriveUnresolved` already established; (2)
|
|
4220
|
+
* the turn's `processing_started_at` age has crossed
|
|
4221
|
+
* `ABSOLUTE_MAX_PROCESSING_MS` — durable and restart-surviving, since arm (1)'s
|
|
4222
|
+
* in-memory streak resets on a scale-to-zero restart.
|
|
4223
|
+
*
|
|
4224
|
+
* INVARIANT — a tripped bound never suppresses the original outcome attempt;
|
|
4225
|
+
* it only adds a fallback after that attempt has failed again. This is only
|
|
4226
|
+
* ever reached from inside the catch of the ORIGINAL outcome PATCH, which is
|
|
4227
|
+
* attempted first on every tick whether or not this bound tripped before —
|
|
4228
|
+
* there is no give-up latch that would short-circuit it. That is what lets a
|
|
4229
|
+
* route-level fault that heals later still deliver the turn's real
|
|
4230
|
+
* `done`/`failed` payload: once the original PATCH succeeds again, this
|
|
4231
|
+
* helper is never entered and the row settles with its real result.
|
|
4232
|
+
*/
|
|
4233
|
+
async boundRedriveOutcome(conv, message, outcome) {
|
|
4234
|
+
const now = this.now();
|
|
4235
|
+
const since = this.redriveOutcomeFailingSince.get(message.id);
|
|
4236
|
+
if (since === void 0) this.redriveOutcomeFailingSince.set(message.id, now);
|
|
4237
|
+
const durationTripped = now - (since ?? now) >= this.pausedMaxWaitMs;
|
|
4238
|
+
const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
|
|
4239
|
+
const absoluteAgeTripped = !Number.isNaN(parsed) && now - parsed >= ABSOLUTE_MAX_PROCESSING_MS;
|
|
4240
|
+
if (!durationTripped && !absoluteAgeTripped) {
|
|
4241
|
+
this.signalRedriveOutcomeUnreported(conv, message, outcome);
|
|
4242
|
+
return "retry";
|
|
4243
|
+
}
|
|
4244
|
+
const arm = durationTripped ? "failure_window" : "absolute_age";
|
|
4245
|
+
try {
|
|
4246
|
+
await this.markFailed(
|
|
4247
|
+
conv.id,
|
|
4248
|
+
message.id,
|
|
4249
|
+
void 0,
|
|
4250
|
+
_ChannelDriver.REDRIVE_ABANDON_ERROR[outcome]
|
|
4251
|
+
);
|
|
4252
|
+
} catch (err) {
|
|
4253
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4254
|
+
this.log({
|
|
4255
|
+
level: "warn",
|
|
4256
|
+
message: `Re-drive bound: fallback markFailed for message ${message.id.slice(0, 8)} also failed (arm ${arm}, will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
4257
|
+
conversation_id: conv.id,
|
|
4258
|
+
message_id: message.id
|
|
2469
4259
|
});
|
|
4260
|
+
if (!this.redriveOutcomeAbandonedSignalled.has(message.id)) {
|
|
4261
|
+
this.redriveOutcomeAbandonedSignalled.add(message.id);
|
|
4262
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_abandoned", {
|
|
4263
|
+
attempted_outcome: outcome,
|
|
4264
|
+
reported: false,
|
|
4265
|
+
arm
|
|
4266
|
+
});
|
|
4267
|
+
}
|
|
4268
|
+
return "retry";
|
|
4269
|
+
}
|
|
4270
|
+
this.clearRedriveUnresolved(message.id);
|
|
4271
|
+
void this.postSignal(conv.id, message.id, "redrive_outcome_abandoned", {
|
|
4272
|
+
attempted_outcome: outcome,
|
|
4273
|
+
reported: true,
|
|
4274
|
+
arm
|
|
4275
|
+
});
|
|
4276
|
+
return "abandoned";
|
|
4277
|
+
}
|
|
4278
|
+
/**
|
|
4279
|
+
* Record one poll outcome toward the re-drive fence's consecutive-identical-
|
|
4280
|
+
* failure streak (#1348) and return the resulting count. `signature === null`
|
|
4281
|
+
* (a thrown exception, H1) always clears the streak and returns `0` — it is
|
|
4282
|
+
* never countable. Otherwise the streak continues only when BOTH the session
|
|
4283
|
+
* and the signature match the previous failure; anything else (a different
|
|
4284
|
+
* session, or the same session failing a DIFFERENT way) starts a fresh streak
|
|
4285
|
+
* at `1`.
|
|
4286
|
+
*/
|
|
4287
|
+
recordRedrivePollFailure(messageId, sessionId, signature) {
|
|
4288
|
+
if (signature === null) {
|
|
4289
|
+
this.redrivePollFailures.delete(messageId);
|
|
4290
|
+
return 0;
|
|
4291
|
+
}
|
|
4292
|
+
const existing = this.redrivePollFailures.get(messageId);
|
|
4293
|
+
if (existing && existing.sessionId === sessionId && existing.signature === signature) {
|
|
4294
|
+
existing.count += 1;
|
|
4295
|
+
return existing.count;
|
|
4296
|
+
}
|
|
4297
|
+
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
4298
|
+
return 1;
|
|
4299
|
+
}
|
|
4300
|
+
/**
|
|
4301
|
+
* Record one UNCONFIRMED-dispatch outcome (a `pending` row with no stored
|
|
4302
|
+
* `opencode_message_id` whose `sendPromptAsync` returned `null`) toward the
|
|
4303
|
+
* bound in `processConversation`'s dispatch loop, and return the resulting
|
|
4304
|
+
* count. Mirrors `recordRedrivePollFailure`'s session-scoping: a session
|
|
4305
|
+
* change starts a fresh streak at `1` rather than inheriting the old one's
|
|
4306
|
+
* count, since a new session is a genuinely different attempt.
|
|
4307
|
+
*/
|
|
4308
|
+
recordUnconfirmedDispatch(messageId, sessionId) {
|
|
4309
|
+
const existing = this.unconfirmedDispatchFailures.get(messageId);
|
|
4310
|
+
if (existing && existing.sessionId === sessionId) {
|
|
4311
|
+
existing.count += 1;
|
|
4312
|
+
return existing.count;
|
|
4313
|
+
}
|
|
4314
|
+
this.unconfirmedDispatchFailures.set(messageId, { sessionId, count: 1 });
|
|
4315
|
+
return 1;
|
|
4316
|
+
}
|
|
4317
|
+
/**
|
|
4318
|
+
* Record that `sessionId` is no longer a valid binding for `conversationId`
|
|
4319
|
+
* (#553). Keyed by conversation and hard-capped, so it cannot grow with the
|
|
4320
|
+
* number of failures — see the `supersededSessions` field doc.
|
|
4321
|
+
*/
|
|
4322
|
+
supersede(conversationId, sessionId) {
|
|
4323
|
+
this.supersededSessions.delete(conversationId);
|
|
4324
|
+
this.supersededSessions.set(conversationId, sessionId);
|
|
4325
|
+
while (this.supersededSessions.size > MAX_SUPERSEDED_CONVERSATIONS) {
|
|
4326
|
+
const oldest = this.supersededSessions.keys().next().value;
|
|
4327
|
+
if (oldest === void 0) return;
|
|
4328
|
+
this.supersededSessions.delete(oldest);
|
|
2470
4329
|
}
|
|
2471
|
-
this.ensureWatcherRunning(sessionId);
|
|
2472
|
-
return dispatched;
|
|
2473
4330
|
}
|
|
4331
|
+
/** Whether `sessionId` is the session this conversation has abandoned (#553). */
|
|
4332
|
+
isSuperseded(conversationId, sessionId) {
|
|
4333
|
+
return this.supersededSessions.get(conversationId) === sessionId;
|
|
4334
|
+
}
|
|
4335
|
+
/**
|
|
4336
|
+
* Resolve the opencode session to run this conversation's turns in.
|
|
4337
|
+
*
|
|
4338
|
+
* `refusedSessionId` is set when the #553 guard fired — i.e. the persisted
|
|
4339
|
+
* binding was an id this runner had abandoned, so a resurrection genuinely
|
|
4340
|
+
* happened and a fresh session was bound instead. The caller reports it.
|
|
4341
|
+
*
|
|
4342
|
+
* `created` says the returned session was made JUST NOW, so it provably holds
|
|
4343
|
+
* no prior turn. The re-drive fence needs that as CONTRARY evidence ("nothing
|
|
4344
|
+
* to reconcile against") — distinct from the ambiguous "I polled and saw an
|
|
4345
|
+
* empty transcript", which stays a deferral. Keep it separate from
|
|
4346
|
+
* `refusedSessionId`: only the latter means a #553 resurrection happened, and
|
|
4347
|
+
* only it may drive the `session_superseded` signal.
|
|
4348
|
+
*/
|
|
2474
4349
|
async ensureSession(conv) {
|
|
2475
4350
|
const bound = this.sessions.get(conv.id) ?? conv.opencode_session_id ?? null;
|
|
4351
|
+
if (bound && this.isSuperseded(conv.id, bound)) {
|
|
4352
|
+
this.log({
|
|
4353
|
+
level: "warn",
|
|
4354
|
+
message: `OpenCode session ${bound.slice(0, 8)} was abandoned for conversation ${conv.id.slice(0, 8)} after a failed dispatch but is still bound to it (the persisted id was written back by a turn already in flight) \u2014 ignoring it and binding a fresh session.`,
|
|
4355
|
+
conversation_id: conv.id
|
|
4356
|
+
});
|
|
4357
|
+
this.sessions.delete(conv.id);
|
|
4358
|
+
return {
|
|
4359
|
+
sessionId: await this.createAndBindSession(conv.id),
|
|
4360
|
+
refusedSessionId: bound,
|
|
4361
|
+
created: true
|
|
4362
|
+
};
|
|
4363
|
+
}
|
|
2476
4364
|
if (bound) {
|
|
2477
4365
|
const exists = await sessionExists(this.port, bound);
|
|
2478
4366
|
if (exists === false) {
|
|
@@ -2482,12 +4370,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2482
4370
|
conversation_id: conv.id
|
|
2483
4371
|
});
|
|
2484
4372
|
this.sessions.delete(conv.id);
|
|
2485
|
-
return this.createAndBindSession(conv.id);
|
|
4373
|
+
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
2486
4374
|
}
|
|
2487
4375
|
this.sessions.set(conv.id, bound);
|
|
2488
|
-
return bound;
|
|
4376
|
+
return { sessionId: bound, created: false };
|
|
2489
4377
|
}
|
|
2490
|
-
return this.createAndBindSession(conv.id);
|
|
4378
|
+
return { sessionId: await this.createAndBindSession(conv.id), created: true };
|
|
2491
4379
|
}
|
|
2492
4380
|
/**
|
|
2493
4381
|
* Create a fresh OpenCode session for a conversation, cache the binding, and
|
|
@@ -2498,7 +4386,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2498
4386
|
const directory = await this.resolveOpenCodeDirectory();
|
|
2499
4387
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
2500
4388
|
this.sessions.set(conversationId, sessionId);
|
|
2501
|
-
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
4389
|
+
await this.persistSession(conversationId, sessionId).catch((err) => {
|
|
4390
|
+
this.log({
|
|
4391
|
+
level: "warn",
|
|
4392
|
+
message: `Persisting the OpenCode session binding ${sessionId.slice(0, 8)} for conversation ${conversationId.slice(0, 8)} failed (best-effort, not retried) \u2014 the completion PATCH also carries opencode_session_id, so the binding is repaired when the turn finishes: ${err instanceof Error ? err.message : String(err)}`,
|
|
4393
|
+
conversation_id: conversationId
|
|
4394
|
+
});
|
|
2502
4395
|
});
|
|
2503
4396
|
return sessionId;
|
|
2504
4397
|
}
|
|
@@ -2690,12 +4583,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2690
4583
|
stuckReported: false,
|
|
2691
4584
|
lastAliveAt: 0,
|
|
2692
4585
|
aliveInFlight: false,
|
|
4586
|
+
titleSynced: false,
|
|
4587
|
+
titleSyncInFlight: false,
|
|
2693
4588
|
awaitingHumanLatched: false,
|
|
2694
4589
|
pausedOnQuestion: false,
|
|
2695
4590
|
pausedOnPermission: false,
|
|
2696
4591
|
pausedClearConfirmed: false,
|
|
2697
4592
|
pausedInFlight: false,
|
|
2698
|
-
deliveryDeadlineAnchored: false
|
|
4593
|
+
deliveryDeadlineAnchored: false,
|
|
4594
|
+
b2PinnedSinceMs: 0,
|
|
4595
|
+
b2LastDescendantCheckMs: 0,
|
|
4596
|
+
b2AbandonedSignalled: false
|
|
2699
4597
|
});
|
|
2700
4598
|
}
|
|
2701
4599
|
/**
|
|
@@ -2710,9 +4608,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2710
4608
|
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
2711
4609
|
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
2712
4610
|
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
2713
|
-
* handed to the cron.
|
|
2714
|
-
*
|
|
2715
|
-
*
|
|
4611
|
+
* handed to the cron. Real invariant (#965): the cron MAY reclaim a row this
|
|
4612
|
+
* runner still holds; a reclaimed row that already ran is never re-dispatched
|
|
4613
|
+
* while opencode reports its turn ongoing (readopt's own gate here, and the
|
|
4614
|
+
* `pending`-row re-drive fence, `resolveRedrive`). `dispatchedAt` stays `now`
|
|
2716
4615
|
* (only the appear-guard uses it).
|
|
2717
4616
|
*
|
|
2718
4617
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
@@ -2763,12 +4662,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2763
4662
|
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
2764
4663
|
lastAliveAt: 0,
|
|
2765
4664
|
aliveInFlight: false,
|
|
4665
|
+
titleSynced: false,
|
|
4666
|
+
titleSyncInFlight: false,
|
|
2766
4667
|
awaitingHumanLatched: false,
|
|
2767
4668
|
pausedOnQuestion: false,
|
|
2768
4669
|
pausedOnPermission: false,
|
|
2769
4670
|
pausedClearConfirmed: false,
|
|
2770
4671
|
pausedInFlight: false,
|
|
2771
|
-
deliveryDeadlineAnchored: false
|
|
4672
|
+
deliveryDeadlineAnchored: false,
|
|
4673
|
+
b2PinnedSinceMs: 0,
|
|
4674
|
+
b2LastDescendantCheckMs: 0,
|
|
4675
|
+
b2AbandonedSignalled: false
|
|
2772
4676
|
});
|
|
2773
4677
|
}
|
|
2774
4678
|
/**
|
|
@@ -2900,9 +4804,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2900
4804
|
const awaitingHuman = observedOpen || latchedPaused;
|
|
2901
4805
|
if ((state === "running" || state === "done" || state === "failed") && !inFlight.started) {
|
|
2902
4806
|
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
2903
|
-
let claimed;
|
|
2904
4807
|
try {
|
|
2905
|
-
|
|
4808
|
+
await this.markProcessing(
|
|
2906
4809
|
conv.id,
|
|
2907
4810
|
inFlight.evidentMessageId,
|
|
2908
4811
|
sessionId,
|
|
@@ -2911,77 +4814,27 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2911
4814
|
);
|
|
2912
4815
|
} catch (err) {
|
|
2913
4816
|
if (err instanceof ChannelAuthError) throw err;
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
inFlight.started = true;
|
|
2923
|
-
if (!claimed) {
|
|
2924
|
-
this.log({
|
|
2925
|
-
level: "debug",
|
|
2926
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} already marked processing \u2014 continuing`,
|
|
2927
|
-
conversation_id: conv.id,
|
|
2928
|
-
message_id: inFlight.evidentMessageId
|
|
2929
|
-
});
|
|
2930
|
-
}
|
|
2931
|
-
}
|
|
2932
|
-
if (state === "done") {
|
|
2933
|
-
this.anchorDeliveryDeadline(inFlight);
|
|
2934
|
-
if (!inFlight.done) {
|
|
2935
|
-
this.log({
|
|
2936
|
-
level: "info",
|
|
2937
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
2938
|
-
conversation_id: conv.id,
|
|
2939
|
-
message_id: inFlight.evidentMessageId
|
|
2940
|
-
});
|
|
2941
|
-
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
2942
|
-
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
2943
|
-
try {
|
|
2944
|
-
await this.markDone(
|
|
2945
|
-
conv.id,
|
|
2946
|
-
inFlight.evidentMessageId,
|
|
2947
|
-
sessionId,
|
|
2948
|
-
inFlight.opencodeMessageId,
|
|
2949
|
-
title,
|
|
2950
|
-
usage
|
|
2951
|
-
);
|
|
2952
|
-
} catch (err) {
|
|
2953
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
2954
|
-
if (err instanceof ChannelTerminalError) {
|
|
2955
|
-
this.log({
|
|
2956
|
-
level: "warn",
|
|
2957
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
2958
|
-
conversation_id: conv.id,
|
|
2959
|
-
message_id: inFlight.evidentMessageId
|
|
2960
|
-
});
|
|
2961
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2962
|
-
return;
|
|
2963
|
-
}
|
|
2964
|
-
if (this.now() >= inFlight.deadline) {
|
|
2965
|
-
this.log({
|
|
2966
|
-
level: "warn",
|
|
2967
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
2968
|
-
conversation_id: conv.id,
|
|
2969
|
-
message_id: inFlight.evidentMessageId
|
|
2970
|
-
});
|
|
2971
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
2972
|
-
return;
|
|
2973
|
-
}
|
|
4817
|
+
if (err instanceof ChannelTerminalError) {
|
|
4818
|
+
this.log({
|
|
4819
|
+
level: "error",
|
|
4820
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (terminal HTTP ${err.status}) \u2014 the server definitively refused the swap`,
|
|
4821
|
+
conversation_id: conv.id,
|
|
4822
|
+
message_id: inFlight.evidentMessageId
|
|
4823
|
+
});
|
|
4824
|
+
} else {
|
|
2974
4825
|
this.log({
|
|
2975
4826
|
level: "warn",
|
|
2976
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)}
|
|
4827
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} processing (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
2977
4828
|
conversation_id: conv.id,
|
|
2978
4829
|
message_id: inFlight.evidentMessageId
|
|
2979
4830
|
});
|
|
2980
4831
|
return;
|
|
2981
4832
|
}
|
|
2982
|
-
inFlight.done = true;
|
|
2983
4833
|
}
|
|
2984
|
-
|
|
4834
|
+
inFlight.started = true;
|
|
4835
|
+
}
|
|
4836
|
+
if (state === "done") {
|
|
4837
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
2985
4838
|
return;
|
|
2986
4839
|
}
|
|
2987
4840
|
if (state === "failed") {
|
|
@@ -2995,8 +4848,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2995
4848
|
message_id: inFlight.evidentMessageId
|
|
2996
4849
|
});
|
|
2997
4850
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
4851
|
+
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
2998
4852
|
try {
|
|
2999
|
-
await this.markFailed(
|
|
4853
|
+
await this.markFailed(
|
|
4854
|
+
conv.id,
|
|
4855
|
+
inFlight.evidentMessageId,
|
|
4856
|
+
sessionId,
|
|
4857
|
+
error2,
|
|
4858
|
+
usage,
|
|
4859
|
+
failure
|
|
4860
|
+
);
|
|
3000
4861
|
} catch (err) {
|
|
3001
4862
|
if (err instanceof ChannelAuthError) throw err;
|
|
3002
4863
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3041,6 +4902,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3041
4902
|
});
|
|
3042
4903
|
}
|
|
3043
4904
|
const activelyRunning = state === "running" && !awaitingHuman;
|
|
4905
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
4906
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
4907
|
+
if (!pinnedNow) {
|
|
4908
|
+
if (snapshotReadable) {
|
|
4909
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
4910
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
4911
|
+
inFlight.b2AbandonedSignalled = false;
|
|
4912
|
+
}
|
|
4913
|
+
} else {
|
|
4914
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
4915
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
4916
|
+
return;
|
|
4917
|
+
}
|
|
4918
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
4919
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
4920
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
4921
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
4922
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
4923
|
+
if (isB2AbandonmentConfirmed({
|
|
4924
|
+
pinnedForMs,
|
|
4925
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
4926
|
+
descendantOngoing
|
|
4927
|
+
})) {
|
|
4928
|
+
inFlight.b2AbandonedSignalled = true;
|
|
4929
|
+
this.log({
|
|
4930
|
+
level: "warn",
|
|
4931
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
4932
|
+
conversation_id: conv.id,
|
|
4933
|
+
message_id: id
|
|
4934
|
+
});
|
|
4935
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
4936
|
+
watched_for_ms: pinnedForMs
|
|
4937
|
+
});
|
|
4938
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
4939
|
+
return;
|
|
4940
|
+
}
|
|
4941
|
+
}
|
|
4942
|
+
}
|
|
3044
4943
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
3045
4944
|
this.log({
|
|
3046
4945
|
level: "warn",
|
|
@@ -3060,6 +4959,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3060
4959
|
inFlight.aliveInFlight = false;
|
|
3061
4960
|
if (ok) inFlight.lastAliveAt = this.now();
|
|
3062
4961
|
});
|
|
4962
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
4963
|
+
inFlight.titleSyncInFlight = true;
|
|
4964
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
4965
|
+
if (!title) {
|
|
4966
|
+
inFlight.titleSyncInFlight = false;
|
|
4967
|
+
return;
|
|
4968
|
+
}
|
|
4969
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
4970
|
+
inFlight.titleSyncInFlight = false;
|
|
4971
|
+
if (ok) inFlight.titleSynced = true;
|
|
4972
|
+
});
|
|
4973
|
+
}
|
|
3063
4974
|
}
|
|
3064
4975
|
if (awaitingHuman) {
|
|
3065
4976
|
if (!inFlight.awaitingHumanLatched) {
|
|
@@ -3097,6 +5008,70 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3097
5008
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3098
5009
|
}
|
|
3099
5010
|
}
|
|
5011
|
+
/**
|
|
5012
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
5013
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
5014
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
5015
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
5016
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
5017
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
5018
|
+
* and risking the two copies silently drifting apart.
|
|
5019
|
+
*/
|
|
5020
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
5021
|
+
const conv = watcher.conv;
|
|
5022
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
5023
|
+
if (!inFlight.done) {
|
|
5024
|
+
this.log({
|
|
5025
|
+
level: "info",
|
|
5026
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
5027
|
+
conversation_id: conv.id,
|
|
5028
|
+
message_id: inFlight.evidentMessageId
|
|
5029
|
+
});
|
|
5030
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
5031
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
5032
|
+
try {
|
|
5033
|
+
await this.markDone(
|
|
5034
|
+
conv.id,
|
|
5035
|
+
inFlight.evidentMessageId,
|
|
5036
|
+
sessionId,
|
|
5037
|
+
inFlight.opencodeMessageId,
|
|
5038
|
+
title,
|
|
5039
|
+
usage
|
|
5040
|
+
);
|
|
5041
|
+
} catch (err) {
|
|
5042
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
5043
|
+
if (err instanceof ChannelTerminalError) {
|
|
5044
|
+
this.log({
|
|
5045
|
+
level: "warn",
|
|
5046
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
5047
|
+
conversation_id: conv.id,
|
|
5048
|
+
message_id: inFlight.evidentMessageId
|
|
5049
|
+
});
|
|
5050
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5051
|
+
return;
|
|
5052
|
+
}
|
|
5053
|
+
if (this.now() >= inFlight.deadline) {
|
|
5054
|
+
this.log({
|
|
5055
|
+
level: "warn",
|
|
5056
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
5057
|
+
conversation_id: conv.id,
|
|
5058
|
+
message_id: inFlight.evidentMessageId
|
|
5059
|
+
});
|
|
5060
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5061
|
+
return;
|
|
5062
|
+
}
|
|
5063
|
+
this.log({
|
|
5064
|
+
level: "warn",
|
|
5065
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
5066
|
+
conversation_id: conv.id,
|
|
5067
|
+
message_id: inFlight.evidentMessageId
|
|
5068
|
+
});
|
|
5069
|
+
return;
|
|
5070
|
+
}
|
|
5071
|
+
inFlight.done = true;
|
|
5072
|
+
}
|
|
5073
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
5074
|
+
}
|
|
3100
5075
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
3101
5076
|
/**
|
|
3102
5077
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -3196,7 +5171,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3196
5171
|
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
3197
5172
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
3198
5173
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
3199
|
-
* errored turn is reported failed on restart, NOT re-dispatched
|
|
5174
|
+
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
5175
|
+
* EXCEPT a restart-ABORTED turn under a not-ongoing session,
|
|
5176
|
+
* which is a restart orphan wearing a terminal error and is
|
|
5177
|
+
* re-dispatched instead (issue #1310, see the branch below);
|
|
3200
5178
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
3201
5179
|
* tracking the stored id so the reply correlates by it;
|
|
3202
5180
|
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
@@ -3260,9 +5238,19 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3260
5238
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
3261
5239
|
return;
|
|
3262
5240
|
}
|
|
3263
|
-
|
|
5241
|
+
const restartAborted = state === "failed" && sessionOngoing === false && isAbortedTerminalReply(messages, ocId ?? "");
|
|
5242
|
+
if (restartAborted) {
|
|
5243
|
+
this.log({
|
|
5244
|
+
level: "info",
|
|
5245
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
5246
|
+
conversation_id: row.conversation_id,
|
|
5247
|
+
message_id: row.id
|
|
5248
|
+
});
|
|
5249
|
+
}
|
|
5250
|
+
if (state === "failed" && !restartAborted) {
|
|
3264
5251
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3265
5252
|
const usage = messageUsage(messages, ocId ?? "");
|
|
5253
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3266
5254
|
this.log({
|
|
3267
5255
|
level: "error",
|
|
3268
5256
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -3270,7 +5258,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3270
5258
|
message_id: row.id
|
|
3271
5259
|
});
|
|
3272
5260
|
try {
|
|
3273
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
5261
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage, failure);
|
|
3274
5262
|
} catch (err) {
|
|
3275
5263
|
if (err instanceof ChannelAuthError) throw err;
|
|
3276
5264
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3473,15 +5461,39 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3473
5461
|
}
|
|
3474
5462
|
if (ocId === null) {
|
|
3475
5463
|
this.awaitingReadopt.delete(row.id);
|
|
5464
|
+
const streak = this.recordUnconfirmedDispatch(row.id, sessionId);
|
|
5465
|
+
if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES) {
|
|
5466
|
+
this.unconfirmedDispatchFailures.delete(row.id);
|
|
5467
|
+
this.sessions.delete(readoptConv.id);
|
|
5468
|
+
this.supersede(readoptConv.id, sessionId);
|
|
5469
|
+
const errorMessage = `OpenCode accepted this re-dispatched message ${streak} times in a row but never confirmed its assigned id (session ${sessionId.slice(0, 8)}'s message list could not be read back) \u2014 the session was abandoned; a fresh one is used for further messages.`;
|
|
5470
|
+
this.log({
|
|
5471
|
+
level: "error",
|
|
5472
|
+
message: errorMessage,
|
|
5473
|
+
conversation_id: row.conversation_id,
|
|
5474
|
+
message_id: row.id
|
|
5475
|
+
});
|
|
5476
|
+
await this.markFailed(row.conversation_id, row.id, null, errorMessage).catch((markErr) => {
|
|
5477
|
+
this.log({
|
|
5478
|
+
level: "warn",
|
|
5479
|
+
message: `markFailed PATCH for message ${row.id.slice(0, 8)} (conversation ${row.conversation_id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
5480
|
+
conversation_id: row.conversation_id,
|
|
5481
|
+
message_id: row.id
|
|
5482
|
+
});
|
|
5483
|
+
});
|
|
5484
|
+
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
5485
|
+
return;
|
|
5486
|
+
}
|
|
3476
5487
|
this.log({
|
|
3477
5488
|
level: "warn",
|
|
3478
|
-
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next drain`,
|
|
5489
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} re-dispatched but its opencode id could not be read back (${streak}/${MAX_IDENTICAL_REDRIVE_POLL_FAILURES}) \u2014 leaving un-tracked to retry next drain`,
|
|
3479
5490
|
conversation_id: row.conversation_id,
|
|
3480
5491
|
message_id: row.id
|
|
3481
5492
|
});
|
|
3482
5493
|
void this.postSignal(row.conversation_id, row.id, "readopt_orphan_unsent");
|
|
3483
5494
|
return;
|
|
3484
5495
|
}
|
|
5496
|
+
this.unconfirmedDispatchFailures.delete(row.id);
|
|
3485
5497
|
this.registerReadopted(readoptConv, sessionId, readoptMessage, ocId, this.processedAtMs(row));
|
|
3486
5498
|
this.dispatched.add(row.id);
|
|
3487
5499
|
this.readopted.add(row.id);
|
|
@@ -3537,7 +5549,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3537
5549
|
opencode_model: row.opencode_model,
|
|
3538
5550
|
source_message_id: row.source_message_id,
|
|
3539
5551
|
slack_user_id: row.slack_user_id,
|
|
3540
|
-
attachments: row.attachments ?? null
|
|
5552
|
+
attachments: row.attachments ?? null,
|
|
5553
|
+
opencode_message_id: row.opencode_message_id
|
|
3541
5554
|
};
|
|
3542
5555
|
}
|
|
3543
5556
|
/**
|
|
@@ -3676,6 +5689,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3676
5689
|
}
|
|
3677
5690
|
return false;
|
|
3678
5691
|
}
|
|
5692
|
+
/**
|
|
5693
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
5694
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
5695
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
5696
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
5697
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
5698
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
5699
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
5700
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
5701
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
5702
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
5703
|
+
* not ongoing".
|
|
5704
|
+
*
|
|
5705
|
+
* Return contract:
|
|
5706
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
5707
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
5708
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
5709
|
+
* CONFIRMED NOT a descendant of it.
|
|
5710
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
5711
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
5712
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
5713
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
5714
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
5715
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
5716
|
+
* here.
|
|
5717
|
+
*
|
|
5718
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
5719
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
5720
|
+
* by interaction attribution or the recovery path.
|
|
5721
|
+
*/
|
|
5722
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
5723
|
+
let current = sessionId;
|
|
5724
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
5725
|
+
if (current === rootSessionId) return true;
|
|
5726
|
+
const parent = await this.resolveSessionParent(current);
|
|
5727
|
+
if (parent === void 0) return null;
|
|
5728
|
+
if (parent === null) return false;
|
|
5729
|
+
current = parent;
|
|
5730
|
+
}
|
|
5731
|
+
return null;
|
|
5732
|
+
}
|
|
3679
5733
|
/**
|
|
3680
5734
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
3681
5735
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -3760,6 +5814,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3760
5814
|
}
|
|
3761
5815
|
return null;
|
|
3762
5816
|
}
|
|
5817
|
+
/**
|
|
5818
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
5819
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
5820
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
5821
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
5822
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
5823
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
5824
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
5825
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
5826
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
5827
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
5828
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
5829
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
5830
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
5831
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
5832
|
+
*
|
|
5833
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
5834
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
5835
|
+
* caller only latches `titleSynced` on `true`).
|
|
5836
|
+
*/
|
|
5837
|
+
async patchConversationTitle(conversationId, title) {
|
|
5838
|
+
try {
|
|
5839
|
+
const res = await this.fetchImpl(
|
|
5840
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
5841
|
+
{
|
|
5842
|
+
method: "PATCH",
|
|
5843
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
5844
|
+
body: JSON.stringify({ title })
|
|
5845
|
+
}
|
|
5846
|
+
);
|
|
5847
|
+
if (!res.ok) {
|
|
5848
|
+
this.log({
|
|
5849
|
+
level: "debug",
|
|
5850
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
5851
|
+
conversation_id: conversationId
|
|
5852
|
+
});
|
|
5853
|
+
return false;
|
|
5854
|
+
}
|
|
5855
|
+
return true;
|
|
5856
|
+
} catch (err) {
|
|
5857
|
+
this.log({
|
|
5858
|
+
level: "debug",
|
|
5859
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
5860
|
+
conversation_id: conversationId
|
|
5861
|
+
});
|
|
5862
|
+
return false;
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
3763
5865
|
/**
|
|
3764
5866
|
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
3765
5867
|
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
@@ -3818,6 +5920,84 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3818
5920
|
}
|
|
3819
5921
|
return false;
|
|
3820
5922
|
}
|
|
5923
|
+
/**
|
|
5924
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
5925
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
5926
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
5927
|
+
*
|
|
5928
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
5929
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
5930
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
5931
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
5932
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
5933
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
5934
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
5935
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
5936
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
5937
|
+
* executing, between its step's completion and the next generation step"
|
|
5938
|
+
* transcript gap that a transcript-based check would need a second,
|
|
5939
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
5940
|
+
* message timestamps at all.
|
|
5941
|
+
*
|
|
5942
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
5943
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
5944
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
5945
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
5946
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
5947
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
5948
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
5949
|
+
* delegation — which the root's status is not.
|
|
5950
|
+
*
|
|
5951
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
5952
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
5953
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
5954
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
5955
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
5956
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
5957
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
5958
|
+
* instead.
|
|
5959
|
+
*
|
|
5960
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
5961
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
5962
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
5963
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
5964
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
5965
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
5966
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
5967
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
5968
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
5969
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
5970
|
+
* or pathological chain partway through the parent walk), OR at
|
|
5971
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
5972
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
5973
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
5974
|
+
* (unlike the recovery cross-check's contract) — see
|
|
5975
|
+
* `isB2AbandonmentConfirmed`.
|
|
5976
|
+
*/
|
|
5977
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
5978
|
+
const sessions = await listSessions(this.port);
|
|
5979
|
+
if (!sessions) {
|
|
5980
|
+
this.log({
|
|
5981
|
+
level: "warn",
|
|
5982
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
5983
|
+
});
|
|
5984
|
+
return null;
|
|
5985
|
+
}
|
|
5986
|
+
let indeterminate = false;
|
|
5987
|
+
for (const candidate of sessions) {
|
|
5988
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
5989
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
5990
|
+
if (membership === null) {
|
|
5991
|
+
indeterminate = true;
|
|
5992
|
+
continue;
|
|
5993
|
+
}
|
|
5994
|
+
if (membership === false) continue;
|
|
5995
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
5996
|
+
if (ongoing === true) return true;
|
|
5997
|
+
if (ongoing === null) indeterminate = true;
|
|
5998
|
+
}
|
|
5999
|
+
return indeterminate ? null : false;
|
|
6000
|
+
}
|
|
3821
6001
|
/**
|
|
3822
6002
|
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
3823
6003
|
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
@@ -3946,24 +6126,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3946
6126
|
}
|
|
3947
6127
|
return messages;
|
|
3948
6128
|
}
|
|
6129
|
+
/**
|
|
6130
|
+
* The `opencode_session_id` fragment of a status PATCH body — `{}` when this
|
|
6131
|
+
* conversation has ABANDONED that session (#553). The field is optional
|
|
6132
|
+
* server-side and an absent one leaves the persisted binding untouched, so
|
|
6133
|
+
* omitting it is how a routine status write stops resurrecting it.
|
|
6134
|
+
*
|
|
6135
|
+
* ONLY for writes whose sole cost is a lost deep link. The `processing` notice
|
|
6136
|
+
* degrades to no "View in Evident" link (the reaction swap still fires) and the
|
|
6137
|
+
* turn-failure notice is built from the PATCH's own `error` text with a link off
|
|
6138
|
+
* the persisted row — neither loses content the user came for. `markDone`
|
|
6139
|
+
* deliberately does NOT use this helper: the server fetches the reply text
|
|
6140
|
+
* THROUGH the session id it is given, so suppressing there would replace the
|
|
6141
|
+
* agent's answer with a bare "✅ Done!" (the #183/#187 failure). The
|
|
6142
|
+
* `ensureSession` guard, not this suppression, is what makes the self-heal
|
|
6143
|
+
* stick.
|
|
6144
|
+
*/
|
|
6145
|
+
sessionIdBody(sessionId, conversationId, messageId, status2) {
|
|
6146
|
+
if (!this.isSuperseded(conversationId, sessionId)) return { opencode_session_id: sessionId };
|
|
6147
|
+
this.log({
|
|
6148
|
+
level: "debug",
|
|
6149
|
+
message: `Omitting the abandoned OpenCode session ${sessionId.slice(0, 8)} from the '${status2}' update for message ${messageId.slice(0, 8)} so it is not re-bound to conversation ${conversationId.slice(0, 8)}`,
|
|
6150
|
+
conversation_id: conversationId,
|
|
6151
|
+
message_id: messageId
|
|
6152
|
+
});
|
|
6153
|
+
return {};
|
|
6154
|
+
}
|
|
3949
6155
|
/**
|
|
3950
6156
|
* EXISTING combinedAuth route — now fired by the watcher on queued→running
|
|
3951
6157
|
* (Task 3.3), NOT at dispatch/claim time. `{status:'processing',
|
|
3952
6158
|
* opencode_session_id}` → `notifyMessageStarted` (hourglass→runner swap +
|
|
3953
6159
|
* deep-linked "View in Evident" notice).
|
|
3954
6160
|
*
|
|
3955
|
-
*
|
|
3956
|
-
* -
|
|
3957
|
-
*
|
|
3958
|
-
*
|
|
3959
|
-
*
|
|
3960
|
-
*
|
|
3961
|
-
*
|
|
3962
|
-
*
|
|
3963
|
-
*
|
|
3964
|
-
*
|
|
3965
|
-
*
|
|
3966
|
-
*
|
|
6161
|
+
* Outcome contract (consumed by the watcher's swap-to-running guard):
|
|
6162
|
+
* - resolves (`void`) → the server transitioned the row to
|
|
6163
|
+
* processing (or idempotently confirmed
|
|
6164
|
+
* already-processing — that answer is
|
|
6165
|
+
* still a 200, never a refusal);
|
|
6166
|
+
* - throws `ChannelAuthError` → 401/403 (terminal auth failure);
|
|
6167
|
+
* - throws `ChannelTerminalError` → a definitive non-retryable, non-auth 4xx
|
|
6168
|
+
* (404 the row or its conversation is
|
|
6169
|
+
* gone, 400 the update was rejected).
|
|
6170
|
+
* Retrying cannot help;
|
|
6171
|
+
* - throws a plain `Error` → a TRANSIENT failure (retryable 5xx/429
|
|
6172
|
+
* status, or a network-level error from
|
|
6173
|
+
* `fetch`) — i.e. NO definitive server
|
|
6174
|
+
* response — so the caller leaves the
|
|
6175
|
+
* message un-started and retries the swap
|
|
6176
|
+
* on the next tick.
|
|
3967
6177
|
* A single attempt (no internal retry): the watcher's per-tick loop is the
|
|
3968
6178
|
* retry vehicle for the swap-to-running.
|
|
3969
6179
|
*/
|
|
@@ -3975,18 +6185,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3975
6185
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
3976
6186
|
body: JSON.stringify({
|
|
3977
6187
|
status: "processing",
|
|
3978
|
-
|
|
6188
|
+
...this.sessionIdBody(sessionId, conversationId, messageId, "processing"),
|
|
3979
6189
|
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
3980
6190
|
...title ? { title } : {}
|
|
3981
6191
|
})
|
|
3982
6192
|
}
|
|
3983
6193
|
);
|
|
3984
6194
|
this.assertAuth(res, "marking message as processing");
|
|
3985
|
-
if (res.ok) return
|
|
6195
|
+
if (res.ok) return;
|
|
3986
6196
|
if (isRetryableStatus(res.status)) {
|
|
3987
6197
|
throw new Error(`marking message as processing: HTTP ${res.status}`);
|
|
3988
6198
|
}
|
|
3989
|
-
|
|
6199
|
+
throw new ChannelTerminalError(`marking message as processing: HTTP ${res.status}`, res.status);
|
|
3990
6200
|
}
|
|
3991
6201
|
/**
|
|
3992
6202
|
* EXISTING combinedAuth completion route — idempotent (WI-CHAN-2). `PATCH
|
|
@@ -4024,6 +6234,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4024
6234
|
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
4025
6235
|
body: JSON.stringify({
|
|
4026
6236
|
status: "done",
|
|
6237
|
+
// ALWAYS sent, even for a session this conversation has abandoned
|
|
6238
|
+
// (#553): the server reads the reply text back out of THIS session id
|
|
6239
|
+
// to deliver it. Omitting it would leave the user with "✅ Done!"
|
|
6240
|
+
// instead of the answer — a worse regression than the resurrection it
|
|
6241
|
+
// would prevent, which `ensureSession`'s guard handles anyway.
|
|
4027
6242
|
opencode_session_id: sessionId,
|
|
4028
6243
|
...opencodeMessageId ? { opencode_message_id: opencodeMessageId } : {},
|
|
4029
6244
|
...title ? { title } : {},
|
|
@@ -4040,16 +6255,31 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4040
6255
|
}
|
|
4041
6256
|
/**
|
|
4042
6257
|
* Mark a message `failed`. `sessionId` / `error` are threaded to the API ONLY
|
|
4043
|
-
* when provided (issue #182)
|
|
4044
|
-
* `
|
|
4045
|
-
*
|
|
4046
|
-
*
|
|
6258
|
+
* when provided (issue #182). Three states for `sessionId`:
|
|
6259
|
+
* - omitted (`undefined`) → don't send the field, leave the persisted
|
|
6260
|
+
* session untouched (unused today; kept for API symmetry).
|
|
6261
|
+
* - a real id (`string`) → send it, update the persisted session (the
|
|
6262
|
+
* turn-failure call sites: an errored OpenCode turn).
|
|
6263
|
+
* - explicit `null` → send it, CLEAR the persisted session (issue
|
|
6264
|
+
* #485's dispatch-handoff-failure call site: the session id still
|
|
6265
|
+
* exists but is wedged, so the next attempt must get a fresh one
|
|
6266
|
+
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
4047
6267
|
*/
|
|
4048
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
6268
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
4049
6269
|
const body = { status: "failed" };
|
|
4050
|
-
if (sessionId
|
|
6270
|
+
if (sessionId === null) {
|
|
6271
|
+
body.opencode_session_id = null;
|
|
6272
|
+
} else if (sessionId !== void 0) {
|
|
6273
|
+
Object.assign(body, this.sessionIdBody(sessionId, conversationId, messageId, "failed"));
|
|
6274
|
+
}
|
|
4051
6275
|
if (error2 !== void 0) body.error = error2;
|
|
4052
6276
|
if (usage) Object.assign(body, usage);
|
|
6277
|
+
if (failure) {
|
|
6278
|
+
body.failure_kind = failure.kind;
|
|
6279
|
+
body.failure_provider_id = failure.providerId;
|
|
6280
|
+
body.failure_model_id = failure.modelId;
|
|
6281
|
+
body.failure_reason = failure.reason;
|
|
6282
|
+
}
|
|
4053
6283
|
await this.callWithRetry(
|
|
4054
6284
|
"marking message as failed",
|
|
4055
6285
|
() => this.fetchImpl(
|
|
@@ -4062,6 +6292,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4062
6292
|
)
|
|
4063
6293
|
);
|
|
4064
6294
|
}
|
|
6295
|
+
/**
|
|
6296
|
+
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
6297
|
+
*
|
|
6298
|
+
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
6299
|
+
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
6300
|
+
* to the P1-2b zero-provider check — one extra loopback call to
|
|
6301
|
+
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
6302
|
+
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
6303
|
+
* (`null`/indeterminate) leaves the classification `null`, which produces
|
|
6304
|
+
* today's byte-identical PATCH body via `markFailed`'s `if (failure)` guard.
|
|
6305
|
+
*/
|
|
6306
|
+
async classifyModelAuthFailure(messages, userMessageId) {
|
|
6307
|
+
const classified = messageFailure(messages, userMessageId);
|
|
6308
|
+
if (classified != null) return classified;
|
|
6309
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
6310
|
+
const hasProvider = await hasAnyConfiguredProvider(this.port);
|
|
6311
|
+
return applyZeroProviderFallback(
|
|
6312
|
+
classified,
|
|
6313
|
+
hasProvider,
|
|
6314
|
+
reply?.info?.providerID ?? null,
|
|
6315
|
+
reply?.info?.modelID ?? null
|
|
6316
|
+
);
|
|
6317
|
+
}
|
|
4065
6318
|
/**
|
|
4066
6319
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
4067
6320
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -4214,10 +6467,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4214
6467
|
import chalk5 from "chalk";
|
|
4215
6468
|
import ora2 from "ora";
|
|
4216
6469
|
import { select as select2 } from "@inquirer/prompts";
|
|
6470
|
+
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
4217
6471
|
async function ensureOpenCodeRunning(ctx) {
|
|
4218
6472
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
4219
6473
|
if (healthCheck.healthy) {
|
|
4220
|
-
return {
|
|
6474
|
+
return {
|
|
6475
|
+
port: ctx.port,
|
|
6476
|
+
process: null,
|
|
6477
|
+
version: healthCheck.version ?? null,
|
|
6478
|
+
notReadyReason: null
|
|
6479
|
+
};
|
|
4221
6480
|
}
|
|
4222
6481
|
const runningInstances = await findHealthyOpenCodeInstances();
|
|
4223
6482
|
if (runningInstances.length > 0) {
|
|
@@ -4238,7 +6497,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4238
6497
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
4239
6498
|
console.log(
|
|
4240
6499
|
chalk5.dim(
|
|
4241
|
-
` ${getCliName()} run --
|
|
6500
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
4242
6501
|
)
|
|
4243
6502
|
);
|
|
4244
6503
|
}
|
|
@@ -4258,14 +6517,22 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4258
6517
|
if (!ctx.interactive) {
|
|
4259
6518
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
4260
6519
|
const proc = await startOpenCode(ctx.port);
|
|
4261
|
-
const health = await waitForOpenCodeHealth(ctx.port,
|
|
6520
|
+
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
4262
6521
|
if (!health.healthy) {
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
6522
|
+
return {
|
|
6523
|
+
port: ctx.port,
|
|
6524
|
+
process: proc,
|
|
6525
|
+
version: null,
|
|
6526
|
+
notReadyReason: `it did not become healthy within ${Math.round(ctx.startTimeoutMs / 1e3)}s`
|
|
6527
|
+
};
|
|
4266
6528
|
}
|
|
4267
6529
|
ctx.log(`OpenCode started on port ${ctx.port}${health.version ? ` (v${health.version})` : ""}`);
|
|
4268
|
-
return {
|
|
6530
|
+
return {
|
|
6531
|
+
port: ctx.port,
|
|
6532
|
+
process: proc,
|
|
6533
|
+
version: health.version ?? null,
|
|
6534
|
+
notReadyReason: null
|
|
6535
|
+
};
|
|
4269
6536
|
}
|
|
4270
6537
|
let port = ctx.port;
|
|
4271
6538
|
if (isPortInUse(port)) {
|
|
@@ -4318,122 +6585,15 @@ Port ${port} is already in use.`));
|
|
|
4318
6585
|
if (action === "start") {
|
|
4319
6586
|
const spinner = ora2("Starting OpenCode...").start();
|
|
4320
6587
|
const proc = await startOpenCode(port);
|
|
4321
|
-
const health = await waitForOpenCodeHealth(port,
|
|
6588
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
4322
6589
|
if (!health.healthy) {
|
|
4323
6590
|
spinner.fail("Failed to start OpenCode");
|
|
4324
6591
|
throw new Error("OpenCode failed to start");
|
|
4325
6592
|
}
|
|
4326
6593
|
spinner.stop();
|
|
4327
|
-
return { port, process: proc, version: health.version ?? null };
|
|
4328
|
-
}
|
|
4329
|
-
return { port, process: null, version: null };
|
|
4330
|
-
}
|
|
4331
|
-
|
|
4332
|
-
// src/commands/agent-lookup.ts
|
|
4333
|
-
async function readErrorMessage(response) {
|
|
4334
|
-
const text = await response.text().catch(() => "");
|
|
4335
|
-
if (!text) return response.statusText || void 0;
|
|
4336
|
-
try {
|
|
4337
|
-
const data = JSON.parse(text);
|
|
4338
|
-
const message = data.message ?? data.error;
|
|
4339
|
-
if (typeof message === "string" && message.trim()) {
|
|
4340
|
-
return message;
|
|
4341
|
-
}
|
|
4342
|
-
} catch {
|
|
4343
|
-
}
|
|
4344
|
-
return text.trim() || response.statusText || void 0;
|
|
4345
|
-
}
|
|
4346
|
-
function authFailureHint(apiUrl, serverMessage) {
|
|
4347
|
-
const reason = serverMessage ? `: ${serverMessage}` : "";
|
|
4348
|
-
return `Authentication failed${reason}. Your credentials were rejected by ${apiUrl}. This usually means you logged in against a different environment, or your session expired \u2014 log in again pointing at this endpoint and retry.`;
|
|
4349
|
-
}
|
|
4350
|
-
async function resolveAgentIdFromKey(authHeader) {
|
|
4351
|
-
const apiUrl = getApiUrlConfig();
|
|
4352
|
-
try {
|
|
4353
|
-
const response = await fetch(`${apiUrl}/me`, {
|
|
4354
|
-
headers: { Authorization: authHeader }
|
|
4355
|
-
});
|
|
4356
|
-
if (response.status === 401) {
|
|
4357
|
-
const serverMessage = await readErrorMessage(response);
|
|
4358
|
-
return { error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
4359
|
-
}
|
|
4360
|
-
if (!response.ok) {
|
|
4361
|
-
const serverMessage = await readErrorMessage(response);
|
|
4362
|
-
return {
|
|
4363
|
-
error: `Failed to resolve runner from key (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4364
|
-
};
|
|
4365
|
-
}
|
|
4366
|
-
const data = await response.json();
|
|
4367
|
-
if (data.auth_type === "agent_key" && data.agent_id) {
|
|
4368
|
-
return { agent_id: data.agent_id };
|
|
4369
|
-
}
|
|
4370
|
-
return {
|
|
4371
|
-
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --agent explicitly."
|
|
4372
|
-
};
|
|
4373
|
-
} catch (error2) {
|
|
4374
|
-
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
4375
|
-
return { error: `Failed to resolve runner from key: ${message}` };
|
|
4376
|
-
}
|
|
4377
|
-
}
|
|
4378
|
-
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
4379
|
-
const apiUrl = getApiUrlConfig();
|
|
4380
|
-
try {
|
|
4381
|
-
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
4382
|
-
method: "POST",
|
|
4383
|
-
headers: { Authorization: authHeader }
|
|
4384
|
-
});
|
|
4385
|
-
if (!response.ok) {
|
|
4386
|
-
const serverMessage = await readErrorMessage(response);
|
|
4387
|
-
return {
|
|
4388
|
-
ok: false,
|
|
4389
|
-
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4390
|
-
};
|
|
4391
|
-
}
|
|
4392
|
-
return { ok: true };
|
|
4393
|
-
} catch (error2) {
|
|
4394
|
-
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
4395
|
-
}
|
|
4396
|
-
}
|
|
4397
|
-
async function getAgentInfo(agentId, authHeader) {
|
|
4398
|
-
const apiUrl = getApiUrlConfig();
|
|
4399
|
-
try {
|
|
4400
|
-
const response = await fetch(`${apiUrl}/runners/${agentId}`, {
|
|
4401
|
-
headers: { Authorization: authHeader }
|
|
4402
|
-
});
|
|
4403
|
-
if (response.status === 401) {
|
|
4404
|
-
const serverMessage = await readErrorMessage(response);
|
|
4405
|
-
return { valid: false, error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
4406
|
-
}
|
|
4407
|
-
if (response.status === 403) {
|
|
4408
|
-
const serverMessage = await readErrorMessage(response);
|
|
4409
|
-
return {
|
|
4410
|
-
valid: false,
|
|
4411
|
-
error: serverMessage ?? "You do not have access to this runner (it may belong to a different team or organization)."
|
|
4412
|
-
};
|
|
4413
|
-
}
|
|
4414
|
-
if (response.status === 404) {
|
|
4415
|
-
const serverMessage = await readErrorMessage(response);
|
|
4416
|
-
return { valid: false, error: serverMessage ?? `Runner ${agentId} not found` };
|
|
4417
|
-
}
|
|
4418
|
-
if (!response.ok) {
|
|
4419
|
-
const serverMessage = await readErrorMessage(response);
|
|
4420
|
-
return {
|
|
4421
|
-
valid: false,
|
|
4422
|
-
error: `API error (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4423
|
-
};
|
|
4424
|
-
}
|
|
4425
|
-
const agent = await response.json();
|
|
4426
|
-
if (agent.agent_type !== "local") {
|
|
4427
|
-
return {
|
|
4428
|
-
valid: false,
|
|
4429
|
-
error: `Runner is type '${agent.agent_type}', must be 'local' for CLI connection`
|
|
4430
|
-
};
|
|
4431
|
-
}
|
|
4432
|
-
return { valid: true, agent };
|
|
4433
|
-
} catch (error2) {
|
|
4434
|
-
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
4435
|
-
return { valid: false, error: `Failed to validate runner: ${message}` };
|
|
6594
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
4436
6595
|
}
|
|
6596
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
4437
6597
|
}
|
|
4438
6598
|
|
|
4439
6599
|
// src/commands/run.ts
|
|
@@ -4441,6 +6601,7 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
4441
6601
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
4442
6602
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
4443
6603
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
6604
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
4444
6605
|
function resolveLogLevel(options) {
|
|
4445
6606
|
const accepted = Object.keys(LOG_LEVELS);
|
|
4446
6607
|
const validate = (value, source) => {
|
|
@@ -4464,6 +6625,89 @@ function resolveLogLevel(options) {
|
|
|
4464
6625
|
}
|
|
4465
6626
|
return "info";
|
|
4466
6627
|
}
|
|
6628
|
+
function resolveFileSyncDirectories(raw, homeDir) {
|
|
6629
|
+
const directories = [];
|
|
6630
|
+
for (const entry of raw ?? []) {
|
|
6631
|
+
const trimmed = entry.trim();
|
|
6632
|
+
if (trimmed === "") {
|
|
6633
|
+
throw new Error("--enable-file-sync-to requires a directory path (got an empty value)");
|
|
6634
|
+
}
|
|
6635
|
+
const expanded = trimmed === "~" ? homeDir : trimmed.startsWith("~/") ? join3(homeDir, trimmed.slice(2)) : trimmed;
|
|
6636
|
+
if (!isAbsolute2(expanded)) {
|
|
6637
|
+
throw new Error(`--enable-file-sync-to requires an absolute directory path; got "${entry}"`);
|
|
6638
|
+
}
|
|
6639
|
+
const normalized = resolvePath(expanded);
|
|
6640
|
+
if (parse(normalized).root === normalized) {
|
|
6641
|
+
throw new Error(
|
|
6642
|
+
`--enable-file-sync-to will not allow-list the filesystem root ("${entry}"); name the specific directory the credentials belong in (for example ~/.claude)`
|
|
6643
|
+
);
|
|
6644
|
+
}
|
|
6645
|
+
if (!directories.includes(normalized)) {
|
|
6646
|
+
directories.push(normalized);
|
|
6647
|
+
}
|
|
6648
|
+
}
|
|
6649
|
+
if (directories.length > MAX_FILE_SYNC_DIRECTORIES) {
|
|
6650
|
+
throw new Error(
|
|
6651
|
+
`--enable-file-sync-to accepts at most ${MAX_FILE_SYNC_DIRECTORIES} directories; got ${directories.length}`
|
|
6652
|
+
);
|
|
6653
|
+
}
|
|
6654
|
+
return directories;
|
|
6655
|
+
}
|
|
6656
|
+
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
6657
|
+
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
6658
|
+
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
6659
|
+
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
6660
|
+
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
6661
|
+
let raw;
|
|
6662
|
+
let source;
|
|
6663
|
+
if (options.opencodeStartTimeout !== void 0) {
|
|
6664
|
+
raw = options.opencodeStartTimeout;
|
|
6665
|
+
source = "--opencode-start-timeout";
|
|
6666
|
+
} else if (env[OPENCODE_START_TIMEOUT_ENV] !== void 0 && env[OPENCODE_START_TIMEOUT_ENV] !== "") {
|
|
6667
|
+
raw = env[OPENCODE_START_TIMEOUT_ENV];
|
|
6668
|
+
source = OPENCODE_START_TIMEOUT_ENV;
|
|
6669
|
+
} else {
|
|
6670
|
+
return { timeoutMs: defaultMs, warnings: [] };
|
|
6671
|
+
}
|
|
6672
|
+
const trimmed = raw.trim();
|
|
6673
|
+
const seconds = Number(trimmed);
|
|
6674
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(seconds) && seconds > 0;
|
|
6675
|
+
if (!isPositiveInteger || seconds > MAX_OPENCODE_START_TIMEOUT_SECONDS) {
|
|
6676
|
+
return {
|
|
6677
|
+
timeoutMs: defaultMs,
|
|
6678
|
+
warnings: [
|
|
6679
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer number of seconds (at most ${MAX_OPENCODE_START_TIMEOUT_SECONDS}); using the default ${DEFAULT_OPENCODE_START_TIMEOUT_SECONDS}s`
|
|
6680
|
+
]
|
|
6681
|
+
};
|
|
6682
|
+
}
|
|
6683
|
+
return { timeoutMs: seconds * 1e3, warnings: [] };
|
|
6684
|
+
}
|
|
6685
|
+
var MAX_ACTIVE_SESSIONS_ENV = "EVIDENT_MAX_ACTIVE_SESSIONS";
|
|
6686
|
+
function resolveMaxActiveSessions(options, env = process.env) {
|
|
6687
|
+
let raw;
|
|
6688
|
+
let source;
|
|
6689
|
+
if (options.maxActiveSessions !== void 0) {
|
|
6690
|
+
raw = options.maxActiveSessions;
|
|
6691
|
+
source = "--max-active-sessions";
|
|
6692
|
+
} else if (env[MAX_ACTIVE_SESSIONS_ENV] !== void 0 && env[MAX_ACTIVE_SESSIONS_ENV] !== "") {
|
|
6693
|
+
raw = env[MAX_ACTIVE_SESSIONS_ENV];
|
|
6694
|
+
source = MAX_ACTIVE_SESSIONS_ENV;
|
|
6695
|
+
} else {
|
|
6696
|
+
return { value: void 0, warnings: [] };
|
|
6697
|
+
}
|
|
6698
|
+
const trimmed = raw.trim();
|
|
6699
|
+
const count = Number(trimmed);
|
|
6700
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(count) && count > 0;
|
|
6701
|
+
if (!isPositiveInteger) {
|
|
6702
|
+
return {
|
|
6703
|
+
value: void 0,
|
|
6704
|
+
warnings: [
|
|
6705
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer; using unlimited`
|
|
6706
|
+
]
|
|
6707
|
+
};
|
|
6708
|
+
}
|
|
6709
|
+
return { value: count, warnings: [] };
|
|
6710
|
+
}
|
|
4467
6711
|
function meetsThreshold(state, level) {
|
|
4468
6712
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
4469
6713
|
}
|
|
@@ -4485,6 +6729,10 @@ function log2(state, message, level = "info") {
|
|
|
4485
6729
|
function logActivity(state, entry) {
|
|
4486
6730
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
4487
6731
|
if (!meetsThreshold(state, level)) return;
|
|
6732
|
+
forwardRunnerActivity(
|
|
6733
|
+
{ level, message: entry.message, error: entry.error },
|
|
6734
|
+
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
6735
|
+
);
|
|
4488
6736
|
const fullEntry = {
|
|
4489
6737
|
...entry,
|
|
4490
6738
|
level,
|
|
@@ -4584,23 +6832,46 @@ async function handleAuthError(state, error2) {
|
|
|
4584
6832
|
}
|
|
4585
6833
|
async function driveChannels(state, driver) {
|
|
4586
6834
|
let idlePolls = 0;
|
|
6835
|
+
let idleMs = 0;
|
|
6836
|
+
let consecutiveDrainFailures = 0;
|
|
6837
|
+
let unreachableMs = 0;
|
|
4587
6838
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
6839
|
+
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
4588
6840
|
while (state.running) {
|
|
6841
|
+
const cycleStartedAtMs = performance.now();
|
|
6842
|
+
let idleThisCycle = false;
|
|
6843
|
+
let unreachableThisCycle = false;
|
|
4589
6844
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
4590
6845
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
4591
6846
|
if (state.interactive) displayStatus(state);
|
|
4592
6847
|
await state.connection.reconnectPromise;
|
|
4593
6848
|
}
|
|
6849
|
+
const carriedOverFileSync = driver.fileSyncActivity().inFlight;
|
|
6850
|
+
void driver.syncPendingFiles().catch(
|
|
6851
|
+
(error2) => logActivity(state, {
|
|
6852
|
+
type: "error",
|
|
6853
|
+
error: `Runner file sync failed: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
6854
|
+
})
|
|
6855
|
+
);
|
|
4594
6856
|
try {
|
|
4595
6857
|
const processed = await driver.drainPending();
|
|
6858
|
+
consecutiveDrainFailures = 0;
|
|
6859
|
+
unreachableMs = 0;
|
|
4596
6860
|
state.messageCount += processed;
|
|
4597
6861
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
4598
6862
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
4599
|
-
|
|
6863
|
+
const appliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
6864
|
+
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
6865
|
+
const fileActivity = carriedOverFileSync || filesApplied;
|
|
6866
|
+
lastSeenAppliedFiles = appliedFiles;
|
|
6867
|
+
if (filesApplied) state.claudeUsageRearm?.();
|
|
6868
|
+
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
4600
6869
|
idlePolls = 0;
|
|
6870
|
+
idleMs = 0;
|
|
4601
6871
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
4602
6872
|
} else if (state.idleTimeout !== null) {
|
|
4603
6873
|
idlePolls++;
|
|
6874
|
+
idleThisCycle = true;
|
|
4604
6875
|
if (idlePolls === 1) {
|
|
4605
6876
|
logActivity(state, {
|
|
4606
6877
|
type: "info",
|
|
@@ -4624,21 +6895,44 @@ async function driveChannels(state, driver) {
|
|
|
4624
6895
|
const errorMessage = error2 instanceof Error ? error2.message : String(error2);
|
|
4625
6896
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
4626
6897
|
if (state.interactive) displayStatus(state);
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
if (
|
|
4634
|
-
|
|
6898
|
+
if (driver.hasInFlightWatchers()) {
|
|
6899
|
+
consecutiveDrainFailures = 0;
|
|
6900
|
+
unreachableMs = 0;
|
|
6901
|
+
} else if (state.idleTimeout !== null) {
|
|
6902
|
+
consecutiveDrainFailures++;
|
|
6903
|
+
unreachableThisCycle = true;
|
|
6904
|
+
if (consecutiveDrainFailures === 1) {
|
|
6905
|
+
logActivity(state, {
|
|
6906
|
+
type: "info",
|
|
6907
|
+
message: `Cannot reach Evident, will exit if this persists past the idle timeout (timeout: ${state.idleTimeout}s)...`
|
|
6908
|
+
});
|
|
6909
|
+
if (state.interactive) displayStatus(state);
|
|
6910
|
+
}
|
|
4635
6911
|
}
|
|
4636
6912
|
}
|
|
6913
|
+
await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
|
|
6914
|
+
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
6915
|
+
if (idleThisCycle) idleMs += cycleMs;
|
|
6916
|
+
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
6917
|
+
if (state.idleTimeout !== null && consecutiveDrainFailures >= 2 && unreachableMs > state.idleTimeout * 1e3) {
|
|
6918
|
+
logActivity(state, {
|
|
6919
|
+
type: "info",
|
|
6920
|
+
level: "warn",
|
|
6921
|
+
message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
|
|
6922
|
+
});
|
|
6923
|
+
if (state.interactive) displayStatus(state);
|
|
6924
|
+
break;
|
|
6925
|
+
}
|
|
6926
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
6927
|
+
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
6928
|
+
if (state.interactive) displayStatus(state);
|
|
6929
|
+
break;
|
|
6930
|
+
}
|
|
4637
6931
|
}
|
|
4638
6932
|
}
|
|
4639
6933
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
4640
|
-
async function runSweep(state, driver,
|
|
4641
|
-
const mode = `age=${
|
|
6934
|
+
async function runSweep(state, driver, config) {
|
|
6935
|
+
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
4642
6936
|
try {
|
|
4643
6937
|
const sessions = await listSessions(state.port);
|
|
4644
6938
|
if (sessions === null) {
|
|
@@ -4651,8 +6945,8 @@ async function runSweep(state, driver, config2) {
|
|
|
4651
6945
|
const toDelete = selectSessionsToDelete(
|
|
4652
6946
|
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
4653
6947
|
{
|
|
4654
|
-
maxAgeMs:
|
|
4655
|
-
maxCount:
|
|
6948
|
+
maxAgeMs: config.maxAgeMs,
|
|
6949
|
+
maxCount: config.maxCount,
|
|
4656
6950
|
nowMs: Date.now(),
|
|
4657
6951
|
protectedIds: driver.protectedSessionIds()
|
|
4658
6952
|
}
|
|
@@ -4688,7 +6982,7 @@ async function runSweep(state, driver, config2) {
|
|
|
4688
6982
|
}
|
|
4689
6983
|
}
|
|
4690
6984
|
function scheduleSessionCleanup(state, driver, options) {
|
|
4691
|
-
const
|
|
6985
|
+
const config = resolveSessionCleanupConfig(
|
|
4692
6986
|
{
|
|
4693
6987
|
maxAge: options.sessionCleanupMaxAge,
|
|
4694
6988
|
maxCount: options.sessionCleanupMaxCount,
|
|
@@ -4696,21 +6990,129 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
4696
6990
|
},
|
|
4697
6991
|
process.env
|
|
4698
6992
|
);
|
|
4699
|
-
for (const warning2 of
|
|
6993
|
+
for (const warning2 of config.warnings) {
|
|
4700
6994
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
4701
6995
|
}
|
|
4702
|
-
if (!
|
|
6996
|
+
if (!config.enabled) return;
|
|
4703
6997
|
logActivity(state, {
|
|
4704
6998
|
type: "info",
|
|
4705
|
-
message: `Session cleanup enabled (age=${
|
|
6999
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`
|
|
4706
7000
|
});
|
|
4707
|
-
const interval = setInterval(() => void runSweep(state, driver,
|
|
7001
|
+
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
4708
7002
|
const firstSweep = setTimeout(
|
|
4709
|
-
() => void runSweep(state, driver,
|
|
7003
|
+
() => void runSweep(state, driver, config),
|
|
4710
7004
|
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
4711
7005
|
);
|
|
4712
7006
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
4713
7007
|
}
|
|
7008
|
+
function claudeUsageFailureStreakSuffix(consecutiveFailures) {
|
|
7009
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
7010
|
+
}
|
|
7011
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
7012
|
+
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
7013
|
+
options.claudeUsageReporting,
|
|
7014
|
+
process.env
|
|
7015
|
+
);
|
|
7016
|
+
for (const warning2 of warnings) {
|
|
7017
|
+
logActivity(state, {
|
|
7018
|
+
type: "info",
|
|
7019
|
+
level: "warn",
|
|
7020
|
+
message: `Claude usage reporting: ${warning2}`
|
|
7021
|
+
});
|
|
7022
|
+
}
|
|
7023
|
+
if (mode === "off") {
|
|
7024
|
+
logActivity(state, {
|
|
7025
|
+
type: "info",
|
|
7026
|
+
level: "debug",
|
|
7027
|
+
message: "Claude usage reporting is off (--claude-usage-reporting off)"
|
|
7028
|
+
});
|
|
7029
|
+
return null;
|
|
7030
|
+
}
|
|
7031
|
+
let consecutiveFailures = 0;
|
|
7032
|
+
let armed = false;
|
|
7033
|
+
let rearmRequested = false;
|
|
7034
|
+
const scheduleNextTick = () => {
|
|
7035
|
+
armed = true;
|
|
7036
|
+
rearmRequested = false;
|
|
7037
|
+
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
7038
|
+
};
|
|
7039
|
+
const rearm = () => {
|
|
7040
|
+
if (armed) {
|
|
7041
|
+
rearmRequested = true;
|
|
7042
|
+
return;
|
|
7043
|
+
}
|
|
7044
|
+
rearmRequested = false;
|
|
7045
|
+
armed = true;
|
|
7046
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7047
|
+
};
|
|
7048
|
+
const tick = async (isProbe) => {
|
|
7049
|
+
try {
|
|
7050
|
+
const usage = await getClaudeUsage();
|
|
7051
|
+
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
7052
|
+
if (result.ok) {
|
|
7053
|
+
if (consecutiveFailures > 0) {
|
|
7054
|
+
logActivity(state, {
|
|
7055
|
+
type: "info",
|
|
7056
|
+
level: "info",
|
|
7057
|
+
message: "Claude usage reporting recovered"
|
|
7058
|
+
});
|
|
7059
|
+
}
|
|
7060
|
+
consecutiveFailures = 0;
|
|
7061
|
+
logActivity(state, {
|
|
7062
|
+
type: "info",
|
|
7063
|
+
level: "debug",
|
|
7064
|
+
message: "Reported Claude usage to Evident"
|
|
7065
|
+
});
|
|
7066
|
+
} else {
|
|
7067
|
+
consecutiveFailures++;
|
|
7068
|
+
logActivity(state, {
|
|
7069
|
+
type: "info",
|
|
7070
|
+
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7071
|
+
message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
|
|
7072
|
+
});
|
|
7073
|
+
}
|
|
7074
|
+
scheduleNextTick();
|
|
7075
|
+
} catch (error2) {
|
|
7076
|
+
if (error2 instanceof ClaudeUsageError && isLocalCredentialProblem(error2)) {
|
|
7077
|
+
if (mode === "on") {
|
|
7078
|
+
logActivity(state, {
|
|
7079
|
+
type: "info",
|
|
7080
|
+
level: "warn",
|
|
7081
|
+
message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
|
|
7082
|
+
});
|
|
7083
|
+
scheduleNextTick();
|
|
7084
|
+
} else if (isProbe) {
|
|
7085
|
+
logActivity(state, {
|
|
7086
|
+
type: "info",
|
|
7087
|
+
level: "debug",
|
|
7088
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
7089
|
+
});
|
|
7090
|
+
armed = false;
|
|
7091
|
+
if (rearmRequested) rearm();
|
|
7092
|
+
} else {
|
|
7093
|
+
logActivity(state, {
|
|
7094
|
+
type: "info",
|
|
7095
|
+
level: "debug",
|
|
7096
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
7097
|
+
});
|
|
7098
|
+
scheduleNextTick();
|
|
7099
|
+
}
|
|
7100
|
+
} else {
|
|
7101
|
+
consecutiveFailures++;
|
|
7102
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
7103
|
+
logActivity(state, {
|
|
7104
|
+
type: "info",
|
|
7105
|
+
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
7106
|
+
message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
|
|
7107
|
+
});
|
|
7108
|
+
scheduleNextTick();
|
|
7109
|
+
}
|
|
7110
|
+
}
|
|
7111
|
+
};
|
|
7112
|
+
armed = true;
|
|
7113
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
7114
|
+
return rearm;
|
|
7115
|
+
}
|
|
4714
7116
|
async function notifyOffline(state) {
|
|
4715
7117
|
if (!state.agentId || !state.authHeader) return;
|
|
4716
7118
|
if (!state.connected) {
|
|
@@ -4728,13 +7130,29 @@ async function notifyOffline(state) {
|
|
|
4728
7130
|
if (state.interactive) displayStatus(state);
|
|
4729
7131
|
}
|
|
4730
7132
|
}
|
|
7133
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
7134
|
+
const startedAt = Date.now();
|
|
7135
|
+
try {
|
|
7136
|
+
return await run2();
|
|
7137
|
+
} finally {
|
|
7138
|
+
const elapsedMs = Date.now() - startedAt;
|
|
7139
|
+
durations[name] = elapsedMs;
|
|
7140
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
4731
7143
|
async function cleanup(state, opts = {}) {
|
|
7144
|
+
const durations = {};
|
|
4732
7145
|
state.running = false;
|
|
4733
7146
|
for (const timer of state.sessionCleanupTimers) {
|
|
4734
7147
|
clearInterval(timer);
|
|
4735
7148
|
clearTimeout(timer);
|
|
4736
7149
|
}
|
|
4737
7150
|
state.sessionCleanupTimers = [];
|
|
7151
|
+
if (state.claudeUsageTimer) {
|
|
7152
|
+
clearTimeout(state.claudeUsageTimer);
|
|
7153
|
+
state.claudeUsageTimer = null;
|
|
7154
|
+
}
|
|
7155
|
+
state.claudeUsageRearm = null;
|
|
4738
7156
|
if (opts.graceful && state.channelDriver) {
|
|
4739
7157
|
state.channelDriver.stop();
|
|
4740
7158
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -4742,7 +7160,13 @@ async function cleanup(state, opts = {}) {
|
|
|
4742
7160
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
4743
7161
|
displayStatus(state);
|
|
4744
7162
|
}
|
|
4745
|
-
const
|
|
7163
|
+
const driver = state.channelDriver;
|
|
7164
|
+
const settled = await timeShutdownPhase(
|
|
7165
|
+
state,
|
|
7166
|
+
durations,
|
|
7167
|
+
"drain",
|
|
7168
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
7169
|
+
);
|
|
4746
7170
|
if (!settled) {
|
|
4747
7171
|
logActivity(state, {
|
|
4748
7172
|
type: "info",
|
|
@@ -4751,13 +7175,15 @@ async function cleanup(state, opts = {}) {
|
|
|
4751
7175
|
if (state.interactive) displayStatus(state);
|
|
4752
7176
|
}
|
|
4753
7177
|
}
|
|
4754
|
-
await notifyOffline(state);
|
|
7178
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
4755
7179
|
if (state.connection) {
|
|
4756
|
-
state.connection
|
|
7180
|
+
const connection = state.connection;
|
|
7181
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
4757
7182
|
state.connection = null;
|
|
4758
7183
|
}
|
|
4759
7184
|
if (state.opencodeProcess) {
|
|
4760
|
-
|
|
7185
|
+
const opencodeProcess = state.opencodeProcess;
|
|
7186
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
4761
7187
|
if (state.interactive) {
|
|
4762
7188
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
4763
7189
|
displayStatus(state);
|
|
@@ -4766,12 +7192,15 @@ async function cleanup(state, opts = {}) {
|
|
|
4766
7192
|
}
|
|
4767
7193
|
state.opencodeProcess = null;
|
|
4768
7194
|
}
|
|
7195
|
+
return durations;
|
|
4769
7196
|
}
|
|
4770
7197
|
async function run(options) {
|
|
4771
7198
|
const interactive = isInteractive(options.json);
|
|
4772
7199
|
let logLevel;
|
|
7200
|
+
let fileSyncDirectories;
|
|
4773
7201
|
try {
|
|
4774
7202
|
logLevel = resolveLogLevel(options);
|
|
7203
|
+
fileSyncDirectories = resolveFileSyncDirectories(options.enableFileSyncTo, homedir3());
|
|
4775
7204
|
} catch (error2) {
|
|
4776
7205
|
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
4777
7206
|
if (options.json) {
|
|
@@ -4804,8 +7233,16 @@ async function run(options) {
|
|
|
4804
7233
|
messageCount: 0,
|
|
4805
7234
|
lastProxiedActivityAt: null,
|
|
4806
7235
|
sessionCleanupTimers: [],
|
|
7236
|
+
claudeUsageTimer: null,
|
|
7237
|
+
claudeUsageRearm: null,
|
|
4807
7238
|
authHeader: ""
|
|
4808
7239
|
};
|
|
7240
|
+
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
7241
|
+
if (fileSyncDirectories.length > 0) {
|
|
7242
|
+
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
7243
|
+
} else {
|
|
7244
|
+
log2(state, "File sync is disabled (no --enable-file-sync-to given)", "debug");
|
|
7245
|
+
}
|
|
4809
7246
|
if (!options.runner && options.agent) {
|
|
4810
7247
|
telemetry.info(
|
|
4811
7248
|
EventTypes.DEPRECATED_AGENT_FLAG_USED,
|
|
@@ -4829,14 +7266,38 @@ async function run(options) {
|
|
|
4829
7266
|
const handleSignal = async () => {
|
|
4830
7267
|
if (state.shuttingDown) return;
|
|
4831
7268
|
state.shuttingDown = true;
|
|
7269
|
+
const shutdownStartedAt = Date.now();
|
|
4832
7270
|
if (state.interactive) {
|
|
4833
7271
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
4834
7272
|
displayStatus(state);
|
|
4835
7273
|
} else {
|
|
4836
7274
|
log2(state, "Shutting down...");
|
|
4837
7275
|
}
|
|
4838
|
-
await cleanup(state, { graceful: true });
|
|
4839
|
-
|
|
7276
|
+
const durations = await cleanup(state, { graceful: true });
|
|
7277
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
7278
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
7279
|
+
let timer;
|
|
7280
|
+
const flushed = shutdownTelemetry().then(
|
|
7281
|
+
() => true,
|
|
7282
|
+
(error2) => {
|
|
7283
|
+
log2(
|
|
7284
|
+
state,
|
|
7285
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
7286
|
+
"warn"
|
|
7287
|
+
);
|
|
7288
|
+
return true;
|
|
7289
|
+
}
|
|
7290
|
+
);
|
|
7291
|
+
const timedOut = new Promise((resolve3) => {
|
|
7292
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
7293
|
+
});
|
|
7294
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
7295
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
7296
|
+
}
|
|
7297
|
+
clearTimeout(timer);
|
|
7298
|
+
});
|
|
7299
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
7300
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
4840
7301
|
process.exit(0);
|
|
4841
7302
|
};
|
|
4842
7303
|
process.on("SIGINT", handleSignal);
|
|
@@ -4853,6 +7314,7 @@ async function run(options) {
|
|
|
4853
7314
|
console.log(chalk6.dim("Or run `evident login` for interactive authentication"));
|
|
4854
7315
|
blank();
|
|
4855
7316
|
process.exit(1);
|
|
7317
|
+
return;
|
|
4856
7318
|
}
|
|
4857
7319
|
blank();
|
|
4858
7320
|
console.log(chalk6.yellow("You are not logged in to Evident."));
|
|
@@ -4897,6 +7359,7 @@ async function run(options) {
|
|
|
4897
7359
|
} else {
|
|
4898
7360
|
printError(resolved.error || "Failed to resolve runner ID from key");
|
|
4899
7361
|
process.exit(1);
|
|
7362
|
+
return;
|
|
4900
7363
|
}
|
|
4901
7364
|
} else {
|
|
4902
7365
|
printError(
|
|
@@ -4910,6 +7373,7 @@ async function run(options) {
|
|
|
4910
7373
|
);
|
|
4911
7374
|
blank();
|
|
4912
7375
|
process.exit(1);
|
|
7376
|
+
return;
|
|
4913
7377
|
}
|
|
4914
7378
|
}
|
|
4915
7379
|
telemetry.info(
|
|
@@ -4950,40 +7414,71 @@ async function run(options) {
|
|
|
4950
7414
|
}
|
|
4951
7415
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
4952
7416
|
state.agentName = validation.agent.name;
|
|
7417
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
7418
|
+
if (microvmId) {
|
|
7419
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
7420
|
+
if (reported.ok) {
|
|
7421
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
7422
|
+
} else {
|
|
7423
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
7424
|
+
log2(state, message, "warn");
|
|
7425
|
+
if (state.interactive && !state.json) {
|
|
7426
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
7427
|
+
}
|
|
7428
|
+
}
|
|
7429
|
+
} else {
|
|
7430
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
7431
|
+
}
|
|
7432
|
+
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
7433
|
+
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
7434
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
7435
|
+
}
|
|
7436
|
+
const { value: maxActiveSessions, warnings: maxActiveSessionsWarnings } = resolveMaxActiveSessions(options, process.env);
|
|
7437
|
+
for (const warning2 of maxActiveSessionsWarnings) {
|
|
7438
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
7439
|
+
}
|
|
4953
7440
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
4954
7441
|
try {
|
|
4955
7442
|
const oc = await ensureOpenCodeRunning({
|
|
4956
7443
|
port: state.port,
|
|
4957
7444
|
interactive: state.interactive,
|
|
4958
7445
|
agentId: state.agentId,
|
|
4959
|
-
log: (message) => log2(state, message)
|
|
7446
|
+
log: (message) => log2(state, message),
|
|
7447
|
+
startTimeoutMs: opencodeStartTimeoutMs
|
|
4960
7448
|
});
|
|
4961
7449
|
state.port = oc.port;
|
|
4962
7450
|
state.opencodeProcess = oc.process;
|
|
4963
7451
|
state.opencodeVersion = oc.version;
|
|
4964
|
-
state.opencodeConnected = oc.
|
|
7452
|
+
state.opencodeConnected = oc.notReadyReason === null;
|
|
4965
7453
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
4966
7454
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
7455
|
+
if (!state.interactive && oc.notReadyReason !== null) {
|
|
7456
|
+
const message = `OpenCode is not ready on port ${state.port}: ${oc.notReadyReason}. The runner will still come online, but messages will fail until opencode answers \u2014 raise the wait with --opencode-start-timeout <seconds> (env ${OPENCODE_START_TIMEOUT_ENV}).`;
|
|
7457
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
7458
|
+
} else {
|
|
7459
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
7460
|
+
if (versionWarning) {
|
|
7461
|
+
log2(state, versionWarning, "warn");
|
|
7462
|
+
if (state.interactive && !state.json) {
|
|
7463
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
7464
|
+
}
|
|
4972
7465
|
}
|
|
4973
|
-
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
|
|
4980
|
-
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
7466
|
+
const noProviderWarning = buildNoProviderWarning(
|
|
7467
|
+
await hasAnyConfiguredProvider(state.port)
|
|
7468
|
+
);
|
|
7469
|
+
if (noProviderWarning) {
|
|
7470
|
+
log2(state, noProviderWarning, "warn");
|
|
7471
|
+
if (state.interactive && !state.json) {
|
|
7472
|
+
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
7473
|
+
blank();
|
|
7474
|
+
console.log(chalk6.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
7475
|
+
console.log(
|
|
7476
|
+
chalk6.dim(
|
|
7477
|
+
`Run ${chalk6.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
7478
|
+
)
|
|
7479
|
+
);
|
|
7480
|
+
blank();
|
|
7481
|
+
}
|
|
4987
7482
|
}
|
|
4988
7483
|
}
|
|
4989
7484
|
} catch (error2) {
|
|
@@ -4998,6 +7493,11 @@ async function run(options) {
|
|
|
4998
7493
|
getAuthHeader: () => state.authHeader,
|
|
4999
7494
|
conversationFilter: state.conversationFilter,
|
|
5000
7495
|
stuckQueuedMs: CHANNEL_STUCK_QUEUED_MS,
|
|
7496
|
+
// #559: the `--enable-file-sync-to` allow-list. Empty ⇒ pending files are
|
|
7497
|
+
// REJECTED with `file_sync_disabled` on the ack, not silently ignored.
|
|
7498
|
+
fileSyncDirectories,
|
|
7499
|
+
homeDir: homedir3(),
|
|
7500
|
+
maxActiveSessions,
|
|
5001
7501
|
log: (entry) => (
|
|
5002
7502
|
// Thread the driver's real level straight through so `debug`/`warn`
|
|
5003
7503
|
// survive the sink filter (they no longer collapse to info). `type`
|
|
@@ -5024,6 +7524,18 @@ async function run(options) {
|
|
|
5024
7524
|
type: "info",
|
|
5025
7525
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
5026
7526
|
});
|
|
7527
|
+
if (options.tunnelReadyFile) {
|
|
7528
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
7529
|
+
if (marker.ok) {
|
|
7530
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
7531
|
+
} else {
|
|
7532
|
+
log2(
|
|
7533
|
+
state,
|
|
7534
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
7535
|
+
"error"
|
|
7536
|
+
);
|
|
7537
|
+
}
|
|
7538
|
+
}
|
|
5027
7539
|
emitAgentConnected(state.agentId, {
|
|
5028
7540
|
port: state.port,
|
|
5029
7541
|
cli_version: getCliVersion(),
|
|
@@ -5079,6 +7591,12 @@ async function run(options) {
|
|
|
5079
7591
|
onDrainPing: () => {
|
|
5080
7592
|
if (!state.running) return;
|
|
5081
7593
|
logActivity(state, { type: "info", message: "Drain ping received \u2014 draining" });
|
|
7594
|
+
void channelDriver.syncPendingFiles().catch(
|
|
7595
|
+
(error2) => logActivity(state, {
|
|
7596
|
+
type: "error",
|
|
7597
|
+
error: `Runner file sync failed on ping: ${error2 instanceof Error ? error2.message : String(error2)}`
|
|
7598
|
+
})
|
|
7599
|
+
);
|
|
5082
7600
|
channelDriver.drainPending().then((processed) => {
|
|
5083
7601
|
if (processed > 0) {
|
|
5084
7602
|
state.messageCount += processed;
|
|
@@ -5108,6 +7626,7 @@ async function run(options) {
|
|
|
5108
7626
|
throw error2;
|
|
5109
7627
|
}
|
|
5110
7628
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
7629
|
+
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
5111
7630
|
if (!interactive || state.json) {
|
|
5112
7631
|
log2(state, "Driving channel messages...");
|
|
5113
7632
|
}
|
|
@@ -5162,18 +7681,40 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
5162
7681
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
5163
7682
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
5164
7683
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
5165
|
-
program.command("
|
|
7684
|
+
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
7685
|
+
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
7686
|
+
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
7687
|
+
"-a, --agent [id]",
|
|
7688
|
+
"Deprecated alias for --runner (still supported; --runner wins if both are given)"
|
|
7689
|
+
).option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
5166
7690
|
"--log-level <level>",
|
|
5167
7691
|
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
5168
|
-
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
7692
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
7693
|
+
"--opencode-start-timeout <seconds>",
|
|
7694
|
+
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
7695
|
+
).option("--json", "Output in JSON format").option(
|
|
5169
7696
|
"--session-cleanup-max-age <duration>",
|
|
5170
7697
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
5171
7698
|
).option(
|
|
5172
7699
|
"--session-cleanup-max-count <n>",
|
|
5173
7700
|
"Keep only the newest N OpenCode sessions. Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_COUNT"
|
|
7701
|
+
).option(
|
|
7702
|
+
"--max-active-sessions <n>",
|
|
7703
|
+
"Cap how many sessions this runner works on at once (default: unlimited). Env: EVIDENT_MAX_ACTIVE_SESSIONS"
|
|
5174
7704
|
).option(
|
|
5175
7705
|
"--session-cleanup-interval <duration>",
|
|
5176
7706
|
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
7707
|
+
).option(
|
|
7708
|
+
"--claude-usage-reporting <mode>",
|
|
7709
|
+
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
7710
|
+
).option(
|
|
7711
|
+
"--enable-file-sync-to <dir>",
|
|
7712
|
+
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
7713
|
+
(value, previous) => previous.concat([value]),
|
|
7714
|
+
[]
|
|
7715
|
+
).option(
|
|
7716
|
+
"--tunnel-ready-file <path>",
|
|
7717
|
+
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
5177
7718
|
).action(
|
|
5178
7719
|
(options) => {
|
|
5179
7720
|
run({
|
|
@@ -5186,11 +7727,22 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5186
7727
|
verbose: options.verbose,
|
|
5187
7728
|
conversation: options.conversation,
|
|
5188
7729
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
7730
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
7731
|
+
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
7732
|
+
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
5189
7733
|
json: options.json,
|
|
5190
7734
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
5191
7735
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
5192
7736
|
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
5193
|
-
|
|
7737
|
+
maxActiveSessions: options.maxActiveSessions,
|
|
7738
|
+
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
7739
|
+
// Raw string — the resolver in run.ts single-sources parsing
|
|
7740
|
+
// (resolveClaudeUsageReportingMode).
|
|
7741
|
+
claudeUsageReporting: options.claudeUsageReporting,
|
|
7742
|
+
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
7743
|
+
// resolveFileSyncDirectories.
|
|
7744
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
7745
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
5194
7746
|
});
|
|
5195
7747
|
}
|
|
5196
7748
|
);
|