@evident-ai/cli 3.1.1-dev.341ff6c → 3.1.1-dev.3583438
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -0
- package/dist/index.js +1908 -393
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -38,17 +38,39 @@ function getApiUrl() {
|
|
|
38
38
|
function getTunnelUrl() {
|
|
39
39
|
return tunnelOverride ?? process.env.EVIDENT_TUNNEL_URL ?? defaults.tunnelUrl;
|
|
40
40
|
}
|
|
41
|
-
var config = new Conf({
|
|
42
|
-
projectName: "evident",
|
|
43
|
-
projectSuffix: "",
|
|
44
|
-
defaults
|
|
45
|
-
});
|
|
46
41
|
var credentials = new Conf({
|
|
47
42
|
projectName: "evident",
|
|
48
43
|
projectSuffix: "",
|
|
49
44
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
45
|
+
defaults: {},
|
|
46
|
+
configFileMode: 384
|
|
51
47
|
});
|
|
48
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
49
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
50
|
+
var permissionWarningEmitted = false;
|
|
51
|
+
function hardenCredentialsPermissions() {
|
|
52
|
+
if (process.platform === "win32") {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const file = credentials.path;
|
|
56
|
+
for (const [path, mode] of [
|
|
57
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
58
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
59
|
+
]) {
|
|
60
|
+
try {
|
|
61
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
62
|
+
chmodSync(path, mode);
|
|
63
|
+
}
|
|
64
|
+
} catch (err) {
|
|
65
|
+
if (!permissionWarningEmitted) {
|
|
66
|
+
permissionWarningEmitted = true;
|
|
67
|
+
console.error(
|
|
68
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
52
74
|
function getApiUrlConfig() {
|
|
53
75
|
return getApiUrl();
|
|
54
76
|
}
|
|
@@ -59,6 +81,7 @@ function credentialsKey() {
|
|
|
59
81
|
return getApiUrl();
|
|
60
82
|
}
|
|
61
83
|
function getCredentials() {
|
|
84
|
+
hardenCredentialsPermissions();
|
|
62
85
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
86
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
87
|
}
|
|
@@ -70,14 +93,17 @@ function setCredentials(creds) {
|
|
|
70
93
|
expiresAt: creds.expiresAt
|
|
71
94
|
};
|
|
72
95
|
credentials.set("byEndpoint", byEndpoint);
|
|
96
|
+
hardenCredentialsPermissions();
|
|
73
97
|
}
|
|
74
98
|
function clearCredentials() {
|
|
75
99
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
100
|
delete byEndpoint[credentialsKey()];
|
|
77
101
|
credentials.set("byEndpoint", byEndpoint);
|
|
102
|
+
hardenCredentialsPermissions();
|
|
78
103
|
}
|
|
79
104
|
function clearAllCredentials() {
|
|
80
105
|
credentials.clear();
|
|
106
|
+
hardenCredentialsPermissions();
|
|
81
107
|
}
|
|
82
108
|
function getCliName() {
|
|
83
109
|
const argv1 = process.argv[1] || "";
|
|
@@ -236,16 +262,28 @@ async function getToken() {
|
|
|
236
262
|
}
|
|
237
263
|
return null;
|
|
238
264
|
}
|
|
265
|
+
function toError(err) {
|
|
266
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
267
|
+
}
|
|
239
268
|
async function deleteToken(options = {}) {
|
|
240
269
|
const keytar = await getKeytar();
|
|
270
|
+
const failures = [];
|
|
241
271
|
if (keytar) {
|
|
242
272
|
if (options.all) {
|
|
243
|
-
|
|
273
|
+
let accounts = [];
|
|
274
|
+
try {
|
|
275
|
+
accounts = await keytar.findCredentials(SERVICE_NAME);
|
|
276
|
+
} catch (err) {
|
|
277
|
+
failures.push({ type: "enumerate", error: toError(err) });
|
|
278
|
+
}
|
|
244
279
|
await Promise.all(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
280
|
+
accounts.map(async (entry) => {
|
|
281
|
+
try {
|
|
282
|
+
await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
283
|
+
} catch (err) {
|
|
284
|
+
failures.push({ type: "delete", account: entry.account, error: toError(err) });
|
|
285
|
+
}
|
|
286
|
+
})
|
|
249
287
|
);
|
|
250
288
|
} else {
|
|
251
289
|
await keytar.deletePassword(SERVICE_NAME, keychainAccount());
|
|
@@ -256,6 +294,7 @@ async function deleteToken(options = {}) {
|
|
|
256
294
|
} else {
|
|
257
295
|
clearCredentials();
|
|
258
296
|
}
|
|
297
|
+
return { failures };
|
|
259
298
|
}
|
|
260
299
|
|
|
261
300
|
// src/utils/ui.ts
|
|
@@ -373,7 +412,10 @@ async function deviceFlowLogin(options) {
|
|
|
373
412
|
}
|
|
374
413
|
async function tokenLogin() {
|
|
375
414
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
415
|
+
console.log("Create a token under Settings \u2192 CLI tokens in the dashboard, then paste it below.");
|
|
416
|
+
console.log(
|
|
417
|
+
"(Alternatively, run `evident login` on a machine with a browser, or set EVIDENT_TOKEN for CI.)"
|
|
418
|
+
);
|
|
377
419
|
blank();
|
|
378
420
|
process.stdout.write("Paste token: ");
|
|
379
421
|
const token = await new Promise((resolve3) => {
|
|
@@ -397,13 +439,22 @@ async function tokenLogin() {
|
|
|
397
439
|
printError("No token provided.");
|
|
398
440
|
process.exit(1);
|
|
399
441
|
}
|
|
442
|
+
await validateAndStoreToken(token);
|
|
443
|
+
}
|
|
444
|
+
async function validateAndStoreToken(token) {
|
|
400
445
|
const spinner = ora("Validating token...").start();
|
|
401
446
|
try {
|
|
402
|
-
const result = await api.
|
|
447
|
+
const result = await api.get("/me", {
|
|
448
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
449
|
+
});
|
|
450
|
+
if (!result.user) {
|
|
451
|
+
throw new Error(
|
|
452
|
+
"This token is not a user login (e.g. a runner key). Paste a CLI token instead."
|
|
453
|
+
);
|
|
454
|
+
}
|
|
403
455
|
await storeToken({
|
|
404
456
|
token,
|
|
405
|
-
user: result.user
|
|
406
|
-
expiresAt: result.expires_at
|
|
457
|
+
user: { email: result.user.email }
|
|
407
458
|
});
|
|
408
459
|
spinner.stop();
|
|
409
460
|
printSuccess(`Logged in as ${chalk2.bold(result.user.email)}`);
|
|
@@ -423,47 +474,522 @@ async function login(options) {
|
|
|
423
474
|
}
|
|
424
475
|
|
|
425
476
|
// src/commands/logout.ts
|
|
477
|
+
function describeFailure(failure) {
|
|
478
|
+
if (failure.type === "enumerate") {
|
|
479
|
+
return `could not list stored keychain entries (${failure.error.message})`;
|
|
480
|
+
}
|
|
481
|
+
return `${failure.account} (${failure.error.message})`;
|
|
482
|
+
}
|
|
426
483
|
async function logout(options = {}) {
|
|
427
484
|
if (options.all) {
|
|
428
|
-
await deleteToken({ all: true });
|
|
485
|
+
const result = await deleteToken({ all: true });
|
|
486
|
+
if (result.failures.length > 0) {
|
|
487
|
+
printError(
|
|
488
|
+
`Failed to fully clear your keychain: ${result.failures.map(describeFailure).join("; ")}. Your local credentials file was cleared, but stale keychain entries may remain \u2014 run \`evident logout --all\` again, or remove them manually from your OS keychain / credential manager.`
|
|
489
|
+
);
|
|
490
|
+
process.exitCode = 1;
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
429
493
|
printSuccess("Logged out of all endpoints.");
|
|
430
494
|
return;
|
|
431
495
|
}
|
|
432
496
|
const credentials2 = await getToken();
|
|
433
497
|
if (!credentials2) {
|
|
434
|
-
printWarning(`You are not logged in to ${getApiUrlConfig()}.`);
|
|
435
|
-
return;
|
|
498
|
+
printWarning(`You are not logged in to ${getApiUrlConfig()}.`);
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
await deleteToken();
|
|
502
|
+
printSuccess(`Logged out of ${getApiUrlConfig()}.`);
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// src/commands/whoami.ts
|
|
506
|
+
import chalk3 from "chalk";
|
|
507
|
+
async function whoami() {
|
|
508
|
+
const apiUrl = getApiUrlConfig();
|
|
509
|
+
const credentials2 = await getToken();
|
|
510
|
+
if (!credentials2) {
|
|
511
|
+
printError(`Not logged in to ${apiUrl}. Run the \`login\` command to authenticate.`);
|
|
512
|
+
process.exit(1);
|
|
513
|
+
}
|
|
514
|
+
blank();
|
|
515
|
+
console.log(keyValue("Endpoint", apiUrl));
|
|
516
|
+
console.log(keyValue("User", chalk3.bold(credentials2.user.email)));
|
|
517
|
+
if (credentials2.user.id) {
|
|
518
|
+
console.log(keyValue("User ID", credentials2.user.id));
|
|
519
|
+
}
|
|
520
|
+
if (credentials2.expiresAt) {
|
|
521
|
+
const expiresAt = new Date(credentials2.expiresAt);
|
|
522
|
+
const now = /* @__PURE__ */ new Date();
|
|
523
|
+
if (expiresAt < now) {
|
|
524
|
+
console.log(keyValue("Status", chalk3.red("Token expired")));
|
|
525
|
+
} else {
|
|
526
|
+
const daysRemaining = Math.ceil(
|
|
527
|
+
(expiresAt.getTime() - now.getTime()) / (1e3 * 60 * 60 * 24)
|
|
528
|
+
);
|
|
529
|
+
console.log(keyValue("Expires", `${daysRemaining} days`));
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
blank();
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// src/lib/auth.ts
|
|
536
|
+
async function getAuthCredentials() {
|
|
537
|
+
const runnerKey = process.env.EVIDENT_RUNNER_KEY;
|
|
538
|
+
const agentKey = process.env.EVIDENT_AGENT_KEY;
|
|
539
|
+
if (runnerKey) {
|
|
540
|
+
return {
|
|
541
|
+
token: runnerKey,
|
|
542
|
+
authType: "agent_key",
|
|
543
|
+
keySource: "runner_key",
|
|
544
|
+
notice: agentKey ? "Both EVIDENT_RUNNER_KEY and EVIDENT_AGENT_KEY are set; using EVIDENT_RUNNER_KEY." : void 0
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
if (agentKey) {
|
|
548
|
+
return { token: agentKey, authType: "agent_key", keySource: "agent_key" };
|
|
549
|
+
}
|
|
550
|
+
const userToken = process.env.EVIDENT_TOKEN;
|
|
551
|
+
if (userToken) {
|
|
552
|
+
return { token: userToken, authType: "bearer" };
|
|
553
|
+
}
|
|
554
|
+
const keychainCreds = await getToken();
|
|
555
|
+
if (keychainCreds) {
|
|
556
|
+
return {
|
|
557
|
+
token: keychainCreds.token,
|
|
558
|
+
authType: "bearer",
|
|
559
|
+
user: keychainCreds.user
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
return null;
|
|
563
|
+
}
|
|
564
|
+
function getAuthHeader(credentials2) {
|
|
565
|
+
if (credentials2.authType === "agent_key") {
|
|
566
|
+
return `SandboxKey ${credentials2.token}`;
|
|
567
|
+
}
|
|
568
|
+
return `Bearer ${credentials2.token}`;
|
|
569
|
+
}
|
|
570
|
+
function isInteractive(jsonOutput) {
|
|
571
|
+
if (jsonOutput) return false;
|
|
572
|
+
if (process.env.CI) return false;
|
|
573
|
+
if (process.env.GITHUB_ACTIONS) return false;
|
|
574
|
+
if (!process.stdin.isTTY) return false;
|
|
575
|
+
return true;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// src/commands/agent-lookup.ts
|
|
579
|
+
async function readErrorMessage(response) {
|
|
580
|
+
const text = await response.text().catch(() => "");
|
|
581
|
+
if (!text) return response.statusText || void 0;
|
|
582
|
+
try {
|
|
583
|
+
const data = JSON.parse(text);
|
|
584
|
+
const message = data.message ?? data.error;
|
|
585
|
+
if (typeof message === "string" && message.trim()) {
|
|
586
|
+
return message;
|
|
587
|
+
}
|
|
588
|
+
} catch {
|
|
589
|
+
}
|
|
590
|
+
return text.trim() || response.statusText || void 0;
|
|
591
|
+
}
|
|
592
|
+
function authFailureHint(apiUrl, serverMessage) {
|
|
593
|
+
const reason = serverMessage ? `: ${serverMessage}` : "";
|
|
594
|
+
return `Authentication failed${reason}. Your credentials were rejected by ${apiUrl}. This usually means you logged in against a different environment, or your session expired \u2014 log in again pointing at this endpoint and retry.`;
|
|
595
|
+
}
|
|
596
|
+
async function resolveAgentIdFromKey(authHeader) {
|
|
597
|
+
const apiUrl = getApiUrlConfig();
|
|
598
|
+
try {
|
|
599
|
+
const response = await fetch(`${apiUrl}/me`, {
|
|
600
|
+
headers: { Authorization: authHeader }
|
|
601
|
+
});
|
|
602
|
+
if (response.status === 401) {
|
|
603
|
+
const serverMessage = await readErrorMessage(response);
|
|
604
|
+
return { error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
605
|
+
}
|
|
606
|
+
if (!response.ok) {
|
|
607
|
+
const serverMessage = await readErrorMessage(response);
|
|
608
|
+
return {
|
|
609
|
+
error: `Failed to resolve runner from key (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
const data = await response.json();
|
|
613
|
+
if (data.auth_type === "agent_key" && data.agent_id) {
|
|
614
|
+
return { agent_id: data.agent_id };
|
|
615
|
+
}
|
|
616
|
+
return {
|
|
617
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
618
|
+
};
|
|
619
|
+
} catch (error2) {
|
|
620
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
621
|
+
return { error: `Failed to resolve runner from key: ${message}` };
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
625
|
+
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
626
|
+
const apiUrl = getApiUrlConfig();
|
|
627
|
+
try {
|
|
628
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
629
|
+
method: "POST",
|
|
630
|
+
headers: { Authorization: authHeader },
|
|
631
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
632
|
+
});
|
|
633
|
+
if (!response.ok) {
|
|
634
|
+
const serverMessage = await readErrorMessage(response);
|
|
635
|
+
return {
|
|
636
|
+
ok: false,
|
|
637
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
return { ok: true };
|
|
641
|
+
} catch (error2) {
|
|
642
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
function describeBestEffortError(error2) {
|
|
646
|
+
const name = error2?.name;
|
|
647
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
648
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
649
|
+
}
|
|
650
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
651
|
+
}
|
|
652
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
653
|
+
try {
|
|
654
|
+
const apiUrl = getApiUrlConfig();
|
|
655
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
656
|
+
method: "POST",
|
|
657
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
658
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
659
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
660
|
+
});
|
|
661
|
+
if (!response.ok) {
|
|
662
|
+
const serverMessage = await readErrorMessage(response);
|
|
663
|
+
return {
|
|
664
|
+
ok: false,
|
|
665
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
return { ok: true };
|
|
669
|
+
} catch (error2) {
|
|
670
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
function toReportedWindow(window) {
|
|
674
|
+
if (!window) return null;
|
|
675
|
+
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
676
|
+
}
|
|
677
|
+
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
678
|
+
try {
|
|
679
|
+
const apiUrl = getApiUrlConfig();
|
|
680
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/claude-usage`, {
|
|
681
|
+
method: "POST",
|
|
682
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
683
|
+
body: JSON.stringify({
|
|
684
|
+
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
685
|
+
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
686
|
+
}),
|
|
687
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
688
|
+
});
|
|
689
|
+
if (!response.ok) {
|
|
690
|
+
const serverMessage = await readErrorMessage(response);
|
|
691
|
+
return {
|
|
692
|
+
ok: false,
|
|
693
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
return { ok: true };
|
|
697
|
+
} catch (error2) {
|
|
698
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
async function getAgentInfo(agentId, authHeader) {
|
|
702
|
+
const apiUrl = getApiUrlConfig();
|
|
703
|
+
try {
|
|
704
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}`, {
|
|
705
|
+
headers: { Authorization: authHeader }
|
|
706
|
+
});
|
|
707
|
+
if (response.status === 401) {
|
|
708
|
+
const serverMessage = await readErrorMessage(response);
|
|
709
|
+
return { valid: false, error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
710
|
+
}
|
|
711
|
+
if (response.status === 403) {
|
|
712
|
+
const serverMessage = await readErrorMessage(response);
|
|
713
|
+
return {
|
|
714
|
+
valid: false,
|
|
715
|
+
error: serverMessage ?? "You do not have access to this runner (it may belong to a different team or organization)."
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
if (response.status === 404) {
|
|
719
|
+
const serverMessage = await readErrorMessage(response);
|
|
720
|
+
return { valid: false, error: serverMessage ?? `Runner ${agentId} not found` };
|
|
721
|
+
}
|
|
722
|
+
if (!response.ok) {
|
|
723
|
+
const serverMessage = await readErrorMessage(response);
|
|
724
|
+
return {
|
|
725
|
+
valid: false,
|
|
726
|
+
error: `API error (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
const agent = await response.json();
|
|
730
|
+
if (agent.agent_type !== "local") {
|
|
731
|
+
return {
|
|
732
|
+
valid: false,
|
|
733
|
+
error: `Runner is type '${agent.agent_type}', must be 'local' for CLI connection`
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
return { valid: true, agent };
|
|
737
|
+
} catch (error2) {
|
|
738
|
+
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
739
|
+
return { valid: false, error: `Failed to validate runner: ${message}` };
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
// src/commands/status.ts
|
|
744
|
+
var STATUS_TIMEOUT_MS = 1e4;
|
|
745
|
+
function authLabelFor(credentials2) {
|
|
746
|
+
if (credentials2.authType === "agent_key") {
|
|
747
|
+
return credentials2.keySource === "agent_key" ? "runner key (EVIDENT_AGENT_KEY)" : "runner key (EVIDENT_RUNNER_KEY)";
|
|
748
|
+
}
|
|
749
|
+
return "user token";
|
|
750
|
+
}
|
|
751
|
+
function describeFetchError(error2) {
|
|
752
|
+
const name = error2?.name;
|
|
753
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
754
|
+
return `timed out after ${STATUS_TIMEOUT_MS}ms waiting for a response`;
|
|
755
|
+
}
|
|
756
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
757
|
+
}
|
|
758
|
+
async function checkStatus(jsonMode) {
|
|
759
|
+
const apiUrl = getApiUrlConfig();
|
|
760
|
+
const credentials2 = await getAuthCredentials();
|
|
761
|
+
if (!credentials2) {
|
|
762
|
+
return {
|
|
763
|
+
ok: false,
|
|
764
|
+
endpoint: apiUrl,
|
|
765
|
+
reason: "no_credentials",
|
|
766
|
+
error: "No credentials configured. Set EVIDENT_RUNNER_KEY (or EVIDENT_AGENT_KEY), or EVIDENT_TOKEN, or run `evident login`.",
|
|
767
|
+
exitCode: 1
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
if (credentials2.notice && !jsonMode) {
|
|
771
|
+
printWarning(credentials2.notice);
|
|
772
|
+
}
|
|
773
|
+
let response;
|
|
774
|
+
try {
|
|
775
|
+
response = await fetch(`${apiUrl}/me`, {
|
|
776
|
+
headers: { Authorization: getAuthHeader(credentials2) },
|
|
777
|
+
signal: AbortSignal.timeout(STATUS_TIMEOUT_MS)
|
|
778
|
+
});
|
|
779
|
+
} catch (error2) {
|
|
780
|
+
return {
|
|
781
|
+
ok: false,
|
|
782
|
+
endpoint: apiUrl,
|
|
783
|
+
authLabel: authLabelFor(credentials2),
|
|
784
|
+
reason: "unreachable",
|
|
785
|
+
error: `Could not reach ${apiUrl}: ${describeFetchError(error2)}. The credentials were NOT validated.`,
|
|
786
|
+
exitCode: 75
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
if (response.status === 401) {
|
|
790
|
+
const serverMessage = await readErrorMessage(response);
|
|
791
|
+
return {
|
|
792
|
+
ok: false,
|
|
793
|
+
endpoint: apiUrl,
|
|
794
|
+
authLabel: authLabelFor(credentials2),
|
|
795
|
+
reason: "unauthorized",
|
|
796
|
+
error: authFailureHint(apiUrl, serverMessage),
|
|
797
|
+
exitCode: 1
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
if (response.status >= 500) {
|
|
801
|
+
const serverMessage = await readErrorMessage(response);
|
|
802
|
+
return {
|
|
803
|
+
ok: false,
|
|
804
|
+
endpoint: apiUrl,
|
|
805
|
+
authLabel: authLabelFor(credentials2),
|
|
806
|
+
reason: "unreachable",
|
|
807
|
+
error: `${apiUrl} returned HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}. The credentials were NOT validated.`,
|
|
808
|
+
exitCode: 75
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
if (!response.ok) {
|
|
812
|
+
const serverMessage = await readErrorMessage(response);
|
|
813
|
+
return {
|
|
814
|
+
ok: false,
|
|
815
|
+
endpoint: apiUrl,
|
|
816
|
+
authLabel: authLabelFor(credentials2),
|
|
817
|
+
reason: "http_error",
|
|
818
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`,
|
|
819
|
+
exitCode: 1
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
const data = await response.json();
|
|
823
|
+
return {
|
|
824
|
+
ok: true,
|
|
825
|
+
endpoint: apiUrl,
|
|
826
|
+
authType: data.auth_type,
|
|
827
|
+
authLabel: authLabelFor(credentials2),
|
|
828
|
+
runnerId: data.auth_type === "agent_key" ? data.agent_id : void 0,
|
|
829
|
+
reason: "ok",
|
|
830
|
+
exitCode: 0
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
function printJson(result) {
|
|
834
|
+
const payload = {
|
|
835
|
+
ok: result.ok,
|
|
836
|
+
endpoint: result.endpoint
|
|
837
|
+
};
|
|
838
|
+
if (result.authType) payload.auth_type = result.authType;
|
|
839
|
+
if (result.runnerId) payload.runner_id = result.runnerId;
|
|
840
|
+
if (result.reason) payload.reason = result.reason;
|
|
841
|
+
if (result.error) payload.error = result.error;
|
|
842
|
+
console.log(JSON.stringify(payload));
|
|
843
|
+
}
|
|
844
|
+
function printHuman(result) {
|
|
845
|
+
blank();
|
|
846
|
+
console.log(keyValue("Endpoint", result.endpoint));
|
|
847
|
+
if (result.ok) {
|
|
848
|
+
console.log(keyValue("Auth", result.authLabel ?? "\u2014"));
|
|
849
|
+
if (result.runnerId) {
|
|
850
|
+
console.log(keyValue("Runner", result.runnerId));
|
|
851
|
+
}
|
|
852
|
+
console.log(keyValue("Status", "OK \u2014 credentials accepted"));
|
|
853
|
+
blank();
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
if (result.authLabel) {
|
|
857
|
+
console.log(keyValue("Auth", result.authLabel));
|
|
858
|
+
}
|
|
859
|
+
blank();
|
|
860
|
+
printError(result.error ?? "Unknown error");
|
|
861
|
+
}
|
|
862
|
+
async function status(options = {}) {
|
|
863
|
+
const result = await checkStatus(Boolean(options.json));
|
|
864
|
+
if (options.json) {
|
|
865
|
+
printJson(result);
|
|
866
|
+
} else {
|
|
867
|
+
printHuman(result);
|
|
868
|
+
}
|
|
869
|
+
process.exit(result.exitCode);
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// src/lib/claude-usage.ts
|
|
873
|
+
import { execFileSync } from "child_process";
|
|
874
|
+
import { readFileSync } from "fs";
|
|
875
|
+
import { homedir } from "os";
|
|
876
|
+
import { join } from "path";
|
|
877
|
+
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
878
|
+
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
879
|
+
function parseClaudeCliCredentials(raw) {
|
|
880
|
+
let parsed;
|
|
881
|
+
try {
|
|
882
|
+
parsed = JSON.parse(raw);
|
|
883
|
+
} catch {
|
|
884
|
+
return null;
|
|
885
|
+
}
|
|
886
|
+
const data = parsed.claudeAiOauth ?? parsed;
|
|
887
|
+
const creds = data;
|
|
888
|
+
if (typeof creds.accessToken !== "string" || typeof creds.expiresAt !== "number") {
|
|
889
|
+
return null;
|
|
890
|
+
}
|
|
891
|
+
return { accessToken: creds.accessToken, expiresAt: creds.expiresAt };
|
|
892
|
+
}
|
|
893
|
+
function readClaudeCliCredentials() {
|
|
894
|
+
if (process.platform === "darwin") {
|
|
895
|
+
try {
|
|
896
|
+
const raw = execFileSync(
|
|
897
|
+
"/usr/bin/security",
|
|
898
|
+
["find-generic-password", "-s", KEYCHAIN_SERVICE, "-w"],
|
|
899
|
+
{ encoding: "utf-8", timeout: 2e3, stdio: ["pipe", "pipe", "ignore"] }
|
|
900
|
+
);
|
|
901
|
+
return parseClaudeCliCredentials(raw);
|
|
902
|
+
} catch {
|
|
903
|
+
return null;
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
try {
|
|
907
|
+
const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
|
|
908
|
+
return parseClaudeCliCredentials(raw);
|
|
909
|
+
} catch {
|
|
910
|
+
return null;
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
var ClaudeUsageError = class extends Error {
|
|
914
|
+
constructor(message, reason) {
|
|
915
|
+
super(message);
|
|
916
|
+
this.reason = reason;
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
function isLocalCredentialProblem(err) {
|
|
920
|
+
return err instanceof ClaudeUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
921
|
+
}
|
|
922
|
+
function normalizeResetsAt(value) {
|
|
923
|
+
const ms = Date.parse(value);
|
|
924
|
+
return Number.isNaN(ms) ? null : new Date(ms).toISOString();
|
|
925
|
+
}
|
|
926
|
+
function toWindow(value) {
|
|
927
|
+
if (!value || typeof value !== "object") {
|
|
928
|
+
return null;
|
|
929
|
+
}
|
|
930
|
+
const window = value;
|
|
931
|
+
if (typeof window.utilization !== "number" || typeof window.resets_at !== "string") {
|
|
932
|
+
return null;
|
|
933
|
+
}
|
|
934
|
+
const resetsAt = normalizeResetsAt(window.resets_at);
|
|
935
|
+
if (resetsAt === null) {
|
|
936
|
+
return null;
|
|
937
|
+
}
|
|
938
|
+
return { utilization: window.utilization, resetsAt };
|
|
939
|
+
}
|
|
940
|
+
async function getClaudeUsage() {
|
|
941
|
+
const credentials2 = readClaudeCliCredentials();
|
|
942
|
+
if (!credentials2) {
|
|
943
|
+
throw new ClaudeUsageError(
|
|
944
|
+
"No local Claude Code login found. Run `claude` once to sign in with your Claude subscription.",
|
|
945
|
+
"no_credentials"
|
|
946
|
+
);
|
|
436
947
|
}
|
|
437
|
-
|
|
438
|
-
|
|
948
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
949
|
+
throw new ClaudeUsageError(
|
|
950
|
+
"Claude Code credentials have expired. Run `claude` to refresh them.",
|
|
951
|
+
"credentials_expired"
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
const res = await fetch(CLAUDE_USAGE_URL, {
|
|
955
|
+
headers: {
|
|
956
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
957
|
+
"Content-Type": "application/json",
|
|
958
|
+
"anthropic-version": "2023-06-01"
|
|
959
|
+
}
|
|
960
|
+
});
|
|
961
|
+
if (!res.ok) {
|
|
962
|
+
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
963
|
+
}
|
|
964
|
+
const body = await res.json();
|
|
965
|
+
return {
|
|
966
|
+
fiveHour: toWindow(body.five_hour),
|
|
967
|
+
sevenDay: toWindow(body.seven_day)
|
|
968
|
+
};
|
|
439
969
|
}
|
|
440
970
|
|
|
441
|
-
// src/commands/
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
const credentials2 = await getToken();
|
|
446
|
-
if (!credentials2) {
|
|
447
|
-
printError(`Not logged in to ${apiUrl}. Run the \`login\` command to authenticate.`);
|
|
448
|
-
process.exit(1);
|
|
971
|
+
// src/commands/claude-usage.ts
|
|
972
|
+
function formatWindow(label, window) {
|
|
973
|
+
if (!window) {
|
|
974
|
+
return keyValue(label, "not available for this plan");
|
|
449
975
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
const
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
);
|
|
463
|
-
|
|
976
|
+
const resetsAt = new Date(window.resetsAt);
|
|
977
|
+
return keyValue(label, `${window.utilization}% used, resets ${resetsAt.toLocaleString()}`);
|
|
978
|
+
}
|
|
979
|
+
async function claudeUsage() {
|
|
980
|
+
try {
|
|
981
|
+
const usage = await getClaudeUsage();
|
|
982
|
+
blank();
|
|
983
|
+
console.log(formatWindow("5-hour session", usage.fiveHour));
|
|
984
|
+
console.log(formatWindow("7-day", usage.sevenDay));
|
|
985
|
+
blank();
|
|
986
|
+
} catch (err) {
|
|
987
|
+
if (err instanceof ClaudeUsageError) {
|
|
988
|
+
printError(err.message);
|
|
989
|
+
process.exit(1);
|
|
464
990
|
}
|
|
991
|
+
throw err;
|
|
465
992
|
}
|
|
466
|
-
blank();
|
|
467
993
|
}
|
|
468
994
|
|
|
469
995
|
// src/commands/run.ts
|
|
@@ -471,6 +997,9 @@ import { homedir as homedir3 } from "os";
|
|
|
471
997
|
import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
|
|
472
998
|
import chalk6 from "chalk";
|
|
473
999
|
|
|
1000
|
+
// ../../packages/types/src/agents/index.ts
|
|
1001
|
+
var MICROVM_MAX_LIFETIME_MS = 8 * 60 * 6e4;
|
|
1002
|
+
|
|
474
1003
|
// ../../packages/types/src/telemetry/index.ts
|
|
475
1004
|
var TelemetryEventTypes = {
|
|
476
1005
|
// Agent activity events (shown in web UI activity log)
|
|
@@ -478,7 +1007,10 @@ var TelemetryEventTypes = {
|
|
|
478
1007
|
AGENT_DISCONNECTED: "agent.disconnected",
|
|
479
1008
|
AGENT_MESSAGE_PROCESSING: "agent.message_processing",
|
|
480
1009
|
AGENT_MESSAGE_DONE: "agent.message_done",
|
|
481
|
-
AGENT_MESSAGE_FAILED: "agent.message_failed"
|
|
1010
|
+
AGENT_MESSAGE_FAILED: "agent.message_failed",
|
|
1011
|
+
// A `warn`/`error` runner-side log line forwarded server-side for
|
|
1012
|
+
// observability (issue #916) — see `apps/cli/src/lib/runner-activity-telemetry.ts`.
|
|
1013
|
+
RUNNER_ACTIVITY: "runner.activity"
|
|
482
1014
|
};
|
|
483
1015
|
|
|
484
1016
|
// ../../packages/types/src/tunnel/index.ts
|
|
@@ -533,6 +1065,13 @@ var isShuttingDown = false;
|
|
|
533
1065
|
var FLUSH_INTERVAL_MS = 5e3;
|
|
534
1066
|
var MAX_BUFFER_SIZE = 50;
|
|
535
1067
|
var FLUSH_TIMEOUT_MS = 3e3;
|
|
1068
|
+
var authProvider = null;
|
|
1069
|
+
function setTelemetryAuthProvider(provider) {
|
|
1070
|
+
authProvider = provider;
|
|
1071
|
+
}
|
|
1072
|
+
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
1073
|
+
var lastFlushFailureLoggedAt = 0;
|
|
1074
|
+
var suppressedFlushFailureCount = 0;
|
|
536
1075
|
function logEvent(eventType, options = {}) {
|
|
537
1076
|
const event = {
|
|
538
1077
|
event_type: eventType,
|
|
@@ -567,9 +1106,16 @@ async function flushEvents() {
|
|
|
567
1106
|
flushTimeout = null;
|
|
568
1107
|
}
|
|
569
1108
|
try {
|
|
570
|
-
const
|
|
571
|
-
|
|
572
|
-
|
|
1109
|
+
const providerContext = authProvider?.();
|
|
1110
|
+
let authHeader;
|
|
1111
|
+
if (providerContext?.authHeader) {
|
|
1112
|
+
authHeader = providerContext.authHeader;
|
|
1113
|
+
} else {
|
|
1114
|
+
const credentials2 = await getToken();
|
|
1115
|
+
if (!credentials2) {
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
authHeader = `Bearer ${credentials2.token}`;
|
|
573
1119
|
}
|
|
574
1120
|
const apiUrl = getApiUrlConfig();
|
|
575
1121
|
const controller = new AbortController();
|
|
@@ -584,7 +1130,7 @@ async function flushEvents() {
|
|
|
584
1130
|
method: "POST",
|
|
585
1131
|
headers: {
|
|
586
1132
|
"Content-Type": "application/json",
|
|
587
|
-
Authorization:
|
|
1133
|
+
Authorization: authHeader
|
|
588
1134
|
},
|
|
589
1135
|
body: JSON.stringify(request),
|
|
590
1136
|
signal: controller.signal
|
|
@@ -596,8 +1142,15 @@ async function flushEvents() {
|
|
|
596
1142
|
clearTimeout(timeout);
|
|
597
1143
|
}
|
|
598
1144
|
} catch (error2) {
|
|
599
|
-
|
|
600
|
-
|
|
1145
|
+
const now = Date.now();
|
|
1146
|
+
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
1147
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
1148
|
+
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
1149
|
+
console.error(`Telemetry flush error: ${message}${suffix}`);
|
|
1150
|
+
lastFlushFailureLoggedAt = now;
|
|
1151
|
+
suppressedFlushFailureCount = 0;
|
|
1152
|
+
} else {
|
|
1153
|
+
suppressedFlushFailureCount++;
|
|
601
1154
|
}
|
|
602
1155
|
}
|
|
603
1156
|
}
|
|
@@ -666,47 +1219,67 @@ var EventTypes = {
|
|
|
666
1219
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
667
1220
|
};
|
|
668
1221
|
|
|
669
|
-
// src/lib/
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
if (
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
1222
|
+
// src/lib/runner-activity-telemetry.ts
|
|
1223
|
+
var FORWARDED_LEVELS = /* @__PURE__ */ new Set(["warn", "error"]);
|
|
1224
|
+
var SEVERITY_BY_LEVEL = {
|
|
1225
|
+
warn: "warning",
|
|
1226
|
+
error: "error"
|
|
1227
|
+
};
|
|
1228
|
+
var MAX_MESSAGE_LENGTH = 500;
|
|
1229
|
+
var TRUNCATION_MARKER = "\u2026";
|
|
1230
|
+
function redact(message) {
|
|
1231
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
1232
|
+
}
|
|
1233
|
+
function truncate(message) {
|
|
1234
|
+
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
1235
|
+
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
1236
|
+
}
|
|
1237
|
+
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
1238
|
+
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
1239
|
+
var windowStartedAt = 0;
|
|
1240
|
+
var windowCount = 0;
|
|
1241
|
+
var windowDroppedCount = 0;
|
|
1242
|
+
function admitUnderRateLimit(now) {
|
|
1243
|
+
if (now - windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
1244
|
+
if (windowDroppedCount > 0) {
|
|
1245
|
+
console.error(
|
|
1246
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
1247
|
+
);
|
|
1248
|
+
}
|
|
1249
|
+
windowStartedAt = now;
|
|
1250
|
+
windowCount = 0;
|
|
1251
|
+
windowDroppedCount = 0;
|
|
687
1252
|
}
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
}
|
|
1253
|
+
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
1254
|
+
windowDroppedCount++;
|
|
1255
|
+
if (windowDroppedCount === 1) {
|
|
1256
|
+
console.error(
|
|
1257
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
return false;
|
|
695
1261
|
}
|
|
696
|
-
|
|
1262
|
+
windowCount++;
|
|
1263
|
+
return true;
|
|
697
1264
|
}
|
|
698
|
-
function
|
|
699
|
-
|
|
700
|
-
|
|
1265
|
+
function forwardRunnerActivity(entry, context) {
|
|
1266
|
+
try {
|
|
1267
|
+
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
1268
|
+
if (!context.agentId || !context.authHeader) return;
|
|
1269
|
+
if (!admitUnderRateLimit(Date.now())) return;
|
|
1270
|
+
const rawMessage = entry.error ?? entry.message ?? "";
|
|
1271
|
+
const message = truncate(redact(rawMessage));
|
|
1272
|
+
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
1273
|
+
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
1274
|
+
message,
|
|
1275
|
+
metadata: { source: "cli.run" },
|
|
1276
|
+
agentId: context.agentId
|
|
1277
|
+
});
|
|
1278
|
+
} catch (err) {
|
|
1279
|
+
console.error(
|
|
1280
|
+
`[runner-activity-telemetry] failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
1281
|
+
);
|
|
701
1282
|
}
|
|
702
|
-
return `Bearer ${credentials2.token}`;
|
|
703
|
-
}
|
|
704
|
-
function isInteractive(jsonOutput) {
|
|
705
|
-
if (jsonOutput) return false;
|
|
706
|
-
if (process.env.CI) return false;
|
|
707
|
-
if (process.env.GITHUB_ACTIONS) return false;
|
|
708
|
-
if (!process.stdin.isTTY) return false;
|
|
709
|
-
return true;
|
|
710
1283
|
}
|
|
711
1284
|
|
|
712
1285
|
// src/lib/opencode/health.ts
|
|
@@ -1500,6 +2073,9 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
1500
2073
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1501
2074
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1502
2075
|
}
|
|
2076
|
+
function isB2AbandonmentConfirmed(params) {
|
|
2077
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
2078
|
+
}
|
|
1503
2079
|
function messageError(messages, userMessageId) {
|
|
1504
2080
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1505
2081
|
const error2 = errorOf(reply);
|
|
@@ -1513,6 +2089,57 @@ function messageError(messages, userMessageId) {
|
|
|
1513
2089
|
}
|
|
1514
2090
|
return "The agent run failed.";
|
|
1515
2091
|
}
|
|
2092
|
+
function isAbortedTerminalReply(messages, userMessageId) {
|
|
2093
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2094
|
+
const error2 = errorOf(reply);
|
|
2095
|
+
if (error2 == null) return false;
|
|
2096
|
+
if (typeof error2 === "string") return error2.trim() === "Aborted";
|
|
2097
|
+
if (typeof error2 === "object") {
|
|
2098
|
+
const e = error2;
|
|
2099
|
+
if (e.name === "MessageAbortedError") return true;
|
|
2100
|
+
if (e.name === "AbortError") return true;
|
|
2101
|
+
const dataMessage = e.data?.message;
|
|
2102
|
+
const rendered = typeof dataMessage === "string" ? dataMessage : typeof e.message === "string" ? e.message : null;
|
|
2103
|
+
return rendered != null && rendered.trim() === "Aborted";
|
|
2104
|
+
}
|
|
2105
|
+
return false;
|
|
2106
|
+
}
|
|
2107
|
+
function messageFailure(messages, userMessageId) {
|
|
2108
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
2109
|
+
const error2 = errorOf(reply);
|
|
2110
|
+
if (error2 == null || typeof error2 !== "object") return null;
|
|
2111
|
+
const e = error2;
|
|
2112
|
+
const replyProviderId = reply?.info?.providerID ?? null;
|
|
2113
|
+
const replyModelId = reply?.info?.modelID ?? null;
|
|
2114
|
+
if (e.name === "ProviderAuthError") {
|
|
2115
|
+
const data = e.data;
|
|
2116
|
+
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
2117
|
+
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
2118
|
+
}
|
|
2119
|
+
if (e.name === "APIError") {
|
|
2120
|
+
const data = e.data;
|
|
2121
|
+
const statusCode = data?.statusCode;
|
|
2122
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
2123
|
+
return {
|
|
2124
|
+
kind: "model_auth",
|
|
2125
|
+
providerId: replyProviderId,
|
|
2126
|
+
modelId: replyModelId,
|
|
2127
|
+
reason: "rejected"
|
|
2128
|
+
};
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return null;
|
|
2132
|
+
}
|
|
2133
|
+
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
2134
|
+
if (classified != null) return classified;
|
|
2135
|
+
if (hasConfiguredProvider !== false) return null;
|
|
2136
|
+
return {
|
|
2137
|
+
kind: "model_auth",
|
|
2138
|
+
providerId: replyProviderId,
|
|
2139
|
+
modelId: replyModelId,
|
|
2140
|
+
reason: "missing"
|
|
2141
|
+
};
|
|
2142
|
+
}
|
|
1516
2143
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1517
2144
|
if (!messages || messages.length === 0) return false;
|
|
1518
2145
|
return messages.some(
|
|
@@ -2041,13 +2668,56 @@ var RunnerConnection = class {
|
|
|
2041
2668
|
}
|
|
2042
2669
|
};
|
|
2043
2670
|
|
|
2671
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2672
|
+
import { writeFileSync } from "fs";
|
|
2673
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2674
|
+
try {
|
|
2675
|
+
writeFileSync(path, `${agentId}
|
|
2676
|
+
`);
|
|
2677
|
+
return { ok: true };
|
|
2678
|
+
} catch (error2) {
|
|
2679
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
// src/lib/claude-usage-reporting.ts
|
|
2684
|
+
var VALID_MODES = ["auto", "on", "off"];
|
|
2685
|
+
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
2686
|
+
const raw = flagValue ?? env.EVIDENT_CLAUDE_USAGE_REPORTING;
|
|
2687
|
+
if (raw === void 0 || raw === "") {
|
|
2688
|
+
return { mode: "auto", warnings: [] };
|
|
2689
|
+
}
|
|
2690
|
+
const normalized = raw.trim().toLowerCase();
|
|
2691
|
+
if (VALID_MODES.includes(normalized)) {
|
|
2692
|
+
return { mode: normalized, warnings: [] };
|
|
2693
|
+
}
|
|
2694
|
+
const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
|
|
2695
|
+
return {
|
|
2696
|
+
mode: "auto",
|
|
2697
|
+
warnings: [
|
|
2698
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
2699
|
+
]
|
|
2700
|
+
};
|
|
2701
|
+
}
|
|
2702
|
+
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2703
|
+
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2704
|
+
function nextReportDelayMs(random = Math.random) {
|
|
2705
|
+
const jitterRangeMs = BASE_REPORT_DELAY_MS * REPORT_DELAY_JITTER_FRACTION;
|
|
2706
|
+
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2707
|
+
}
|
|
2708
|
+
var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
|
|
2709
|
+
var CLAUDE_USAGE_FAILURE_REESCALATION_TICKS = 6;
|
|
2710
|
+
function claudeUsageFailureLogLevel(consecutiveFailures) {
|
|
2711
|
+
return consecutiveFailures === 1 || consecutiveFailures % CLAUDE_USAGE_FAILURE_REESCALATION_TICKS === 0 ? "warn" : "debug";
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2044
2714
|
// src/lib/channels/driver.ts
|
|
2045
2715
|
import { homedir as homedir2 } from "os";
|
|
2046
2716
|
|
|
2047
2717
|
// src/lib/file-push.ts
|
|
2048
2718
|
import { randomUUID } from "crypto";
|
|
2049
2719
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2050
|
-
import { basename, dirname, isAbsolute, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
2720
|
+
import { basename, dirname as dirname2, isAbsolute, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
2051
2721
|
var FILE_MODE = 384;
|
|
2052
2722
|
var DIRECTORY_MODE = 448;
|
|
2053
2723
|
async function writePushedFile(request) {
|
|
@@ -2078,7 +2748,7 @@ async function writePushedFile(request) {
|
|
|
2078
2748
|
}
|
|
2079
2749
|
try {
|
|
2080
2750
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2081
|
-
|
|
2751
|
+
dirname2(candidate)
|
|
2082
2752
|
);
|
|
2083
2753
|
const realTarget = join2(existingAncestor, ...missingSegments, basename(candidate));
|
|
2084
2754
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
@@ -2090,8 +2760,8 @@ async function writePushedFile(request) {
|
|
|
2090
2760
|
}
|
|
2091
2761
|
if (missingSegments.length > 0) {
|
|
2092
2762
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2093
|
-
const realParent = await realpath(
|
|
2094
|
-
if (realParent !==
|
|
2763
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2764
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2095
2765
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2096
2766
|
path: realTarget,
|
|
2097
2767
|
bytes,
|
|
@@ -2134,7 +2804,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
2134
2804
|
try {
|
|
2135
2805
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
2136
2806
|
} catch (err) {
|
|
2137
|
-
const parent =
|
|
2807
|
+
const parent = dirname2(current);
|
|
2138
2808
|
if (err.code !== "ENOENT" || parent === current) {
|
|
2139
2809
|
throw err;
|
|
2140
2810
|
}
|
|
@@ -2195,7 +2865,7 @@ async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
|
2195
2865
|
}
|
|
2196
2866
|
}
|
|
2197
2867
|
async function writeAtomically(realTarget, content) {
|
|
2198
|
-
const temporaryPath = join2(
|
|
2868
|
+
const temporaryPath = join2(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2199
2869
|
let handle;
|
|
2200
2870
|
try {
|
|
2201
2871
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -2362,8 +3032,8 @@ async function applyOne(options, file) {
|
|
|
2362
3032
|
await ack(options, file, "applied");
|
|
2363
3033
|
return true;
|
|
2364
3034
|
}
|
|
2365
|
-
function durableDownloadCode(
|
|
2366
|
-
return
|
|
3035
|
+
function durableDownloadCode(status2) {
|
|
3036
|
+
return status2 === 413 ? "file_too_large" : "write_failed";
|
|
2367
3037
|
}
|
|
2368
3038
|
async function downloadContent(options, file, label) {
|
|
2369
3039
|
try {
|
|
@@ -2396,8 +3066,8 @@ async function downloadContent(options, file, label) {
|
|
|
2396
3066
|
return { ok: false, terminal: false };
|
|
2397
3067
|
}
|
|
2398
3068
|
}
|
|
2399
|
-
async function ack(options, file,
|
|
2400
|
-
const outcome = `${
|
|
3069
|
+
async function ack(options, file, status2, reason) {
|
|
3070
|
+
const outcome = `${status2}${reason ? ` (${reason})` : ""}`;
|
|
2401
3071
|
try {
|
|
2402
3072
|
const res = await options.fetchImpl(
|
|
2403
3073
|
`${options.apiUrl}/runners/${options.agentId}/files/${file.id}/ack`,
|
|
@@ -2407,7 +3077,7 @@ async function ack(options, file, status, reason) {
|
|
|
2407
3077
|
Authorization: options.getAuthHeader(),
|
|
2408
3078
|
"Content-Type": "application/json"
|
|
2409
3079
|
},
|
|
2410
|
-
body: JSON.stringify(reason ? { status, reason } : { status })
|
|
3080
|
+
body: JSON.stringify(reason ? { status: status2, reason } : { status: status2 })
|
|
2411
3081
|
}
|
|
2412
3082
|
);
|
|
2413
3083
|
if (!res.ok) {
|
|
@@ -2467,8 +3137,11 @@ var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
|
2467
3137
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2468
3138
|
var HEARTBEAT_MS = 6e4;
|
|
2469
3139
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
3140
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
3141
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2470
3142
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
2471
3143
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
3144
|
+
var MAX_IDENTICAL_REDRIVE_POLL_FAILURES = 5;
|
|
2472
3145
|
var ChannelAuthError = class extends Error {
|
|
2473
3146
|
constructor(message) {
|
|
2474
3147
|
super(message);
|
|
@@ -2477,10 +3150,10 @@ var ChannelAuthError = class extends Error {
|
|
|
2477
3150
|
};
|
|
2478
3151
|
var ChannelTerminalError = class extends Error {
|
|
2479
3152
|
status;
|
|
2480
|
-
constructor(message,
|
|
3153
|
+
constructor(message, status2) {
|
|
2481
3154
|
super(message);
|
|
2482
3155
|
this.name = "ChannelTerminalError";
|
|
2483
|
-
this.status =
|
|
3156
|
+
this.status = status2;
|
|
2484
3157
|
}
|
|
2485
3158
|
};
|
|
2486
3159
|
function backoffDelay(attempt, policy) {
|
|
@@ -2488,8 +3161,12 @@ function backoffDelay(attempt, policy) {
|
|
|
2488
3161
|
const capped = Math.min(policy.maxDelayMs, exp);
|
|
2489
3162
|
return Math.floor(Math.random() * capped);
|
|
2490
3163
|
}
|
|
2491
|
-
function isRetryableStatus(
|
|
2492
|
-
return
|
|
3164
|
+
function isRetryableStatus(status2) {
|
|
3165
|
+
return status2 === 429 || status2 >= 500 && status2 <= 599;
|
|
3166
|
+
}
|
|
3167
|
+
var VOLATILE_BODY_FIELD_PATTERN = /("(?:ref|requestId|request_id|traceId|trace_id)"\s*:\s*)"[^"]*"/gi;
|
|
3168
|
+
function normalizeRedrivePollFailureBody(body) {
|
|
3169
|
+
return body.replace(VOLATILE_BODY_FIELD_PATTERN, '$1"<redacted>"').replace(/\s+/g, " ").trim().slice(0, 200);
|
|
2493
3170
|
}
|
|
2494
3171
|
var ChannelDriver = class _ChannelDriver {
|
|
2495
3172
|
agentId;
|
|
@@ -2604,6 +3281,33 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2604
3281
|
* processing list, exactly like `dontRedispatch`/`doneUndeliverable`.
|
|
2605
3282
|
*/
|
|
2606
3283
|
readoptPollUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
3284
|
+
/**
|
|
3285
|
+
* "Already emitted `redrive_unresolved` for this row" (#965). Mirrors
|
|
3286
|
+
* `readoptPollUnresolvedSignalled`: `resolveRedrive`'s `unresolved` leaf recurs
|
|
3287
|
+
* every ~2s drain until opencode's status becomes readable, but the
|
|
3288
|
+
* server-visible signal is an OUTCOME, so it fires at most once per row. Cleared
|
|
3289
|
+
* on any non-`unresolved` outcome so the set cannot grow beyond the currently
|
|
3290
|
+
* unresolvable rows.
|
|
3291
|
+
*/
|
|
3292
|
+
redriveUnresolvedSignalled = /* @__PURE__ */ new Set();
|
|
3293
|
+
/**
|
|
3294
|
+
* First `now()` a `pending` row's re-drive was observed `unresolved` (#965). A
|
|
3295
|
+
* `pending` row is invisible to every cron arm (all require `status =
|
|
3296
|
+
* 'processing'`), so an indefinitely-`unresolved` row would be stranded with
|
|
3297
|
+
* nothing driving it. Once `now - since >= pausedMaxWaitMs`, `resolveRedrive`
|
|
3298
|
+
* takes `dispatch` instead of `unresolved` (reusing the existing knob — see
|
|
3299
|
+
* ADR-0047's own "unreachable ⇒ bounded" rule). Cleared on any other outcome.
|
|
3300
|
+
*/
|
|
3301
|
+
redriveUnresolvedSince = /* @__PURE__ */ new Map();
|
|
3302
|
+
/**
|
|
3303
|
+
* Consecutive-identical-poll-failure streak for the re-drive fence (#1348),
|
|
3304
|
+
* keyed by Evident **message id** (not session) so `clearRedriveUnresolved`
|
|
3305
|
+
* can drop it with the other two trackers and it cannot leak. `sessionId` is
|
|
3306
|
+
* carried inside the entry, not the key: a session change is a different
|
|
3307
|
+
* situation and resets the streak, which gives the `(sessionId, message.id)`
|
|
3308
|
+
* pairing #1348 asks for without a composite map key.
|
|
3309
|
+
*/
|
|
3310
|
+
redrivePollFailures = /* @__PURE__ */ new Map();
|
|
2607
3311
|
/**
|
|
2608
3312
|
* "A null-id re-adopt re-dispatch is in flight, awaiting its read-back" (WI-5
|
|
2609
3313
|
* Task 5.4, High-2). Since we no longer send a caller-supplied id, a re-dispatch
|
|
@@ -2692,23 +3396,23 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2692
3396
|
* and stops opencode.
|
|
2693
3397
|
*/
|
|
2694
3398
|
stopped = false;
|
|
2695
|
-
constructor(
|
|
2696
|
-
this.agentId =
|
|
2697
|
-
this.port =
|
|
2698
|
-
this.apiUrl =
|
|
2699
|
-
this.getAuthHeader =
|
|
2700
|
-
this.conversationFilter =
|
|
2701
|
-
this.retry = { ...DEFAULT_RETRY_POLICY, ...
|
|
2702
|
-
this.log =
|
|
3399
|
+
constructor(config) {
|
|
3400
|
+
this.agentId = config.agentId;
|
|
3401
|
+
this.port = config.port;
|
|
3402
|
+
this.apiUrl = config.apiUrl.replace(/\/$/, "");
|
|
3403
|
+
this.getAuthHeader = config.getAuthHeader;
|
|
3404
|
+
this.conversationFilter = config.conversationFilter ?? null;
|
|
3405
|
+
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3406
|
+
this.log = config.log ?? (() => {
|
|
2703
3407
|
});
|
|
2704
|
-
this.fetchImpl =
|
|
2705
|
-
this.sleep =
|
|
2706
|
-
this.pausedPollIntervalMs =
|
|
2707
|
-
this.pausedMaxWaitMs =
|
|
2708
|
-
this.stuckQueuedMs =
|
|
2709
|
-
this.now =
|
|
2710
|
-
this.fileSyncDirectories =
|
|
2711
|
-
this.homeDir =
|
|
3408
|
+
this.fetchImpl = config.fetchImpl ?? fetch;
|
|
3409
|
+
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3410
|
+
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3411
|
+
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
3412
|
+
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
3413
|
+
this.now = config.now ?? (() => Date.now());
|
|
3414
|
+
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3415
|
+
this.homeDir = config.homeDir ?? homedir2();
|
|
2712
3416
|
}
|
|
2713
3417
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
2714
3418
|
get opencodeBase() {
|
|
@@ -2943,6 +3647,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2943
3647
|
skippedAlreadyDispatched += 1;
|
|
2944
3648
|
continue;
|
|
2945
3649
|
}
|
|
3650
|
+
if (message.opencode_message_id) {
|
|
3651
|
+
const outcome = await this.resolveRedrive(conv, sessionId, message, refusedSessionId);
|
|
3652
|
+
if (outcome !== "dispatch") {
|
|
3653
|
+
break;
|
|
3654
|
+
}
|
|
3655
|
+
}
|
|
2946
3656
|
const options = {
|
|
2947
3657
|
agent: message.opencode_agent ?? void 0,
|
|
2948
3658
|
model: message.opencode_model ?? void 0
|
|
@@ -2987,50 +3697,361 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2987
3697
|
this.sessions.delete(conv.id);
|
|
2988
3698
|
this.supersede(conv.id, sessionId);
|
|
2989
3699
|
this.log({
|
|
2990
|
-
level: "warn",
|
|
2991
|
-
message: `Abandoning OpenCode session ${sessionId.slice(0, 8)} as the binding for conversation ${conv.id.slice(0, 8)} (it exists but failed to run a turn) \u2014 a fresh session is created on the next tick, whatever the persisted binding says by then.`,
|
|
3700
|
+
level: "warn",
|
|
3701
|
+
message: `Abandoning OpenCode session ${sessionId.slice(0, 8)} as the binding for conversation ${conv.id.slice(0, 8)} (it exists but failed to run a turn) \u2014 a fresh session is created on the next tick, whatever the persisted binding says by then.`,
|
|
3702
|
+
conversation_id: conv.id,
|
|
3703
|
+
message_id: message.id
|
|
3704
|
+
});
|
|
3705
|
+
await this.markFailed(conv.id, message.id, null, errorMessage).catch((markErr) => {
|
|
3706
|
+
this.log({
|
|
3707
|
+
level: "warn",
|
|
3708
|
+
message: `markFailed PATCH for message ${message.id.slice(0, 8)} (conversation ${conv.id.slice(0, 8)}) failed (best-effort, not retried): ${markErr instanceof Error ? markErr.message : String(markErr)}`,
|
|
3709
|
+
conversation_id: conv.id,
|
|
3710
|
+
message_id: message.id
|
|
3711
|
+
});
|
|
3712
|
+
});
|
|
3713
|
+
this.log({
|
|
3714
|
+
level: "error",
|
|
3715
|
+
message: `Message ${message.id.slice(0, 8)} dispatch failed: ${errorMessage}`,
|
|
3716
|
+
conversation_id: conv.id,
|
|
3717
|
+
message_id: message.id
|
|
3718
|
+
});
|
|
3719
|
+
break;
|
|
3720
|
+
}
|
|
3721
|
+
if (opencodeMessageId === null) {
|
|
3722
|
+
this.log({
|
|
3723
|
+
level: "warn",
|
|
3724
|
+
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
3725
|
+
conversation_id: conv.id,
|
|
3726
|
+
message_id: message.id
|
|
3727
|
+
});
|
|
3728
|
+
continue;
|
|
3729
|
+
}
|
|
3730
|
+
this.dispatched.add(message.id);
|
|
3731
|
+
this.registerInFlight(conv, sessionId, message, opencodeMessageId);
|
|
3732
|
+
dispatched += 1;
|
|
3733
|
+
void this.postSignal(conv.id, message.id, "dispatched");
|
|
3734
|
+
}
|
|
3735
|
+
if (messages.length > 0 && dispatched === 0 && skippedAlreadyDispatched === messages.length) {
|
|
3736
|
+
this.log({
|
|
3737
|
+
level: "warn",
|
|
3738
|
+
message: `Conversation ${conv.id.slice(0, 8)} has ${messages.length} pending message(s) but ALL are already marked dispatched locally (in-flight set: ${this.dispatched.size}) \u2014 none sent to OpenCode this tick. If this repeats, a message may be stuck acknowledged-but-never-dispatched (its watcher never settled).`,
|
|
3739
|
+
conversation_id: conv.id
|
|
3740
|
+
});
|
|
3741
|
+
}
|
|
3742
|
+
this.ensureWatcherRunning(sessionId);
|
|
3743
|
+
return dispatched;
|
|
3744
|
+
}
|
|
3745
|
+
/**
|
|
3746
|
+
* Poll a session's message list for the re-drive fence (#965), via the
|
|
3747
|
+
* INJECTED `fetchImpl` — NOT the imported `getSessionMessages` helper, which
|
|
3748
|
+
* hits the global `fetch` and would bypass the same override every other
|
|
3749
|
+
* opencode poll in this file respects. Mirrors `readoptProcessing`'s own
|
|
3750
|
+
* snapshot fetch (`:3081-3111`).
|
|
3751
|
+
*
|
|
3752
|
+
* Returns `{ ok: true, messages }` on a readable snapshot, or
|
|
3753
|
+
* `{ ok: false, signature }` on failure — `signature` is a string that
|
|
3754
|
+
* repeats across attempts for the SAME underlying fault (used by the
|
|
3755
|
+
* consecutive-identical-failure bound, #1348), or `null` for a thrown
|
|
3756
|
+
* exception, which is NOT countable toward that bound (a network blip / an
|
|
3757
|
+
* opencode restart also throws identically every tick, and must keep
|
|
3758
|
+
* retrying unbounded rather than ever being treated as permanent).
|
|
3759
|
+
*/
|
|
3760
|
+
async pollSessionMessagesForRedrive(conv, message, sessionId) {
|
|
3761
|
+
try {
|
|
3762
|
+
const res = await this.fetchImpl(`${this.opencodeBase}/session/${sessionId}/message`);
|
|
3763
|
+
if (!res.ok) {
|
|
3764
|
+
const rawBody = await res.text();
|
|
3765
|
+
const normalized = normalizeRedrivePollFailureBody(rawBody);
|
|
3766
|
+
this.log({
|
|
3767
|
+
level: "warn",
|
|
3768
|
+
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned HTTP ${res.status}${normalized ? `: ${normalized}` : ""} \u2014 treating as unreadable this tick`,
|
|
3769
|
+
conversation_id: conv.id,
|
|
3770
|
+
message_id: message.id
|
|
3771
|
+
});
|
|
3772
|
+
return { ok: false, signature: `HTTP ${res.status}${normalized ? `: ${normalized}` : ""}` };
|
|
3773
|
+
}
|
|
3774
|
+
const body = await res.json();
|
|
3775
|
+
if (!Array.isArray(body)) {
|
|
3776
|
+
this.log({
|
|
3777
|
+
level: "warn",
|
|
3778
|
+
message: `Re-drive: polling session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} returned a non-array message body \u2014 treating as unreadable this tick`,
|
|
3779
|
+
conversation_id: conv.id,
|
|
3780
|
+
message_id: message.id
|
|
3781
|
+
});
|
|
3782
|
+
return { ok: false, signature: "non-array message body" };
|
|
3783
|
+
}
|
|
3784
|
+
return { ok: true, messages: body };
|
|
3785
|
+
} catch (err) {
|
|
3786
|
+
this.log({
|
|
3787
|
+
level: "warn",
|
|
3788
|
+
message: `Re-drive: failed to poll session ${sessionId.slice(0, 8)} for message ${message.id.slice(0, 8)} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3789
|
+
conversation_id: conv.id,
|
|
3790
|
+
message_id: message.id
|
|
3791
|
+
});
|
|
3792
|
+
return { ok: false, signature: null };
|
|
3793
|
+
}
|
|
3794
|
+
}
|
|
3795
|
+
/**
|
|
3796
|
+
* The re-drive fence for a `pending` row that already carries a stored
|
|
3797
|
+
* `opencode_message_id` (#965) — i.e. it has already been handed to opencode at
|
|
3798
|
+
* least once (see the invariant at `QueuedMessage.opencode_message_id`'s doc).
|
|
3799
|
+
* The lifecycle cron can falsely reclaim a `processing` row back to `pending`
|
|
3800
|
+
* mid-turn (a 5-minute liveness-staleness check racing a still-running turn);
|
|
3801
|
+
* without this fence the drain loop would re-`prompt_async` the SAME turn a
|
|
3802
|
+
* second time against live GitHub state. Mirrors `readoptOne`'s job for the
|
|
3803
|
+
* `processing` re-adopt path, but simpler: no b1/b2 preamble cross-check is
|
|
3804
|
+
* needed here because `refusedSessionId` already handles the one case
|
|
3805
|
+
* (#553 abandoned session) that path exists for.
|
|
3806
|
+
*
|
|
3807
|
+
* Only `ChannelAuthError` propagates. A poll that fails identically
|
|
3808
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row reports the message
|
|
3809
|
+
* failed instead of retrying it (#1348) — SEPARATE from, not a replacement
|
|
3810
|
+
* for, `resolveRedriveUnresolved`'s own `pausedMaxWaitMs` bound below. Every
|
|
3811
|
+
* other failure resolves to `unresolved` and is retried whole on the next
|
|
3812
|
+
* ~2s drain tick.
|
|
3813
|
+
*/
|
|
3814
|
+
async resolveRedrive(conv, sessionId, message, refusedSessionId) {
|
|
3815
|
+
const ocId = message.opencode_message_id ?? null;
|
|
3816
|
+
if (refusedSessionId) {
|
|
3817
|
+
this.clearRedriveUnresolved(message.id);
|
|
3818
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3819
|
+
return "dispatch";
|
|
3820
|
+
}
|
|
3821
|
+
const polled = await this.pollSessionMessagesForRedrive(conv, message, sessionId);
|
|
3822
|
+
if (!polled.ok) {
|
|
3823
|
+
const streak = this.recordRedrivePollFailure(message.id, sessionId, polled.signature);
|
|
3824
|
+
if (streak >= MAX_IDENTICAL_REDRIVE_POLL_FAILURES && polled.signature !== null) {
|
|
3825
|
+
return this.failRedrivePollPermanent(conv, sessionId, message, polled.signature, streak);
|
|
3826
|
+
}
|
|
3827
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3828
|
+
}
|
|
3829
|
+
this.redrivePollFailures.delete(message.id);
|
|
3830
|
+
const messages = polled.messages;
|
|
3831
|
+
if (messages.length === 0) {
|
|
3832
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3833
|
+
}
|
|
3834
|
+
const state = messageRunState(messages, ocId ?? "");
|
|
3835
|
+
if (state === "failed" && isAbortedTerminalReply(messages, ocId ?? "")) {
|
|
3836
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
3837
|
+
if (ongoing === false) {
|
|
3838
|
+
this.log({
|
|
3839
|
+
level: "info",
|
|
3840
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
3841
|
+
conversation_id: conv.id,
|
|
3842
|
+
message_id: message.id
|
|
3843
|
+
});
|
|
3844
|
+
this.clearRedriveUnresolved(message.id);
|
|
3845
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3846
|
+
return "dispatch";
|
|
3847
|
+
}
|
|
3848
|
+
}
|
|
3849
|
+
if (state === "done" || state === "failed") {
|
|
3850
|
+
return this.settleRedrive(conv, sessionId, message, ocId, messages, state);
|
|
3851
|
+
}
|
|
3852
|
+
if (state === "running" || state === "queued") {
|
|
3853
|
+
const ongoing = await isSessionOngoing(this.port, sessionId);
|
|
3854
|
+
if (ongoing === true) {
|
|
3855
|
+
return this.reattachRedrive(conv, sessionId, message, ocId);
|
|
3856
|
+
}
|
|
3857
|
+
if (ongoing === false) {
|
|
3858
|
+
this.clearRedriveUnresolved(message.id);
|
|
3859
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3860
|
+
return "dispatch";
|
|
3861
|
+
}
|
|
3862
|
+
return this.resolveRedriveUnresolved(conv, message);
|
|
3863
|
+
}
|
|
3864
|
+
this.clearRedriveUnresolved(message.id);
|
|
3865
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
3866
|
+
return "dispatch";
|
|
3867
|
+
}
|
|
3868
|
+
/**
|
|
3869
|
+
* The `reattached` outcome (Task 3.3): the prior turn is STILL ONGOING per
|
|
3870
|
+
* opencode's own status map — undo the false reclaim instead of starting a
|
|
3871
|
+
* second turn.
|
|
3872
|
+
*/
|
|
3873
|
+
async reattachRedrive(conv, sessionId, message, ocId) {
|
|
3874
|
+
let anchorMs;
|
|
3875
|
+
const parsed = message.processing_started_at ? Date.parse(message.processing_started_at) : NaN;
|
|
3876
|
+
if (!Number.isNaN(parsed)) {
|
|
3877
|
+
anchorMs = parsed;
|
|
3878
|
+
} else {
|
|
3879
|
+
anchorMs = this.now();
|
|
3880
|
+
this.log({
|
|
3881
|
+
level: "error",
|
|
3882
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} has null/unparseable processing_started_at (${String(message.processing_started_at)}) \u2014 anchoring the watcher's absolute-age ceiling to now (defensive)`,
|
|
3883
|
+
conversation_id: conv.id,
|
|
3884
|
+
message_id: message.id
|
|
3885
|
+
});
|
|
3886
|
+
}
|
|
3887
|
+
const title = await this.resolveSessionTitle(sessionId, conv.id);
|
|
3888
|
+
try {
|
|
3889
|
+
await this.markProcessing(conv.id, message.id, sessionId, ocId, title);
|
|
3890
|
+
} catch (err) {
|
|
3891
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3892
|
+
this.log({
|
|
3893
|
+
level: "warn",
|
|
3894
|
+
message: `Re-drive: failed to restore message ${message.id.slice(0, 8)} to processing (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3895
|
+
conversation_id: conv.id,
|
|
3896
|
+
message_id: message.id
|
|
3897
|
+
});
|
|
3898
|
+
return "unresolved";
|
|
3899
|
+
}
|
|
3900
|
+
this.clearRedriveUnresolved(message.id);
|
|
3901
|
+
this.registerReadopted(conv, sessionId, message, ocId ?? "", anchorMs);
|
|
3902
|
+
this.dispatched.add(message.id);
|
|
3903
|
+
this.readopted.add(message.id);
|
|
3904
|
+
this.ensureWatcherRunning(sessionId);
|
|
3905
|
+
const watchedForMs = this.now() - anchorMs;
|
|
3906
|
+
void this.postSignal(conv.id, message.id, "redrive_reattached", {
|
|
3907
|
+
watched_for_ms: watchedForMs
|
|
3908
|
+
});
|
|
3909
|
+
this.log({
|
|
3910
|
+
level: "warn",
|
|
3911
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) was wrongly reclaimed to pending while its turn was still running (watched ${watchedForMs}ms) \u2014 restored to processing instead of re-dispatching`,
|
|
3912
|
+
conversation_id: conv.id,
|
|
3913
|
+
message_id: message.id
|
|
3914
|
+
});
|
|
3915
|
+
return "reattached";
|
|
3916
|
+
}
|
|
3917
|
+
/**
|
|
3918
|
+
* The `settled` outcome (Task 3.2): the prior turn already finished (or
|
|
3919
|
+
* errored) while nobody was watching — deliver/report it instead of re-running.
|
|
3920
|
+
* Mirrors `readoptOne`'s `done`/`failed` branches' error discipline, simplified
|
|
3921
|
+
* (no `doneUndeliverable` park: a terminal PATCH failure here just retries next
|
|
3922
|
+
* drain, same as any other non-auth failure). The restart-abort carve-out that
|
|
3923
|
+
* keeps the two in step for `failed` lives in the caller (`resolveRedrive`, #1310),
|
|
3924
|
+
* so a row reaching this `failed` branch is a GENUINE failure.
|
|
3925
|
+
*/
|
|
3926
|
+
async settleRedrive(conv, sessionId, message, ocId, messages, state) {
|
|
3927
|
+
try {
|
|
3928
|
+
if (state === "done") {
|
|
3929
|
+
const title = await this.resolveSessionTitle(sessionId, conv.id);
|
|
3930
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
3931
|
+
this.log({
|
|
3932
|
+
level: "info",
|
|
3933
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} completed while its row was wrongly reclaimed to pending \u2014 marking done instead of re-dispatching`,
|
|
2992
3934
|
conversation_id: conv.id,
|
|
2993
3935
|
message_id: message.id
|
|
2994
3936
|
});
|
|
2995
|
-
await this.
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
message_id: message.id
|
|
3001
|
-
});
|
|
3002
|
-
});
|
|
3937
|
+
await this.markDone(conv.id, message.id, sessionId, ocId, title, usage);
|
|
3938
|
+
} else {
|
|
3939
|
+
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3940
|
+
const usage = messageUsage(messages, ocId ?? "");
|
|
3941
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3003
3942
|
this.log({
|
|
3004
3943
|
level: "error",
|
|
3005
|
-
message: `
|
|
3006
|
-
conversation_id: conv.id,
|
|
3007
|
-
message_id: message.id
|
|
3008
|
-
});
|
|
3009
|
-
break;
|
|
3010
|
-
}
|
|
3011
|
-
if (opencodeMessageId === null) {
|
|
3012
|
-
this.log({
|
|
3013
|
-
level: "warn",
|
|
3014
|
-
message: `Message ${message.id.slice(0, 8)} dispatched but its opencode id could not be read back \u2014 leaving un-tracked to retry next tick`,
|
|
3944
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} errored while its row was wrongly reclaimed to pending \u2014 marking failed instead of re-dispatching: ${error2 ?? "(no error text)"}`,
|
|
3015
3945
|
conversation_id: conv.id,
|
|
3016
3946
|
message_id: message.id
|
|
3017
3947
|
});
|
|
3018
|
-
|
|
3948
|
+
await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
|
|
3019
3949
|
}
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3950
|
+
} catch (err) {
|
|
3951
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3952
|
+
this.log({
|
|
3953
|
+
level: "warn",
|
|
3954
|
+
message: `Re-drive: failed to report message ${message.id.slice(0, 8)} ${state} (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3955
|
+
conversation_id: conv.id,
|
|
3956
|
+
message_id: message.id
|
|
3957
|
+
});
|
|
3958
|
+
return "unresolved";
|
|
3024
3959
|
}
|
|
3025
|
-
|
|
3960
|
+
this.clearRedriveUnresolved(message.id);
|
|
3961
|
+
void this.postSignal(conv.id, message.id, "redrive_settled");
|
|
3962
|
+
return "settled";
|
|
3963
|
+
}
|
|
3964
|
+
/**
|
|
3965
|
+
* The permanent-failure outcome (#1348): the fence's own poll of this session
|
|
3966
|
+
* failed with the SAME opencode-answered signature
|
|
3967
|
+
* `MAX_IDENTICAL_REDRIVE_POLL_FAILURES` times in a row — a transient blip
|
|
3968
|
+
* would have varied or eventually cleared (see `pollSessionMessagesForRedrive`
|
|
3969
|
+
* and `recordRedrivePollFailure`), so this is a durable fault (e.g. #1345's
|
|
3970
|
+
* corrupted opencode session) rather than something worth retrying forever.
|
|
3971
|
+
* Mirrors `settleRedrive`'s error discipline: no `usage`/`failure` args to
|
|
3972
|
+
* `markFailed` (no opencode snapshot to extract them from — this poll never
|
|
3973
|
+
* got a readable one).
|
|
3974
|
+
*/
|
|
3975
|
+
async failRedrivePollPermanent(conv, sessionId, message, signature, streak) {
|
|
3976
|
+
this.log({
|
|
3977
|
+
level: "error",
|
|
3978
|
+
message: `Re-drive: message ${message.id.slice(0, 8)} (session ${sessionId.slice(0, 8)}) failed to poll with the identical signature "${signature}" ${streak} times in a row \u2014 reporting the message failed instead of retrying forever`,
|
|
3979
|
+
conversation_id: conv.id,
|
|
3980
|
+
message_id: message.id
|
|
3981
|
+
});
|
|
3982
|
+
try {
|
|
3983
|
+
await this.markFailed(
|
|
3984
|
+
conv.id,
|
|
3985
|
+
message.id,
|
|
3986
|
+
sessionId,
|
|
3987
|
+
`The runner could not read this conversation's state from OpenCode (${signature}). The same failure repeated ${streak} times in a row, so the message was not retried further.`
|
|
3988
|
+
);
|
|
3989
|
+
} catch (err) {
|
|
3990
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
3026
3991
|
this.log({
|
|
3027
3992
|
level: "warn",
|
|
3028
|
-
message: `
|
|
3029
|
-
conversation_id: conv.id
|
|
3993
|
+
message: `Re-drive: failed to report message ${message.id.slice(0, 8)} permanently failed (will retry next drain): ${err instanceof Error ? err.message : String(err)}`,
|
|
3994
|
+
conversation_id: conv.id,
|
|
3995
|
+
message_id: message.id
|
|
3030
3996
|
});
|
|
3997
|
+
return "unresolved";
|
|
3031
3998
|
}
|
|
3032
|
-
this.
|
|
3033
|
-
|
|
3999
|
+
this.clearRedriveUnresolved(message.id);
|
|
4000
|
+
void this.postSignal(conv.id, message.id, "redrive_poll_failed");
|
|
4001
|
+
return "settled";
|
|
4002
|
+
}
|
|
4003
|
+
/**
|
|
4004
|
+
* The bounded `unresolved` outcome (Task 3.4): opencode's state could not be
|
|
4005
|
+
* observed (snapshot unreadable/empty, or `isSessionOngoing` returned `null`).
|
|
4006
|
+
* A `pending` row is invisible to every cron arm (all require `status =
|
|
4007
|
+
* 'processing'`), so an indefinitely-`unresolved` row would be stranded with
|
|
4008
|
+
* nothing driving it — bound it to the existing `pausedMaxWaitMs` window
|
|
4009
|
+
* (reusing the knob, not a new constant) and take `dispatch` once elapsed.
|
|
4010
|
+
*/
|
|
4011
|
+
resolveRedriveUnresolved(conv, message) {
|
|
4012
|
+
const now = this.now();
|
|
4013
|
+
const since = this.redriveUnresolvedSince.get(message.id);
|
|
4014
|
+
if (since !== void 0 && now - since >= this.pausedMaxWaitMs) {
|
|
4015
|
+
this.clearRedriveUnresolved(message.id);
|
|
4016
|
+
void this.postSignal(conv.id, message.id, "redrive_redispatched");
|
|
4017
|
+
return "dispatch";
|
|
4018
|
+
}
|
|
4019
|
+
if (since === void 0) {
|
|
4020
|
+
this.redriveUnresolvedSince.set(message.id, now);
|
|
4021
|
+
}
|
|
4022
|
+
if (!this.redriveUnresolvedSignalled.has(message.id)) {
|
|
4023
|
+
this.redriveUnresolvedSignalled.add(message.id);
|
|
4024
|
+
void this.postSignal(conv.id, message.id, "redrive_unresolved");
|
|
4025
|
+
}
|
|
4026
|
+
return "unresolved";
|
|
4027
|
+
}
|
|
4028
|
+
/** Clear all three `unresolved`/failure-streak trackers for a row (any non-`unresolved` outcome). */
|
|
4029
|
+
clearRedriveUnresolved(messageId) {
|
|
4030
|
+
this.redriveUnresolvedSince.delete(messageId);
|
|
4031
|
+
this.redriveUnresolvedSignalled.delete(messageId);
|
|
4032
|
+
this.redrivePollFailures.delete(messageId);
|
|
4033
|
+
}
|
|
4034
|
+
/**
|
|
4035
|
+
* Record one poll outcome toward the re-drive fence's consecutive-identical-
|
|
4036
|
+
* failure streak (#1348) and return the resulting count. `signature === null`
|
|
4037
|
+
* (a thrown exception, H1) always clears the streak and returns `0` — it is
|
|
4038
|
+
* never countable. Otherwise the streak continues only when BOTH the session
|
|
4039
|
+
* and the signature match the previous failure; anything else (a different
|
|
4040
|
+
* session, or the same session failing a DIFFERENT way) starts a fresh streak
|
|
4041
|
+
* at `1`.
|
|
4042
|
+
*/
|
|
4043
|
+
recordRedrivePollFailure(messageId, sessionId, signature) {
|
|
4044
|
+
if (signature === null) {
|
|
4045
|
+
this.redrivePollFailures.delete(messageId);
|
|
4046
|
+
return 0;
|
|
4047
|
+
}
|
|
4048
|
+
const existing = this.redrivePollFailures.get(messageId);
|
|
4049
|
+
if (existing && existing.sessionId === sessionId && existing.signature === signature) {
|
|
4050
|
+
existing.count += 1;
|
|
4051
|
+
return existing.count;
|
|
4052
|
+
}
|
|
4053
|
+
this.redrivePollFailures.set(messageId, { sessionId, signature, count: 1 });
|
|
4054
|
+
return 1;
|
|
3034
4055
|
}
|
|
3035
4056
|
/**
|
|
3036
4057
|
* Record that `sessionId` is no longer a valid binding for `conversationId`
|
|
@@ -3093,7 +4114,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3093
4114
|
const directory = await this.resolveOpenCodeDirectory();
|
|
3094
4115
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
3095
4116
|
this.sessions.set(conversationId, sessionId);
|
|
3096
|
-
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
4117
|
+
await this.persistSession(conversationId, sessionId).catch((err) => {
|
|
4118
|
+
this.log({
|
|
4119
|
+
level: "warn",
|
|
4120
|
+
message: `Persisting the OpenCode session binding ${sessionId.slice(0, 8)} for conversation ${conversationId.slice(0, 8)} failed (best-effort, not retried) \u2014 the completion PATCH also carries opencode_session_id, so the binding is repaired when the turn finishes: ${err instanceof Error ? err.message : String(err)}`,
|
|
4121
|
+
conversation_id: conversationId
|
|
4122
|
+
});
|
|
3097
4123
|
});
|
|
3098
4124
|
return sessionId;
|
|
3099
4125
|
}
|
|
@@ -3285,12 +4311,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3285
4311
|
stuckReported: false,
|
|
3286
4312
|
lastAliveAt: 0,
|
|
3287
4313
|
aliveInFlight: false,
|
|
4314
|
+
titleSynced: false,
|
|
4315
|
+
titleSyncInFlight: false,
|
|
3288
4316
|
awaitingHumanLatched: false,
|
|
3289
4317
|
pausedOnQuestion: false,
|
|
3290
4318
|
pausedOnPermission: false,
|
|
3291
4319
|
pausedClearConfirmed: false,
|
|
3292
4320
|
pausedInFlight: false,
|
|
3293
|
-
deliveryDeadlineAnchored: false
|
|
4321
|
+
deliveryDeadlineAnchored: false,
|
|
4322
|
+
b2PinnedSinceMs: 0,
|
|
4323
|
+
b2LastDescendantCheckMs: 0,
|
|
4324
|
+
b2AbandonedSignalled: false
|
|
3294
4325
|
});
|
|
3295
4326
|
}
|
|
3296
4327
|
/**
|
|
@@ -3305,9 +4336,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3305
4336
|
* opencode reports ACTIVELY `running` is watched to completion (its liveness
|
|
3306
4337
|
* heartbeat keeps the cron off its row), while a re-adopted turn that is paused
|
|
3307
4338
|
* awaiting a human — or queued/unreachable — is still bounded by `deadline` and
|
|
3308
|
-
* handed to the cron.
|
|
3309
|
-
*
|
|
3310
|
-
*
|
|
4339
|
+
* handed to the cron. Real invariant (#965): the cron MAY reclaim a row this
|
|
4340
|
+
* runner still holds; a reclaimed row that already ran is never re-dispatched
|
|
4341
|
+
* while opencode reports its turn ongoing (readopt's own gate here, and the
|
|
4342
|
+
* `pending`-row re-drive fence, `resolveRedrive`). `dispatchedAt` stays `now`
|
|
3311
4343
|
* (only the appear-guard uses it).
|
|
3312
4344
|
*
|
|
3313
4345
|
* `evidentMessageId` addresses the SERVER row (for markProcessing/markDone);
|
|
@@ -3358,12 +4390,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3358
4390
|
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
3359
4391
|
lastAliveAt: 0,
|
|
3360
4392
|
aliveInFlight: false,
|
|
4393
|
+
titleSynced: false,
|
|
4394
|
+
titleSyncInFlight: false,
|
|
3361
4395
|
awaitingHumanLatched: false,
|
|
3362
4396
|
pausedOnQuestion: false,
|
|
3363
4397
|
pausedOnPermission: false,
|
|
3364
4398
|
pausedClearConfirmed: false,
|
|
3365
4399
|
pausedInFlight: false,
|
|
3366
|
-
deliveryDeadlineAnchored: false
|
|
4400
|
+
deliveryDeadlineAnchored: false,
|
|
4401
|
+
b2PinnedSinceMs: 0,
|
|
4402
|
+
b2LastDescendantCheckMs: 0,
|
|
4403
|
+
b2AbandonedSignalled: false
|
|
3367
4404
|
});
|
|
3368
4405
|
}
|
|
3369
4406
|
/**
|
|
@@ -3525,58 +4562,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3525
4562
|
}
|
|
3526
4563
|
}
|
|
3527
4564
|
if (state === "done") {
|
|
3528
|
-
this.
|
|
3529
|
-
if (!inFlight.done) {
|
|
3530
|
-
this.log({
|
|
3531
|
-
level: "info",
|
|
3532
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3533
|
-
conversation_id: conv.id,
|
|
3534
|
-
message_id: inFlight.evidentMessageId
|
|
3535
|
-
});
|
|
3536
|
-
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3537
|
-
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3538
|
-
try {
|
|
3539
|
-
await this.markDone(
|
|
3540
|
-
conv.id,
|
|
3541
|
-
inFlight.evidentMessageId,
|
|
3542
|
-
sessionId,
|
|
3543
|
-
inFlight.opencodeMessageId,
|
|
3544
|
-
title,
|
|
3545
|
-
usage
|
|
3546
|
-
);
|
|
3547
|
-
} catch (err) {
|
|
3548
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
3549
|
-
if (err instanceof ChannelTerminalError) {
|
|
3550
|
-
this.log({
|
|
3551
|
-
level: "warn",
|
|
3552
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3553
|
-
conversation_id: conv.id,
|
|
3554
|
-
message_id: inFlight.evidentMessageId
|
|
3555
|
-
});
|
|
3556
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3557
|
-
return;
|
|
3558
|
-
}
|
|
3559
|
-
if (this.now() >= inFlight.deadline) {
|
|
3560
|
-
this.log({
|
|
3561
|
-
level: "warn",
|
|
3562
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3563
|
-
conversation_id: conv.id,
|
|
3564
|
-
message_id: inFlight.evidentMessageId
|
|
3565
|
-
});
|
|
3566
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3567
|
-
return;
|
|
3568
|
-
}
|
|
3569
|
-
this.log({
|
|
3570
|
-
level: "warn",
|
|
3571
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3572
|
-
conversation_id: conv.id,
|
|
3573
|
-
message_id: inFlight.evidentMessageId
|
|
3574
|
-
});
|
|
3575
|
-
return;
|
|
3576
|
-
}
|
|
3577
|
-
inFlight.done = true;
|
|
3578
|
-
}
|
|
3579
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4565
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3580
4566
|
return;
|
|
3581
4567
|
}
|
|
3582
4568
|
if (state === "failed") {
|
|
@@ -3590,8 +4576,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3590
4576
|
message_id: inFlight.evidentMessageId
|
|
3591
4577
|
});
|
|
3592
4578
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
4579
|
+
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
3593
4580
|
try {
|
|
3594
|
-
await this.markFailed(
|
|
4581
|
+
await this.markFailed(
|
|
4582
|
+
conv.id,
|
|
4583
|
+
inFlight.evidentMessageId,
|
|
4584
|
+
sessionId,
|
|
4585
|
+
error2,
|
|
4586
|
+
usage,
|
|
4587
|
+
failure
|
|
4588
|
+
);
|
|
3595
4589
|
} catch (err) {
|
|
3596
4590
|
if (err instanceof ChannelAuthError) throw err;
|
|
3597
4591
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3636,6 +4630,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3636
4630
|
});
|
|
3637
4631
|
}
|
|
3638
4632
|
const activelyRunning = state === "running" && !awaitingHuman;
|
|
4633
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
4634
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
4635
|
+
if (!pinnedNow) {
|
|
4636
|
+
if (snapshotReadable) {
|
|
4637
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
4638
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
4639
|
+
inFlight.b2AbandonedSignalled = false;
|
|
4640
|
+
}
|
|
4641
|
+
} else {
|
|
4642
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
4643
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
4644
|
+
return;
|
|
4645
|
+
}
|
|
4646
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
4647
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
4648
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
4649
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
4650
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
4651
|
+
if (isB2AbandonmentConfirmed({
|
|
4652
|
+
pinnedForMs,
|
|
4653
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
4654
|
+
descendantOngoing
|
|
4655
|
+
})) {
|
|
4656
|
+
inFlight.b2AbandonedSignalled = true;
|
|
4657
|
+
this.log({
|
|
4658
|
+
level: "warn",
|
|
4659
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
4660
|
+
conversation_id: conv.id,
|
|
4661
|
+
message_id: id
|
|
4662
|
+
});
|
|
4663
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
4664
|
+
watched_for_ms: pinnedForMs
|
|
4665
|
+
});
|
|
4666
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
4667
|
+
return;
|
|
4668
|
+
}
|
|
4669
|
+
}
|
|
4670
|
+
}
|
|
3639
4671
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
3640
4672
|
this.log({
|
|
3641
4673
|
level: "warn",
|
|
@@ -3655,6 +4687,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3655
4687
|
inFlight.aliveInFlight = false;
|
|
3656
4688
|
if (ok) inFlight.lastAliveAt = this.now();
|
|
3657
4689
|
});
|
|
4690
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
4691
|
+
inFlight.titleSyncInFlight = true;
|
|
4692
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
4693
|
+
if (!title) {
|
|
4694
|
+
inFlight.titleSyncInFlight = false;
|
|
4695
|
+
return;
|
|
4696
|
+
}
|
|
4697
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
4698
|
+
inFlight.titleSyncInFlight = false;
|
|
4699
|
+
if (ok) inFlight.titleSynced = true;
|
|
4700
|
+
});
|
|
4701
|
+
}
|
|
3658
4702
|
}
|
|
3659
4703
|
if (awaitingHuman) {
|
|
3660
4704
|
if (!inFlight.awaitingHumanLatched) {
|
|
@@ -3692,6 +4736,70 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3692
4736
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3693
4737
|
}
|
|
3694
4738
|
}
|
|
4739
|
+
/**
|
|
4740
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
4741
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
4742
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
4743
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
4744
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
4745
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
4746
|
+
* and risking the two copies silently drifting apart.
|
|
4747
|
+
*/
|
|
4748
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
4749
|
+
const conv = watcher.conv;
|
|
4750
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
4751
|
+
if (!inFlight.done) {
|
|
4752
|
+
this.log({
|
|
4753
|
+
level: "info",
|
|
4754
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
4755
|
+
conversation_id: conv.id,
|
|
4756
|
+
message_id: inFlight.evidentMessageId
|
|
4757
|
+
});
|
|
4758
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
4759
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
4760
|
+
try {
|
|
4761
|
+
await this.markDone(
|
|
4762
|
+
conv.id,
|
|
4763
|
+
inFlight.evidentMessageId,
|
|
4764
|
+
sessionId,
|
|
4765
|
+
inFlight.opencodeMessageId,
|
|
4766
|
+
title,
|
|
4767
|
+
usage
|
|
4768
|
+
);
|
|
4769
|
+
} catch (err) {
|
|
4770
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4771
|
+
if (err instanceof ChannelTerminalError) {
|
|
4772
|
+
this.log({
|
|
4773
|
+
level: "warn",
|
|
4774
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
4775
|
+
conversation_id: conv.id,
|
|
4776
|
+
message_id: inFlight.evidentMessageId
|
|
4777
|
+
});
|
|
4778
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4779
|
+
return;
|
|
4780
|
+
}
|
|
4781
|
+
if (this.now() >= inFlight.deadline) {
|
|
4782
|
+
this.log({
|
|
4783
|
+
level: "warn",
|
|
4784
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
4785
|
+
conversation_id: conv.id,
|
|
4786
|
+
message_id: inFlight.evidentMessageId
|
|
4787
|
+
});
|
|
4788
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4789
|
+
return;
|
|
4790
|
+
}
|
|
4791
|
+
this.log({
|
|
4792
|
+
level: "warn",
|
|
4793
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
4794
|
+
conversation_id: conv.id,
|
|
4795
|
+
message_id: inFlight.evidentMessageId
|
|
4796
|
+
});
|
|
4797
|
+
return;
|
|
4798
|
+
}
|
|
4799
|
+
inFlight.done = true;
|
|
4800
|
+
}
|
|
4801
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4802
|
+
}
|
|
3695
4803
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
3696
4804
|
/**
|
|
3697
4805
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -3791,7 +4899,10 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3791
4899
|
* re-dispatched (at most once, see `forceReadoptRun`):
|
|
3792
4900
|
* - `done` → `markDone` now (guarded like the watcher's done branch);
|
|
3793
4901
|
* - `failed` → `markFailed` with the surfaced error (issue #182), so an
|
|
3794
|
-
* errored turn is reported failed on restart, NOT re-dispatched
|
|
4902
|
+
* errored turn is reported failed on restart, NOT re-dispatched —
|
|
4903
|
+
* EXCEPT a restart-ABORTED turn under a not-ongoing session,
|
|
4904
|
+
* which is a restart orphan wearing a terminal error and is
|
|
4905
|
+
* re-dispatched instead (issue #1310, see the branch below);
|
|
3795
4906
|
* - `running`/`queued` → re-attach a watcher via `registerReadopted` (no re-dispatch),
|
|
3796
4907
|
* tracking the stored id so the reply correlates by it;
|
|
3797
4908
|
* - `unknown`/null id → re-dispatch (opencode assigns a fresh id) + attach a watcher.
|
|
@@ -3855,9 +4966,19 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3855
4966
|
void this.postSignal(row.conversation_id, row.id, "readopt_done");
|
|
3856
4967
|
return;
|
|
3857
4968
|
}
|
|
3858
|
-
|
|
4969
|
+
const restartAborted = state === "failed" && sessionOngoing === false && isAbortedTerminalReply(messages, ocId ?? "");
|
|
4970
|
+
if (restartAborted) {
|
|
4971
|
+
this.log({
|
|
4972
|
+
level: "info",
|
|
4973
|
+
message: `Re-adopt: message ${row.id.slice(0, 8)} carries an abort-shaped terminal error and session ${sessionId.slice(0, 8)} is not-ongoing per GET /session/status \u2014 restart orphan, re-dispatching instead of marking it permanently failed`,
|
|
4974
|
+
conversation_id: row.conversation_id,
|
|
4975
|
+
message_id: row.id
|
|
4976
|
+
});
|
|
4977
|
+
}
|
|
4978
|
+
if (state === "failed" && !restartAborted) {
|
|
3859
4979
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3860
4980
|
const usage = messageUsage(messages, ocId ?? "");
|
|
4981
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3861
4982
|
this.log({
|
|
3862
4983
|
level: "error",
|
|
3863
4984
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -3865,7 +4986,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3865
4986
|
message_id: row.id
|
|
3866
4987
|
});
|
|
3867
4988
|
try {
|
|
3868
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
4989
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage, failure);
|
|
3869
4990
|
} catch (err) {
|
|
3870
4991
|
if (err instanceof ChannelAuthError) throw err;
|
|
3871
4992
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -4132,7 +5253,8 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4132
5253
|
opencode_model: row.opencode_model,
|
|
4133
5254
|
source_message_id: row.source_message_id,
|
|
4134
5255
|
slack_user_id: row.slack_user_id,
|
|
4135
|
-
attachments: row.attachments ?? null
|
|
5256
|
+
attachments: row.attachments ?? null,
|
|
5257
|
+
opencode_message_id: row.opencode_message_id
|
|
4136
5258
|
};
|
|
4137
5259
|
}
|
|
4138
5260
|
/**
|
|
@@ -4271,6 +5393,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4271
5393
|
}
|
|
4272
5394
|
return false;
|
|
4273
5395
|
}
|
|
5396
|
+
/**
|
|
5397
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
5398
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
5399
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
5400
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
5401
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
5402
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
5403
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
5404
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
5405
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
5406
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
5407
|
+
* not ongoing".
|
|
5408
|
+
*
|
|
5409
|
+
* Return contract:
|
|
5410
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
5411
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
5412
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
5413
|
+
* CONFIRMED NOT a descendant of it.
|
|
5414
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
5415
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
5416
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
5417
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
5418
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
5419
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
5420
|
+
* here.
|
|
5421
|
+
*
|
|
5422
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
5423
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
5424
|
+
* by interaction attribution or the recovery path.
|
|
5425
|
+
*/
|
|
5426
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
5427
|
+
let current = sessionId;
|
|
5428
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
5429
|
+
if (current === rootSessionId) return true;
|
|
5430
|
+
const parent = await this.resolveSessionParent(current);
|
|
5431
|
+
if (parent === void 0) return null;
|
|
5432
|
+
if (parent === null) return false;
|
|
5433
|
+
current = parent;
|
|
5434
|
+
}
|
|
5435
|
+
return null;
|
|
5436
|
+
}
|
|
4274
5437
|
/**
|
|
4275
5438
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
4276
5439
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -4355,6 +5518,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4355
5518
|
}
|
|
4356
5519
|
return null;
|
|
4357
5520
|
}
|
|
5521
|
+
/**
|
|
5522
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
5523
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
5524
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
5525
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
5526
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
5527
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
5528
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
5529
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
5530
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
5531
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
5532
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
5533
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
5534
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
5535
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
5536
|
+
*
|
|
5537
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
5538
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
5539
|
+
* caller only latches `titleSynced` on `true`).
|
|
5540
|
+
*/
|
|
5541
|
+
async patchConversationTitle(conversationId, title) {
|
|
5542
|
+
try {
|
|
5543
|
+
const res = await this.fetchImpl(
|
|
5544
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
5545
|
+
{
|
|
5546
|
+
method: "PATCH",
|
|
5547
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
5548
|
+
body: JSON.stringify({ title })
|
|
5549
|
+
}
|
|
5550
|
+
);
|
|
5551
|
+
if (!res.ok) {
|
|
5552
|
+
this.log({
|
|
5553
|
+
level: "debug",
|
|
5554
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
5555
|
+
conversation_id: conversationId
|
|
5556
|
+
});
|
|
5557
|
+
return false;
|
|
5558
|
+
}
|
|
5559
|
+
return true;
|
|
5560
|
+
} catch (err) {
|
|
5561
|
+
this.log({
|
|
5562
|
+
level: "debug",
|
|
5563
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
5564
|
+
conversation_id: conversationId
|
|
5565
|
+
});
|
|
5566
|
+
return false;
|
|
5567
|
+
}
|
|
5568
|
+
}
|
|
4358
5569
|
/**
|
|
4359
5570
|
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
4360
5571
|
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
@@ -4387,31 +5598,109 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4387
5598
|
* longer exists". The inversion lives in the caller; this method just reports
|
|
4388
5599
|
* true/false/null faithfully.
|
|
4389
5600
|
*
|
|
4390
|
-
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
4391
|
-
* (already proven by the existing child-session interaction tests, via
|
|
4392
|
-
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
4393
|
-
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
4394
|
-
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
4395
|
-
* `SessionStatus` only.
|
|
5601
|
+
* VERIFY-BEFORE-DEPEND: we depend ONLY on (a) `parentID` from `GET /session/:id`
|
|
5602
|
+
* (already proven by the existing child-session interaction tests, via
|
|
5603
|
+
* `resolveSessionParent`/`sessionBelongsTo`) and (b) the child's own message-list
|
|
5604
|
+
* terminal state. We do NOT depend on any session-level `busy`/`idle` field —
|
|
5605
|
+
* there is none on `GET /session/:id`; OpenCode's busy state is in-memory
|
|
5606
|
+
* `SessionStatus` only.
|
|
5607
|
+
*/
|
|
5608
|
+
async isAnyDescendantSessionAlive(rootSessionId) {
|
|
5609
|
+
const sessions = await listSessions(this.port);
|
|
5610
|
+
if (!sessions) {
|
|
5611
|
+
this.log({
|
|
5612
|
+
level: "warn",
|
|
5613
|
+
message: `Re-adopt: could not enumerate sessions to cross-check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating child liveness as indeterminate`
|
|
5614
|
+
});
|
|
5615
|
+
return null;
|
|
5616
|
+
}
|
|
5617
|
+
for (const candidate of sessions) {
|
|
5618
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
5619
|
+
if (!await this.sessionBelongsTo(candidate.id, rootSessionId)) continue;
|
|
5620
|
+
const childMsgs = await getSessionMessages(this.port, candidate.id);
|
|
5621
|
+
if (isSessionActivelyGenerating(childMsgs)) {
|
|
5622
|
+
return true;
|
|
5623
|
+
}
|
|
5624
|
+
}
|
|
5625
|
+
return false;
|
|
5626
|
+
}
|
|
5627
|
+
/**
|
|
5628
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
5629
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
5630
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
5631
|
+
*
|
|
5632
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
5633
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
5634
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
5635
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
5636
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
5637
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
5638
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
5639
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
5640
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
5641
|
+
* executing, between its step's completion and the next generation step"
|
|
5642
|
+
* transcript gap that a transcript-based check would need a second,
|
|
5643
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
5644
|
+
* message timestamps at all.
|
|
5645
|
+
*
|
|
5646
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
5647
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
5648
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
5649
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
5650
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
5651
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
5652
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
5653
|
+
* delegation — which the root's status is not.
|
|
5654
|
+
*
|
|
5655
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
5656
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
5657
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
5658
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
5659
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
5660
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
5661
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
5662
|
+
* instead.
|
|
5663
|
+
*
|
|
5664
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
5665
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
5666
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
5667
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
5668
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
5669
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
5670
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
5671
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
5672
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
5673
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
5674
|
+
* or pathological chain partway through the parent walk), OR at
|
|
5675
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
5676
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
5677
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
5678
|
+
* (unlike the recovery cross-check's contract) — see
|
|
5679
|
+
* `isB2AbandonmentConfirmed`.
|
|
4396
5680
|
*/
|
|
4397
|
-
async
|
|
5681
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
4398
5682
|
const sessions = await listSessions(this.port);
|
|
4399
5683
|
if (!sessions) {
|
|
4400
5684
|
this.log({
|
|
4401
5685
|
level: "warn",
|
|
4402
|
-
message: `
|
|
5686
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
4403
5687
|
});
|
|
4404
5688
|
return null;
|
|
4405
5689
|
}
|
|
5690
|
+
let indeterminate = false;
|
|
4406
5691
|
for (const candidate of sessions) {
|
|
4407
5692
|
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
5693
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
5694
|
+
if (membership === null) {
|
|
5695
|
+
indeterminate = true;
|
|
5696
|
+
continue;
|
|
4412
5697
|
}
|
|
5698
|
+
if (membership === false) continue;
|
|
5699
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
5700
|
+
if (ongoing === true) return true;
|
|
5701
|
+
if (ongoing === null) indeterminate = true;
|
|
4413
5702
|
}
|
|
4414
|
-
return false;
|
|
5703
|
+
return indeterminate ? null : false;
|
|
4415
5704
|
}
|
|
4416
5705
|
/**
|
|
4417
5706
|
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
@@ -4557,11 +5846,11 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4557
5846
|
* `ensureSession` guard, not this suppression, is what makes the self-heal
|
|
4558
5847
|
* stick.
|
|
4559
5848
|
*/
|
|
4560
|
-
sessionIdBody(sessionId, conversationId, messageId,
|
|
5849
|
+
sessionIdBody(sessionId, conversationId, messageId, status2) {
|
|
4561
5850
|
if (!this.isSuperseded(conversationId, sessionId)) return { opencode_session_id: sessionId };
|
|
4562
5851
|
this.log({
|
|
4563
5852
|
level: "debug",
|
|
4564
|
-
message: `Omitting the abandoned OpenCode session ${sessionId.slice(0, 8)} from the '${
|
|
5853
|
+
message: `Omitting the abandoned OpenCode session ${sessionId.slice(0, 8)} from the '${status2}' update for message ${messageId.slice(0, 8)} so it is not re-bound to conversation ${conversationId.slice(0, 8)}`,
|
|
4565
5854
|
conversation_id: conversationId,
|
|
4566
5855
|
message_id: messageId
|
|
4567
5856
|
});
|
|
@@ -4676,7 +5965,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4676
5965
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
4677
5966
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
4678
5967
|
*/
|
|
4679
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
5968
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
4680
5969
|
const body = { status: "failed" };
|
|
4681
5970
|
if (sessionId === null) {
|
|
4682
5971
|
body.opencode_session_id = null;
|
|
@@ -4685,6 +5974,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4685
5974
|
}
|
|
4686
5975
|
if (error2 !== void 0) body.error = error2;
|
|
4687
5976
|
if (usage) Object.assign(body, usage);
|
|
5977
|
+
if (failure) {
|
|
5978
|
+
body.failure_kind = failure.kind;
|
|
5979
|
+
body.failure_provider_id = failure.providerId;
|
|
5980
|
+
body.failure_model_id = failure.modelId;
|
|
5981
|
+
body.failure_reason = failure.reason;
|
|
5982
|
+
}
|
|
4688
5983
|
await this.callWithRetry(
|
|
4689
5984
|
"marking message as failed",
|
|
4690
5985
|
() => this.fetchImpl(
|
|
@@ -4697,6 +5992,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4697
5992
|
)
|
|
4698
5993
|
);
|
|
4699
5994
|
}
|
|
5995
|
+
/**
|
|
5996
|
+
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
5997
|
+
*
|
|
5998
|
+
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
5999
|
+
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
6000
|
+
* to the P1-2b zero-provider check — one extra loopback call to
|
|
6001
|
+
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
6002
|
+
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
6003
|
+
* (`null`/indeterminate) leaves the classification `null`, which produces
|
|
6004
|
+
* today's byte-identical PATCH body via `markFailed`'s `if (failure)` guard.
|
|
6005
|
+
*/
|
|
6006
|
+
async classifyModelAuthFailure(messages, userMessageId) {
|
|
6007
|
+
const classified = messageFailure(messages, userMessageId);
|
|
6008
|
+
if (classified != null) return classified;
|
|
6009
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
6010
|
+
const hasProvider = await hasAnyConfiguredProvider(this.port);
|
|
6011
|
+
return applyZeroProviderFallback(
|
|
6012
|
+
classified,
|
|
6013
|
+
hasProvider,
|
|
6014
|
+
reply?.info?.providerID ?? null,
|
|
6015
|
+
reply?.info?.modelID ?? null
|
|
6016
|
+
);
|
|
6017
|
+
}
|
|
4700
6018
|
/**
|
|
4701
6019
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
4702
6020
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -4849,10 +6167,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4849
6167
|
import chalk5 from "chalk";
|
|
4850
6168
|
import ora2 from "ora";
|
|
4851
6169
|
import { select as select2 } from "@inquirer/prompts";
|
|
6170
|
+
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
4852
6171
|
async function ensureOpenCodeRunning(ctx) {
|
|
4853
6172
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
4854
6173
|
if (healthCheck.healthy) {
|
|
4855
|
-
return {
|
|
6174
|
+
return {
|
|
6175
|
+
port: ctx.port,
|
|
6176
|
+
process: null,
|
|
6177
|
+
version: healthCheck.version ?? null,
|
|
6178
|
+
notReadyReason: null
|
|
6179
|
+
};
|
|
4856
6180
|
}
|
|
4857
6181
|
const runningInstances = await findHealthyOpenCodeInstances();
|
|
4858
6182
|
if (runningInstances.length > 0) {
|
|
@@ -4873,7 +6197,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4873
6197
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
4874
6198
|
console.log(
|
|
4875
6199
|
chalk5.dim(
|
|
4876
|
-
` ${getCliName()} run --
|
|
6200
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
4877
6201
|
)
|
|
4878
6202
|
);
|
|
4879
6203
|
}
|
|
@@ -4893,14 +6217,22 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4893
6217
|
if (!ctx.interactive) {
|
|
4894
6218
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
4895
6219
|
const proc = await startOpenCode(ctx.port);
|
|
4896
|
-
const health = await waitForOpenCodeHealth(ctx.port,
|
|
6220
|
+
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
4897
6221
|
if (!health.healthy) {
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
6222
|
+
return {
|
|
6223
|
+
port: ctx.port,
|
|
6224
|
+
process: proc,
|
|
6225
|
+
version: null,
|
|
6226
|
+
notReadyReason: `it did not become healthy within ${Math.round(ctx.startTimeoutMs / 1e3)}s`
|
|
6227
|
+
};
|
|
4901
6228
|
}
|
|
4902
6229
|
ctx.log(`OpenCode started on port ${ctx.port}${health.version ? ` (v${health.version})` : ""}`);
|
|
4903
|
-
return {
|
|
6230
|
+
return {
|
|
6231
|
+
port: ctx.port,
|
|
6232
|
+
process: proc,
|
|
6233
|
+
version: health.version ?? null,
|
|
6234
|
+
notReadyReason: null
|
|
6235
|
+
};
|
|
4904
6236
|
}
|
|
4905
6237
|
let port = ctx.port;
|
|
4906
6238
|
if (isPortInUse(port)) {
|
|
@@ -4953,122 +6285,15 @@ Port ${port} is already in use.`));
|
|
|
4953
6285
|
if (action === "start") {
|
|
4954
6286
|
const spinner = ora2("Starting OpenCode...").start();
|
|
4955
6287
|
const proc = await startOpenCode(port);
|
|
4956
|
-
const health = await waitForOpenCodeHealth(port,
|
|
6288
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
4957
6289
|
if (!health.healthy) {
|
|
4958
6290
|
spinner.fail("Failed to start OpenCode");
|
|
4959
6291
|
throw new Error("OpenCode failed to start");
|
|
4960
6292
|
}
|
|
4961
6293
|
spinner.stop();
|
|
4962
|
-
return { port, process: proc, version: health.version ?? null };
|
|
4963
|
-
}
|
|
4964
|
-
return { port, process: null, version: null };
|
|
4965
|
-
}
|
|
4966
|
-
|
|
4967
|
-
// src/commands/agent-lookup.ts
|
|
4968
|
-
async function readErrorMessage(response) {
|
|
4969
|
-
const text = await response.text().catch(() => "");
|
|
4970
|
-
if (!text) return response.statusText || void 0;
|
|
4971
|
-
try {
|
|
4972
|
-
const data = JSON.parse(text);
|
|
4973
|
-
const message = data.message ?? data.error;
|
|
4974
|
-
if (typeof message === "string" && message.trim()) {
|
|
4975
|
-
return message;
|
|
4976
|
-
}
|
|
4977
|
-
} catch {
|
|
4978
|
-
}
|
|
4979
|
-
return text.trim() || response.statusText || void 0;
|
|
4980
|
-
}
|
|
4981
|
-
function authFailureHint(apiUrl, serverMessage) {
|
|
4982
|
-
const reason = serverMessage ? `: ${serverMessage}` : "";
|
|
4983
|
-
return `Authentication failed${reason}. Your credentials were rejected by ${apiUrl}. This usually means you logged in against a different environment, or your session expired \u2014 log in again pointing at this endpoint and retry.`;
|
|
4984
|
-
}
|
|
4985
|
-
async function resolveAgentIdFromKey(authHeader) {
|
|
4986
|
-
const apiUrl = getApiUrlConfig();
|
|
4987
|
-
try {
|
|
4988
|
-
const response = await fetch(`${apiUrl}/me`, {
|
|
4989
|
-
headers: { Authorization: authHeader }
|
|
4990
|
-
});
|
|
4991
|
-
if (response.status === 401) {
|
|
4992
|
-
const serverMessage = await readErrorMessage(response);
|
|
4993
|
-
return { error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
4994
|
-
}
|
|
4995
|
-
if (!response.ok) {
|
|
4996
|
-
const serverMessage = await readErrorMessage(response);
|
|
4997
|
-
return {
|
|
4998
|
-
error: `Failed to resolve runner from key (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
4999
|
-
};
|
|
5000
|
-
}
|
|
5001
|
-
const data = await response.json();
|
|
5002
|
-
if (data.auth_type === "agent_key" && data.agent_id) {
|
|
5003
|
-
return { agent_id: data.agent_id };
|
|
5004
|
-
}
|
|
5005
|
-
return {
|
|
5006
|
-
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --agent explicitly."
|
|
5007
|
-
};
|
|
5008
|
-
} catch (error2) {
|
|
5009
|
-
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
5010
|
-
return { error: `Failed to resolve runner from key: ${message}` };
|
|
5011
|
-
}
|
|
5012
|
-
}
|
|
5013
|
-
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
5014
|
-
const apiUrl = getApiUrlConfig();
|
|
5015
|
-
try {
|
|
5016
|
-
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
5017
|
-
method: "POST",
|
|
5018
|
-
headers: { Authorization: authHeader }
|
|
5019
|
-
});
|
|
5020
|
-
if (!response.ok) {
|
|
5021
|
-
const serverMessage = await readErrorMessage(response);
|
|
5022
|
-
return {
|
|
5023
|
-
ok: false,
|
|
5024
|
-
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5025
|
-
};
|
|
5026
|
-
}
|
|
5027
|
-
return { ok: true };
|
|
5028
|
-
} catch (error2) {
|
|
5029
|
-
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
5030
|
-
}
|
|
5031
|
-
}
|
|
5032
|
-
async function getAgentInfo(agentId, authHeader) {
|
|
5033
|
-
const apiUrl = getApiUrlConfig();
|
|
5034
|
-
try {
|
|
5035
|
-
const response = await fetch(`${apiUrl}/runners/${agentId}`, {
|
|
5036
|
-
headers: { Authorization: authHeader }
|
|
5037
|
-
});
|
|
5038
|
-
if (response.status === 401) {
|
|
5039
|
-
const serverMessage = await readErrorMessage(response);
|
|
5040
|
-
return { valid: false, error: authFailureHint(apiUrl, serverMessage), authFailed: true };
|
|
5041
|
-
}
|
|
5042
|
-
if (response.status === 403) {
|
|
5043
|
-
const serverMessage = await readErrorMessage(response);
|
|
5044
|
-
return {
|
|
5045
|
-
valid: false,
|
|
5046
|
-
error: serverMessage ?? "You do not have access to this runner (it may belong to a different team or organization)."
|
|
5047
|
-
};
|
|
5048
|
-
}
|
|
5049
|
-
if (response.status === 404) {
|
|
5050
|
-
const serverMessage = await readErrorMessage(response);
|
|
5051
|
-
return { valid: false, error: serverMessage ?? `Runner ${agentId} not found` };
|
|
5052
|
-
}
|
|
5053
|
-
if (!response.ok) {
|
|
5054
|
-
const serverMessage = await readErrorMessage(response);
|
|
5055
|
-
return {
|
|
5056
|
-
valid: false,
|
|
5057
|
-
error: `API error (HTTP ${response.status})${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5058
|
-
};
|
|
5059
|
-
}
|
|
5060
|
-
const agent = await response.json();
|
|
5061
|
-
if (agent.agent_type !== "local") {
|
|
5062
|
-
return {
|
|
5063
|
-
valid: false,
|
|
5064
|
-
error: `Runner is type '${agent.agent_type}', must be 'local' for CLI connection`
|
|
5065
|
-
};
|
|
5066
|
-
}
|
|
5067
|
-
return { valid: true, agent };
|
|
5068
|
-
} catch (error2) {
|
|
5069
|
-
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
5070
|
-
return { valid: false, error: `Failed to validate runner: ${message}` };
|
|
6294
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
5071
6295
|
}
|
|
6296
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
5072
6297
|
}
|
|
5073
6298
|
|
|
5074
6299
|
// src/commands/run.ts
|
|
@@ -5076,6 +6301,7 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
5076
6301
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
5077
6302
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
5078
6303
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
6304
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
5079
6305
|
function resolveLogLevel(options) {
|
|
5080
6306
|
const accepted = Object.keys(LOG_LEVELS);
|
|
5081
6307
|
const validate = (value, source) => {
|
|
@@ -5127,6 +6353,35 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
5127
6353
|
}
|
|
5128
6354
|
return directories;
|
|
5129
6355
|
}
|
|
6356
|
+
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
6357
|
+
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
6358
|
+
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
6359
|
+
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
6360
|
+
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
6361
|
+
let raw;
|
|
6362
|
+
let source;
|
|
6363
|
+
if (options.opencodeStartTimeout !== void 0) {
|
|
6364
|
+
raw = options.opencodeStartTimeout;
|
|
6365
|
+
source = "--opencode-start-timeout";
|
|
6366
|
+
} else if (env[OPENCODE_START_TIMEOUT_ENV] !== void 0 && env[OPENCODE_START_TIMEOUT_ENV] !== "") {
|
|
6367
|
+
raw = env[OPENCODE_START_TIMEOUT_ENV];
|
|
6368
|
+
source = OPENCODE_START_TIMEOUT_ENV;
|
|
6369
|
+
} else {
|
|
6370
|
+
return { timeoutMs: defaultMs, warnings: [] };
|
|
6371
|
+
}
|
|
6372
|
+
const trimmed = raw.trim();
|
|
6373
|
+
const seconds = Number(trimmed);
|
|
6374
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(seconds) && seconds > 0;
|
|
6375
|
+
if (!isPositiveInteger || seconds > MAX_OPENCODE_START_TIMEOUT_SECONDS) {
|
|
6376
|
+
return {
|
|
6377
|
+
timeoutMs: defaultMs,
|
|
6378
|
+
warnings: [
|
|
6379
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer number of seconds (at most ${MAX_OPENCODE_START_TIMEOUT_SECONDS}); using the default ${DEFAULT_OPENCODE_START_TIMEOUT_SECONDS}s`
|
|
6380
|
+
]
|
|
6381
|
+
};
|
|
6382
|
+
}
|
|
6383
|
+
return { timeoutMs: seconds * 1e3, warnings: [] };
|
|
6384
|
+
}
|
|
5130
6385
|
function meetsThreshold(state, level) {
|
|
5131
6386
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
5132
6387
|
}
|
|
@@ -5148,6 +6403,10 @@ function log2(state, message, level = "info") {
|
|
|
5148
6403
|
function logActivity(state, entry) {
|
|
5149
6404
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
5150
6405
|
if (!meetsThreshold(state, level)) return;
|
|
6406
|
+
forwardRunnerActivity(
|
|
6407
|
+
{ level, message: entry.message, error: entry.error },
|
|
6408
|
+
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
6409
|
+
);
|
|
5151
6410
|
const fullEntry = {
|
|
5152
6411
|
...entry,
|
|
5153
6412
|
level,
|
|
@@ -5247,9 +6506,15 @@ async function handleAuthError(state, error2) {
|
|
|
5247
6506
|
}
|
|
5248
6507
|
async function driveChannels(state, driver) {
|
|
5249
6508
|
let idlePolls = 0;
|
|
6509
|
+
let idleMs = 0;
|
|
6510
|
+
let consecutiveDrainFailures = 0;
|
|
6511
|
+
let unreachableMs = 0;
|
|
5250
6512
|
let lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
5251
6513
|
let lastSeenAppliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
5252
6514
|
while (state.running) {
|
|
6515
|
+
const cycleStartedAtMs = performance.now();
|
|
6516
|
+
let idleThisCycle = false;
|
|
6517
|
+
let unreachableThisCycle = false;
|
|
5253
6518
|
if (state.connection?.reconnecting && state.connection.reconnectPromise) {
|
|
5254
6519
|
logActivity(state, { type: "info", message: "Waiting for tunnel reconnection..." });
|
|
5255
6520
|
if (state.interactive) displayStatus(state);
|
|
@@ -5264,17 +6529,23 @@ async function driveChannels(state, driver) {
|
|
|
5264
6529
|
);
|
|
5265
6530
|
try {
|
|
5266
6531
|
const processed = await driver.drainPending();
|
|
6532
|
+
consecutiveDrainFailures = 0;
|
|
6533
|
+
unreachableMs = 0;
|
|
5267
6534
|
state.messageCount += processed;
|
|
5268
6535
|
const proxiedActivity = state.lastProxiedActivityAt !== lastSeenProxiedActivityAt;
|
|
5269
6536
|
lastSeenProxiedActivityAt = state.lastProxiedActivityAt;
|
|
5270
6537
|
const appliedFiles = driver.fileSyncActivity().appliedFiles;
|
|
5271
|
-
const
|
|
6538
|
+
const filesApplied = appliedFiles !== lastSeenAppliedFiles;
|
|
6539
|
+
const fileActivity = carriedOverFileSync || filesApplied;
|
|
5272
6540
|
lastSeenAppliedFiles = appliedFiles;
|
|
6541
|
+
if (filesApplied) state.claudeUsageRearm?.();
|
|
5273
6542
|
if (processed > 0 || driver.hasInFlightWatchers() || proxiedActivity || fileActivity) {
|
|
5274
6543
|
idlePolls = 0;
|
|
6544
|
+
idleMs = 0;
|
|
5275
6545
|
if (processed > 0 && state.interactive) displayStatus(state);
|
|
5276
6546
|
} else if (state.idleTimeout !== null) {
|
|
5277
6547
|
idlePolls++;
|
|
6548
|
+
idleThisCycle = true;
|
|
5278
6549
|
if (idlePolls === 1) {
|
|
5279
6550
|
logActivity(state, {
|
|
5280
6551
|
type: "info",
|
|
@@ -5298,21 +6569,44 @@ async function driveChannels(state, driver) {
|
|
|
5298
6569
|
const errorMessage = error2 instanceof Error ? error2.message : String(error2);
|
|
5299
6570
|
logActivity(state, { type: "error", error: `Channel processing error: ${errorMessage}` });
|
|
5300
6571
|
if (state.interactive) displayStatus(state);
|
|
6572
|
+
if (driver.hasInFlightWatchers()) {
|
|
6573
|
+
consecutiveDrainFailures = 0;
|
|
6574
|
+
unreachableMs = 0;
|
|
6575
|
+
} else if (state.idleTimeout !== null) {
|
|
6576
|
+
consecutiveDrainFailures++;
|
|
6577
|
+
unreachableThisCycle = true;
|
|
6578
|
+
if (consecutiveDrainFailures === 1) {
|
|
6579
|
+
logActivity(state, {
|
|
6580
|
+
type: "info",
|
|
6581
|
+
message: `Cannot reach Evident, will exit if this persists past the idle timeout (timeout: ${state.idleTimeout}s)...`
|
|
6582
|
+
});
|
|
6583
|
+
if (state.interactive) displayStatus(state);
|
|
6584
|
+
}
|
|
6585
|
+
}
|
|
5301
6586
|
}
|
|
5302
6587
|
await new Promise((resolve3) => setTimeout(resolve3, CHANNEL_POLL_INTERVAL_MS));
|
|
5303
|
-
|
|
5304
|
-
|
|
5305
|
-
|
|
5306
|
-
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
|
|
6588
|
+
const cycleMs = performance.now() - cycleStartedAtMs;
|
|
6589
|
+
if (idleThisCycle) idleMs += cycleMs;
|
|
6590
|
+
if (unreachableThisCycle) unreachableMs += cycleMs;
|
|
6591
|
+
if (state.idleTimeout !== null && consecutiveDrainFailures >= 2 && unreachableMs > state.idleTimeout * 1e3) {
|
|
6592
|
+
logActivity(state, {
|
|
6593
|
+
type: "info",
|
|
6594
|
+
level: "warn",
|
|
6595
|
+
message: `Exiting: could not reach Evident for ${consecutiveDrainFailures} consecutive polls (${Math.round(unreachableMs / 1e3)}s)`
|
|
6596
|
+
});
|
|
6597
|
+
if (state.interactive) displayStatus(state);
|
|
6598
|
+
break;
|
|
6599
|
+
}
|
|
6600
|
+
if (state.idleTimeout !== null && idlePolls >= 2 && idleMs > state.idleTimeout * 1e3) {
|
|
6601
|
+
logActivity(state, { type: "info", message: "Idle timeout reached" });
|
|
6602
|
+
if (state.interactive) displayStatus(state);
|
|
6603
|
+
break;
|
|
5310
6604
|
}
|
|
5311
6605
|
}
|
|
5312
6606
|
}
|
|
5313
6607
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
5314
|
-
async function runSweep(state, driver,
|
|
5315
|
-
const mode = `age=${
|
|
6608
|
+
async function runSweep(state, driver, config) {
|
|
6609
|
+
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
5316
6610
|
try {
|
|
5317
6611
|
const sessions = await listSessions(state.port);
|
|
5318
6612
|
if (sessions === null) {
|
|
@@ -5325,8 +6619,8 @@ async function runSweep(state, driver, config2) {
|
|
|
5325
6619
|
const toDelete = selectSessionsToDelete(
|
|
5326
6620
|
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
5327
6621
|
{
|
|
5328
|
-
maxAgeMs:
|
|
5329
|
-
maxCount:
|
|
6622
|
+
maxAgeMs: config.maxAgeMs,
|
|
6623
|
+
maxCount: config.maxCount,
|
|
5330
6624
|
nowMs: Date.now(),
|
|
5331
6625
|
protectedIds: driver.protectedSessionIds()
|
|
5332
6626
|
}
|
|
@@ -5362,7 +6656,7 @@ async function runSweep(state, driver, config2) {
|
|
|
5362
6656
|
}
|
|
5363
6657
|
}
|
|
5364
6658
|
function scheduleSessionCleanup(state, driver, options) {
|
|
5365
|
-
const
|
|
6659
|
+
const config = resolveSessionCleanupConfig(
|
|
5366
6660
|
{
|
|
5367
6661
|
maxAge: options.sessionCleanupMaxAge,
|
|
5368
6662
|
maxCount: options.sessionCleanupMaxCount,
|
|
@@ -5370,21 +6664,129 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
5370
6664
|
},
|
|
5371
6665
|
process.env
|
|
5372
6666
|
);
|
|
5373
|
-
for (const warning2 of
|
|
6667
|
+
for (const warning2 of config.warnings) {
|
|
5374
6668
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
5375
6669
|
}
|
|
5376
|
-
if (!
|
|
6670
|
+
if (!config.enabled) return;
|
|
5377
6671
|
logActivity(state, {
|
|
5378
6672
|
type: "info",
|
|
5379
|
-
message: `Session cleanup enabled (age=${
|
|
6673
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`
|
|
5380
6674
|
});
|
|
5381
|
-
const interval = setInterval(() => void runSweep(state, driver,
|
|
6675
|
+
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
5382
6676
|
const firstSweep = setTimeout(
|
|
5383
|
-
() => void runSweep(state, driver,
|
|
6677
|
+
() => void runSweep(state, driver, config),
|
|
5384
6678
|
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
5385
6679
|
);
|
|
5386
6680
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
5387
6681
|
}
|
|
6682
|
+
function claudeUsageFailureStreakSuffix(consecutiveFailures) {
|
|
6683
|
+
return consecutiveFailures > 1 ? ` (${consecutiveFailures} consecutive failures)` : "";
|
|
6684
|
+
}
|
|
6685
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
6686
|
+
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
6687
|
+
options.claudeUsageReporting,
|
|
6688
|
+
process.env
|
|
6689
|
+
);
|
|
6690
|
+
for (const warning2 of warnings) {
|
|
6691
|
+
logActivity(state, {
|
|
6692
|
+
type: "info",
|
|
6693
|
+
level: "warn",
|
|
6694
|
+
message: `Claude usage reporting: ${warning2}`
|
|
6695
|
+
});
|
|
6696
|
+
}
|
|
6697
|
+
if (mode === "off") {
|
|
6698
|
+
logActivity(state, {
|
|
6699
|
+
type: "info",
|
|
6700
|
+
level: "debug",
|
|
6701
|
+
message: "Claude usage reporting is off (--claude-usage-reporting off)"
|
|
6702
|
+
});
|
|
6703
|
+
return null;
|
|
6704
|
+
}
|
|
6705
|
+
let consecutiveFailures = 0;
|
|
6706
|
+
let armed = false;
|
|
6707
|
+
let rearmRequested = false;
|
|
6708
|
+
const scheduleNextTick = () => {
|
|
6709
|
+
armed = true;
|
|
6710
|
+
rearmRequested = false;
|
|
6711
|
+
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
6712
|
+
};
|
|
6713
|
+
const rearm = () => {
|
|
6714
|
+
if (armed) {
|
|
6715
|
+
rearmRequested = true;
|
|
6716
|
+
return;
|
|
6717
|
+
}
|
|
6718
|
+
rearmRequested = false;
|
|
6719
|
+
armed = true;
|
|
6720
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
6721
|
+
};
|
|
6722
|
+
const tick = async (isProbe) => {
|
|
6723
|
+
try {
|
|
6724
|
+
const usage = await getClaudeUsage();
|
|
6725
|
+
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
6726
|
+
if (result.ok) {
|
|
6727
|
+
if (consecutiveFailures > 0) {
|
|
6728
|
+
logActivity(state, {
|
|
6729
|
+
type: "info",
|
|
6730
|
+
level: "info",
|
|
6731
|
+
message: "Claude usage reporting recovered"
|
|
6732
|
+
});
|
|
6733
|
+
}
|
|
6734
|
+
consecutiveFailures = 0;
|
|
6735
|
+
logActivity(state, {
|
|
6736
|
+
type: "info",
|
|
6737
|
+
level: "debug",
|
|
6738
|
+
message: "Reported Claude usage to Evident"
|
|
6739
|
+
});
|
|
6740
|
+
} else {
|
|
6741
|
+
consecutiveFailures++;
|
|
6742
|
+
logActivity(state, {
|
|
6743
|
+
type: "info",
|
|
6744
|
+
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
6745
|
+
message: `Failed to report Claude usage: ${result.error}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
|
|
6746
|
+
});
|
|
6747
|
+
}
|
|
6748
|
+
scheduleNextTick();
|
|
6749
|
+
} catch (error2) {
|
|
6750
|
+
if (error2 instanceof ClaudeUsageError && isLocalCredentialProblem(error2)) {
|
|
6751
|
+
if (mode === "on") {
|
|
6752
|
+
logActivity(state, {
|
|
6753
|
+
type: "info",
|
|
6754
|
+
level: "warn",
|
|
6755
|
+
message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
|
|
6756
|
+
});
|
|
6757
|
+
scheduleNextTick();
|
|
6758
|
+
} else if (isProbe) {
|
|
6759
|
+
logActivity(state, {
|
|
6760
|
+
type: "info",
|
|
6761
|
+
level: "debug",
|
|
6762
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
6763
|
+
});
|
|
6764
|
+
armed = false;
|
|
6765
|
+
if (rearmRequested) rearm();
|
|
6766
|
+
} else {
|
|
6767
|
+
logActivity(state, {
|
|
6768
|
+
type: "info",
|
|
6769
|
+
level: "debug",
|
|
6770
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
6771
|
+
});
|
|
6772
|
+
scheduleNextTick();
|
|
6773
|
+
}
|
|
6774
|
+
} else {
|
|
6775
|
+
consecutiveFailures++;
|
|
6776
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
6777
|
+
logActivity(state, {
|
|
6778
|
+
type: "info",
|
|
6779
|
+
level: claudeUsageFailureLogLevel(consecutiveFailures),
|
|
6780
|
+
message: `Claude usage reporting failed: ${message}${claudeUsageFailureStreakSuffix(consecutiveFailures)}`
|
|
6781
|
+
});
|
|
6782
|
+
scheduleNextTick();
|
|
6783
|
+
}
|
|
6784
|
+
}
|
|
6785
|
+
};
|
|
6786
|
+
armed = true;
|
|
6787
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
6788
|
+
return rearm;
|
|
6789
|
+
}
|
|
5388
6790
|
async function notifyOffline(state) {
|
|
5389
6791
|
if (!state.agentId || !state.authHeader) return;
|
|
5390
6792
|
if (!state.connected) {
|
|
@@ -5402,13 +6804,29 @@ async function notifyOffline(state) {
|
|
|
5402
6804
|
if (state.interactive) displayStatus(state);
|
|
5403
6805
|
}
|
|
5404
6806
|
}
|
|
6807
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
6808
|
+
const startedAt = Date.now();
|
|
6809
|
+
try {
|
|
6810
|
+
return await run2();
|
|
6811
|
+
} finally {
|
|
6812
|
+
const elapsedMs = Date.now() - startedAt;
|
|
6813
|
+
durations[name] = elapsedMs;
|
|
6814
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
6815
|
+
}
|
|
6816
|
+
}
|
|
5405
6817
|
async function cleanup(state, opts = {}) {
|
|
6818
|
+
const durations = {};
|
|
5406
6819
|
state.running = false;
|
|
5407
6820
|
for (const timer of state.sessionCleanupTimers) {
|
|
5408
6821
|
clearInterval(timer);
|
|
5409
6822
|
clearTimeout(timer);
|
|
5410
6823
|
}
|
|
5411
6824
|
state.sessionCleanupTimers = [];
|
|
6825
|
+
if (state.claudeUsageTimer) {
|
|
6826
|
+
clearTimeout(state.claudeUsageTimer);
|
|
6827
|
+
state.claudeUsageTimer = null;
|
|
6828
|
+
}
|
|
6829
|
+
state.claudeUsageRearm = null;
|
|
5412
6830
|
if (opts.graceful && state.channelDriver) {
|
|
5413
6831
|
state.channelDriver.stop();
|
|
5414
6832
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -5416,7 +6834,13 @@ async function cleanup(state, opts = {}) {
|
|
|
5416
6834
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
5417
6835
|
displayStatus(state);
|
|
5418
6836
|
}
|
|
5419
|
-
const
|
|
6837
|
+
const driver = state.channelDriver;
|
|
6838
|
+
const settled = await timeShutdownPhase(
|
|
6839
|
+
state,
|
|
6840
|
+
durations,
|
|
6841
|
+
"drain",
|
|
6842
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
6843
|
+
);
|
|
5420
6844
|
if (!settled) {
|
|
5421
6845
|
logActivity(state, {
|
|
5422
6846
|
type: "info",
|
|
@@ -5425,13 +6849,15 @@ async function cleanup(state, opts = {}) {
|
|
|
5425
6849
|
if (state.interactive) displayStatus(state);
|
|
5426
6850
|
}
|
|
5427
6851
|
}
|
|
5428
|
-
await notifyOffline(state);
|
|
6852
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
5429
6853
|
if (state.connection) {
|
|
5430
|
-
state.connection
|
|
6854
|
+
const connection = state.connection;
|
|
6855
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
5431
6856
|
state.connection = null;
|
|
5432
6857
|
}
|
|
5433
6858
|
if (state.opencodeProcess) {
|
|
5434
|
-
|
|
6859
|
+
const opencodeProcess = state.opencodeProcess;
|
|
6860
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
5435
6861
|
if (state.interactive) {
|
|
5436
6862
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
5437
6863
|
displayStatus(state);
|
|
@@ -5440,6 +6866,7 @@ async function cleanup(state, opts = {}) {
|
|
|
5440
6866
|
}
|
|
5441
6867
|
state.opencodeProcess = null;
|
|
5442
6868
|
}
|
|
6869
|
+
return durations;
|
|
5443
6870
|
}
|
|
5444
6871
|
async function run(options) {
|
|
5445
6872
|
const interactive = isInteractive(options.json);
|
|
@@ -5480,8 +6907,11 @@ async function run(options) {
|
|
|
5480
6907
|
messageCount: 0,
|
|
5481
6908
|
lastProxiedActivityAt: null,
|
|
5482
6909
|
sessionCleanupTimers: [],
|
|
6910
|
+
claudeUsageTimer: null,
|
|
6911
|
+
claudeUsageRearm: null,
|
|
5483
6912
|
authHeader: ""
|
|
5484
6913
|
};
|
|
6914
|
+
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
5485
6915
|
if (fileSyncDirectories.length > 0) {
|
|
5486
6916
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
5487
6917
|
} else {
|
|
@@ -5510,14 +6940,38 @@ async function run(options) {
|
|
|
5510
6940
|
const handleSignal = async () => {
|
|
5511
6941
|
if (state.shuttingDown) return;
|
|
5512
6942
|
state.shuttingDown = true;
|
|
6943
|
+
const shutdownStartedAt = Date.now();
|
|
5513
6944
|
if (state.interactive) {
|
|
5514
6945
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
5515
6946
|
displayStatus(state);
|
|
5516
6947
|
} else {
|
|
5517
6948
|
log2(state, "Shutting down...");
|
|
5518
6949
|
}
|
|
5519
|
-
await cleanup(state, { graceful: true });
|
|
5520
|
-
|
|
6950
|
+
const durations = await cleanup(state, { graceful: true });
|
|
6951
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
6952
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
6953
|
+
let timer;
|
|
6954
|
+
const flushed = shutdownTelemetry().then(
|
|
6955
|
+
() => true,
|
|
6956
|
+
(error2) => {
|
|
6957
|
+
log2(
|
|
6958
|
+
state,
|
|
6959
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
6960
|
+
"warn"
|
|
6961
|
+
);
|
|
6962
|
+
return true;
|
|
6963
|
+
}
|
|
6964
|
+
);
|
|
6965
|
+
const timedOut = new Promise((resolve3) => {
|
|
6966
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
6967
|
+
});
|
|
6968
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
6969
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
6970
|
+
}
|
|
6971
|
+
clearTimeout(timer);
|
|
6972
|
+
});
|
|
6973
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
6974
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
5521
6975
|
process.exit(0);
|
|
5522
6976
|
};
|
|
5523
6977
|
process.on("SIGINT", handleSignal);
|
|
@@ -5534,6 +6988,7 @@ async function run(options) {
|
|
|
5534
6988
|
console.log(chalk6.dim("Or run `evident login` for interactive authentication"));
|
|
5535
6989
|
blank();
|
|
5536
6990
|
process.exit(1);
|
|
6991
|
+
return;
|
|
5537
6992
|
}
|
|
5538
6993
|
blank();
|
|
5539
6994
|
console.log(chalk6.yellow("You are not logged in to Evident."));
|
|
@@ -5578,6 +7033,7 @@ async function run(options) {
|
|
|
5578
7033
|
} else {
|
|
5579
7034
|
printError(resolved.error || "Failed to resolve runner ID from key");
|
|
5580
7035
|
process.exit(1);
|
|
7036
|
+
return;
|
|
5581
7037
|
}
|
|
5582
7038
|
} else {
|
|
5583
7039
|
printError(
|
|
@@ -5591,6 +7047,7 @@ async function run(options) {
|
|
|
5591
7047
|
);
|
|
5592
7048
|
blank();
|
|
5593
7049
|
process.exit(1);
|
|
7050
|
+
return;
|
|
5594
7051
|
}
|
|
5595
7052
|
}
|
|
5596
7053
|
telemetry.info(
|
|
@@ -5631,40 +7088,67 @@ async function run(options) {
|
|
|
5631
7088
|
}
|
|
5632
7089
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
5633
7090
|
state.agentName = validation.agent.name;
|
|
7091
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
7092
|
+
if (microvmId) {
|
|
7093
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
7094
|
+
if (reported.ok) {
|
|
7095
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
7096
|
+
} else {
|
|
7097
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
7098
|
+
log2(state, message, "warn");
|
|
7099
|
+
if (state.interactive && !state.json) {
|
|
7100
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
7101
|
+
}
|
|
7102
|
+
}
|
|
7103
|
+
} else {
|
|
7104
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
7105
|
+
}
|
|
7106
|
+
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
7107
|
+
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
7108
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
7109
|
+
}
|
|
5634
7110
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
5635
7111
|
try {
|
|
5636
7112
|
const oc = await ensureOpenCodeRunning({
|
|
5637
7113
|
port: state.port,
|
|
5638
7114
|
interactive: state.interactive,
|
|
5639
7115
|
agentId: state.agentId,
|
|
5640
|
-
log: (message) => log2(state, message)
|
|
7116
|
+
log: (message) => log2(state, message),
|
|
7117
|
+
startTimeoutMs: opencodeStartTimeoutMs
|
|
5641
7118
|
});
|
|
5642
7119
|
state.port = oc.port;
|
|
5643
7120
|
state.opencodeProcess = oc.process;
|
|
5644
7121
|
state.opencodeVersion = oc.version;
|
|
5645
|
-
state.opencodeConnected = oc.
|
|
7122
|
+
state.opencodeConnected = oc.notReadyReason === null;
|
|
5646
7123
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
5647
7124
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
7125
|
+
if (!state.interactive && oc.notReadyReason !== null) {
|
|
7126
|
+
const message = `OpenCode is not ready on port ${state.port}: ${oc.notReadyReason}. The runner will still come online, but messages will fail until opencode answers \u2014 raise the wait with --opencode-start-timeout <seconds> (env ${OPENCODE_START_TIMEOUT_ENV}).`;
|
|
7127
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
7128
|
+
} else {
|
|
7129
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
7130
|
+
if (versionWarning) {
|
|
7131
|
+
log2(state, versionWarning, "warn");
|
|
7132
|
+
if (state.interactive && !state.json) {
|
|
7133
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
7134
|
+
}
|
|
5653
7135
|
}
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5662
|
-
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
7136
|
+
const noProviderWarning = buildNoProviderWarning(
|
|
7137
|
+
await hasAnyConfiguredProvider(state.port)
|
|
7138
|
+
);
|
|
7139
|
+
if (noProviderWarning) {
|
|
7140
|
+
log2(state, noProviderWarning, "warn");
|
|
7141
|
+
if (state.interactive && !state.json) {
|
|
7142
|
+
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
7143
|
+
blank();
|
|
7144
|
+
console.log(chalk6.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
7145
|
+
console.log(
|
|
7146
|
+
chalk6.dim(
|
|
7147
|
+
`Run ${chalk6.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
7148
|
+
)
|
|
7149
|
+
);
|
|
7150
|
+
blank();
|
|
7151
|
+
}
|
|
5668
7152
|
}
|
|
5669
7153
|
}
|
|
5670
7154
|
} catch (error2) {
|
|
@@ -5709,6 +7193,18 @@ async function run(options) {
|
|
|
5709
7193
|
type: "info",
|
|
5710
7194
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
5711
7195
|
});
|
|
7196
|
+
if (options.tunnelReadyFile) {
|
|
7197
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
7198
|
+
if (marker.ok) {
|
|
7199
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
7200
|
+
} else {
|
|
7201
|
+
log2(
|
|
7202
|
+
state,
|
|
7203
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
7204
|
+
"error"
|
|
7205
|
+
);
|
|
7206
|
+
}
|
|
7207
|
+
}
|
|
5712
7208
|
emitAgentConnected(state.agentId, {
|
|
5713
7209
|
port: state.port,
|
|
5714
7210
|
cli_version: getCliVersion(),
|
|
@@ -5799,6 +7295,7 @@ async function run(options) {
|
|
|
5799
7295
|
throw error2;
|
|
5800
7296
|
}
|
|
5801
7297
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
7298
|
+
state.claudeUsageRearm = scheduleClaudeUsageReporting(state, options);
|
|
5802
7299
|
if (!interactive || state.json) {
|
|
5803
7300
|
log2(state, "Driving channel messages...");
|
|
5804
7301
|
}
|
|
@@ -5853,13 +7350,18 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
5853
7350
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
5854
7351
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
5855
7352
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
7353
|
+
program.command("status").description("Check whether the configured credentials can reach Evident").option("--json", "Output in JSON format").action((options) => status({ json: options.json }));
|
|
7354
|
+
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
5856
7355
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
5857
7356
|
"-a, --agent [id]",
|
|
5858
7357
|
"Deprecated alias for --runner (still supported; --runner wins if both are given)"
|
|
5859
7358
|
).option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
5860
7359
|
"--log-level <level>",
|
|
5861
7360
|
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
5862
|
-
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
7361
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
7362
|
+
"--opencode-start-timeout <seconds>",
|
|
7363
|
+
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
7364
|
+
).option("--json", "Output in JSON format").option(
|
|
5863
7365
|
"--session-cleanup-max-age <duration>",
|
|
5864
7366
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
5865
7367
|
).option(
|
|
@@ -5868,11 +7370,17 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5868
7370
|
).option(
|
|
5869
7371
|
"--session-cleanup-interval <duration>",
|
|
5870
7372
|
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
7373
|
+
).option(
|
|
7374
|
+
"--claude-usage-reporting <mode>",
|
|
7375
|
+
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
5871
7376
|
).option(
|
|
5872
7377
|
"--enable-file-sync-to <dir>",
|
|
5873
7378
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
5874
7379
|
(value, previous) => previous.concat([value]),
|
|
5875
7380
|
[]
|
|
7381
|
+
).option(
|
|
7382
|
+
"--tunnel-ready-file <path>",
|
|
7383
|
+
"Path to write once the tunnel is connected (opt-in; unused unless an operator/image sets it)"
|
|
5876
7384
|
).action(
|
|
5877
7385
|
(options) => {
|
|
5878
7386
|
run({
|
|
@@ -5885,14 +7393,21 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5885
7393
|
verbose: options.verbose,
|
|
5886
7394
|
conversation: options.conversation,
|
|
5887
7395
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
7396
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
7397
|
+
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
7398
|
+
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
5888
7399
|
json: options.json,
|
|
5889
7400
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
5890
7401
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
5891
7402
|
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
5892
7403
|
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
7404
|
+
// Raw string — the resolver in run.ts single-sources parsing
|
|
7405
|
+
// (resolveClaudeUsageReportingMode).
|
|
7406
|
+
claudeUsageReporting: options.claudeUsageReporting,
|
|
5893
7407
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
5894
7408
|
// resolveFileSyncDirectories.
|
|
5895
|
-
enableFileSyncTo: options.enableFileSyncTo
|
|
7409
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
7410
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
5896
7411
|
});
|
|
5897
7412
|
}
|
|
5898
7413
|
);
|