@evident-ai/cli 3.1.1-dev.3ec6e7f → 3.1.1-dev.4bc8b32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -0
- package/dist/index.js +1090 -159
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ import chalk2 from "chalk";
|
|
|
11
11
|
|
|
12
12
|
// src/lib/config.ts
|
|
13
13
|
import Conf from "conf";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
14
|
+
import { chmodSync, existsSync, statSync } from "fs";
|
|
15
|
+
import { dirname } from "path";
|
|
16
16
|
var PRODUCTION_API_URL = "https://api.production.evident.run/v1";
|
|
17
17
|
var PRODUCTION_TUNNEL_URL = "wss://tunnel.production.evident.run";
|
|
18
18
|
var defaults = {
|
|
@@ -38,17 +38,39 @@ function getApiUrl() {
|
|
|
38
38
|
function getTunnelUrl() {
|
|
39
39
|
return tunnelOverride ?? process.env.EVIDENT_TUNNEL_URL ?? defaults.tunnelUrl;
|
|
40
40
|
}
|
|
41
|
-
var config = new Conf({
|
|
42
|
-
projectName: "evident",
|
|
43
|
-
projectSuffix: "",
|
|
44
|
-
defaults
|
|
45
|
-
});
|
|
46
41
|
var credentials = new Conf({
|
|
47
42
|
projectName: "evident",
|
|
48
43
|
projectSuffix: "",
|
|
49
44
|
configName: "credentials",
|
|
50
|
-
defaults: {}
|
|
45
|
+
defaults: {},
|
|
46
|
+
configFileMode: 384
|
|
51
47
|
});
|
|
48
|
+
var CREDENTIALS_FILE_MODE = 384;
|
|
49
|
+
var CREDENTIALS_DIR_MODE = 448;
|
|
50
|
+
var permissionWarningEmitted = false;
|
|
51
|
+
function hardenCredentialsPermissions() {
|
|
52
|
+
if (process.platform === "win32") {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const file = credentials.path;
|
|
56
|
+
for (const [path, mode] of [
|
|
57
|
+
[file, CREDENTIALS_FILE_MODE],
|
|
58
|
+
[dirname(file), CREDENTIALS_DIR_MODE]
|
|
59
|
+
]) {
|
|
60
|
+
try {
|
|
61
|
+
if (existsSync(path) && (statSync(path).mode & 511) !== mode) {
|
|
62
|
+
chmodSync(path, mode);
|
|
63
|
+
}
|
|
64
|
+
} catch (err) {
|
|
65
|
+
if (!permissionWarningEmitted) {
|
|
66
|
+
permissionWarningEmitted = true;
|
|
67
|
+
console.error(
|
|
68
|
+
`[config] could not restrict permissions on ${path}; the credentials file may be readable by other users on this machine: ${err instanceof Error ? err.message : String(err)}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
52
74
|
function getApiUrlConfig() {
|
|
53
75
|
return getApiUrl();
|
|
54
76
|
}
|
|
@@ -59,6 +81,7 @@ function credentialsKey() {
|
|
|
59
81
|
return getApiUrl();
|
|
60
82
|
}
|
|
61
83
|
function getCredentials() {
|
|
84
|
+
hardenCredentialsPermissions();
|
|
62
85
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
63
86
|
return byEndpoint[credentialsKey()] ?? {};
|
|
64
87
|
}
|
|
@@ -70,14 +93,17 @@ function setCredentials(creds) {
|
|
|
70
93
|
expiresAt: creds.expiresAt
|
|
71
94
|
};
|
|
72
95
|
credentials.set("byEndpoint", byEndpoint);
|
|
96
|
+
hardenCredentialsPermissions();
|
|
73
97
|
}
|
|
74
98
|
function clearCredentials() {
|
|
75
99
|
const byEndpoint = credentials.get("byEndpoint") ?? {};
|
|
76
100
|
delete byEndpoint[credentialsKey()];
|
|
77
101
|
credentials.set("byEndpoint", byEndpoint);
|
|
102
|
+
hardenCredentialsPermissions();
|
|
78
103
|
}
|
|
79
104
|
function clearAllCredentials() {
|
|
80
105
|
credentials.clear();
|
|
106
|
+
hardenCredentialsPermissions();
|
|
81
107
|
}
|
|
82
108
|
function getCliName() {
|
|
83
109
|
const argv1 = process.argv[1] || "";
|
|
@@ -236,16 +262,28 @@ async function getToken() {
|
|
|
236
262
|
}
|
|
237
263
|
return null;
|
|
238
264
|
}
|
|
265
|
+
function toError(err) {
|
|
266
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
267
|
+
}
|
|
239
268
|
async function deleteToken(options = {}) {
|
|
240
269
|
const keytar = await getKeytar();
|
|
270
|
+
const failures = [];
|
|
241
271
|
if (keytar) {
|
|
242
272
|
if (options.all) {
|
|
243
|
-
|
|
273
|
+
let accounts = [];
|
|
274
|
+
try {
|
|
275
|
+
accounts = await keytar.findCredentials(SERVICE_NAME);
|
|
276
|
+
} catch (err) {
|
|
277
|
+
failures.push({ type: "enumerate", error: toError(err) });
|
|
278
|
+
}
|
|
244
279
|
await Promise.all(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
280
|
+
accounts.map(async (entry) => {
|
|
281
|
+
try {
|
|
282
|
+
await keytar.deletePassword(SERVICE_NAME, entry.account);
|
|
283
|
+
} catch (err) {
|
|
284
|
+
failures.push({ type: "delete", account: entry.account, error: toError(err) });
|
|
285
|
+
}
|
|
286
|
+
})
|
|
249
287
|
);
|
|
250
288
|
} else {
|
|
251
289
|
await keytar.deletePassword(SERVICE_NAME, keychainAccount());
|
|
@@ -256,6 +294,7 @@ async function deleteToken(options = {}) {
|
|
|
256
294
|
} else {
|
|
257
295
|
clearCredentials();
|
|
258
296
|
}
|
|
297
|
+
return { failures };
|
|
259
298
|
}
|
|
260
299
|
|
|
261
300
|
// src/utils/ui.ts
|
|
@@ -373,8 +412,10 @@ async function deviceFlowLogin(options) {
|
|
|
373
412
|
}
|
|
374
413
|
async function tokenLogin() {
|
|
375
414
|
console.log("Token login mode.");
|
|
376
|
-
console.log("
|
|
377
|
-
console.log(
|
|
415
|
+
console.log("Create a token under Settings \u2192 CLI tokens in the dashboard, then paste it below.");
|
|
416
|
+
console.log(
|
|
417
|
+
"(Alternatively, run `evident login` on a machine with a browser, or set EVIDENT_TOKEN for CI.)"
|
|
418
|
+
);
|
|
378
419
|
blank();
|
|
379
420
|
process.stdout.write("Paste token: ");
|
|
380
421
|
const token = await new Promise((resolve3) => {
|
|
@@ -398,13 +439,22 @@ async function tokenLogin() {
|
|
|
398
439
|
printError("No token provided.");
|
|
399
440
|
process.exit(1);
|
|
400
441
|
}
|
|
442
|
+
await validateAndStoreToken(token);
|
|
443
|
+
}
|
|
444
|
+
async function validateAndStoreToken(token) {
|
|
401
445
|
const spinner = ora("Validating token...").start();
|
|
402
446
|
try {
|
|
403
|
-
const result = await api.
|
|
447
|
+
const result = await api.get("/me", {
|
|
448
|
+
headers: { Authorization: `Bearer ${token}` }
|
|
449
|
+
});
|
|
450
|
+
if (!result.user) {
|
|
451
|
+
throw new Error(
|
|
452
|
+
"This token is not a user login (e.g. a runner key). Paste a CLI token instead."
|
|
453
|
+
);
|
|
454
|
+
}
|
|
404
455
|
await storeToken({
|
|
405
456
|
token,
|
|
406
|
-
user: result.user
|
|
407
|
-
expiresAt: result.expires_at
|
|
457
|
+
user: { email: result.user.email }
|
|
408
458
|
});
|
|
409
459
|
spinner.stop();
|
|
410
460
|
printSuccess(`Logged in as ${chalk2.bold(result.user.email)}`);
|
|
@@ -424,9 +474,22 @@ async function login(options) {
|
|
|
424
474
|
}
|
|
425
475
|
|
|
426
476
|
// src/commands/logout.ts
|
|
477
|
+
function describeFailure(failure) {
|
|
478
|
+
if (failure.type === "enumerate") {
|
|
479
|
+
return `could not list stored keychain entries (${failure.error.message})`;
|
|
480
|
+
}
|
|
481
|
+
return `${failure.account} (${failure.error.message})`;
|
|
482
|
+
}
|
|
427
483
|
async function logout(options = {}) {
|
|
428
484
|
if (options.all) {
|
|
429
|
-
await deleteToken({ all: true });
|
|
485
|
+
const result = await deleteToken({ all: true });
|
|
486
|
+
if (result.failures.length > 0) {
|
|
487
|
+
printError(
|
|
488
|
+
`Failed to fully clear your keychain: ${result.failures.map(describeFailure).join("; ")}. Your local credentials file was cleared, but stale keychain entries may remain \u2014 run \`evident logout --all\` again, or remove them manually from your OS keychain / credential manager.`
|
|
489
|
+
);
|
|
490
|
+
process.exitCode = 1;
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
430
493
|
printSuccess("Logged out of all endpoints.");
|
|
431
494
|
return;
|
|
432
495
|
}
|
|
@@ -451,7 +514,9 @@ async function whoami() {
|
|
|
451
514
|
blank();
|
|
452
515
|
console.log(keyValue("Endpoint", apiUrl));
|
|
453
516
|
console.log(keyValue("User", chalk3.bold(credentials2.user.email)));
|
|
454
|
-
|
|
517
|
+
if (credentials2.user.id) {
|
|
518
|
+
console.log(keyValue("User ID", credentials2.user.id));
|
|
519
|
+
}
|
|
455
520
|
if (credentials2.expiresAt) {
|
|
456
521
|
const expiresAt = new Date(credentials2.expiresAt);
|
|
457
522
|
const now = /* @__PURE__ */ new Date();
|
|
@@ -467,6 +532,121 @@ async function whoami() {
|
|
|
467
532
|
blank();
|
|
468
533
|
}
|
|
469
534
|
|
|
535
|
+
// src/lib/claude-usage.ts
|
|
536
|
+
import { execFileSync } from "child_process";
|
|
537
|
+
import { readFileSync } from "fs";
|
|
538
|
+
import { homedir } from "os";
|
|
539
|
+
import { join } from "path";
|
|
540
|
+
var CLAUDE_USAGE_URL = "https://api.anthropic.com/api/oauth/usage";
|
|
541
|
+
var KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
542
|
+
function parseClaudeCliCredentials(raw) {
|
|
543
|
+
let parsed;
|
|
544
|
+
try {
|
|
545
|
+
parsed = JSON.parse(raw);
|
|
546
|
+
} catch {
|
|
547
|
+
return null;
|
|
548
|
+
}
|
|
549
|
+
const data = parsed.claudeAiOauth ?? parsed;
|
|
550
|
+
const creds = data;
|
|
551
|
+
if (typeof creds.accessToken !== "string" || typeof creds.expiresAt !== "number") {
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
return { accessToken: creds.accessToken, expiresAt: creds.expiresAt };
|
|
555
|
+
}
|
|
556
|
+
function readClaudeCliCredentials() {
|
|
557
|
+
if (process.platform === "darwin") {
|
|
558
|
+
try {
|
|
559
|
+
const raw = execFileSync(
|
|
560
|
+
"/usr/bin/security",
|
|
561
|
+
["find-generic-password", "-s", KEYCHAIN_SERVICE, "-w"],
|
|
562
|
+
{ encoding: "utf-8", timeout: 2e3, stdio: ["pipe", "pipe", "ignore"] }
|
|
563
|
+
);
|
|
564
|
+
return parseClaudeCliCredentials(raw);
|
|
565
|
+
} catch {
|
|
566
|
+
return null;
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
try {
|
|
570
|
+
const raw = readFileSync(join(homedir(), ".claude", ".credentials.json"), "utf-8");
|
|
571
|
+
return parseClaudeCliCredentials(raw);
|
|
572
|
+
} catch {
|
|
573
|
+
return null;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
var ClaudeUsageError = class extends Error {
|
|
577
|
+
constructor(message, reason) {
|
|
578
|
+
super(message);
|
|
579
|
+
this.reason = reason;
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
function isLocalCredentialProblem(err) {
|
|
583
|
+
return err instanceof ClaudeUsageError && (err.reason === "no_credentials" || err.reason === "credentials_expired");
|
|
584
|
+
}
|
|
585
|
+
function toWindow(value) {
|
|
586
|
+
if (!value || typeof value !== "object") {
|
|
587
|
+
return null;
|
|
588
|
+
}
|
|
589
|
+
const window = value;
|
|
590
|
+
if (typeof window.utilization !== "number" || typeof window.resets_at !== "string") {
|
|
591
|
+
return null;
|
|
592
|
+
}
|
|
593
|
+
return { utilization: window.utilization, resetsAt: window.resets_at };
|
|
594
|
+
}
|
|
595
|
+
async function getClaudeUsage() {
|
|
596
|
+
const credentials2 = readClaudeCliCredentials();
|
|
597
|
+
if (!credentials2) {
|
|
598
|
+
throw new ClaudeUsageError(
|
|
599
|
+
"No local Claude Code login found. Run `claude` once to sign in with your Claude subscription.",
|
|
600
|
+
"no_credentials"
|
|
601
|
+
);
|
|
602
|
+
}
|
|
603
|
+
if (credentials2.expiresAt < Date.now()) {
|
|
604
|
+
throw new ClaudeUsageError(
|
|
605
|
+
"Claude Code credentials have expired. Run `claude` to refresh them.",
|
|
606
|
+
"credentials_expired"
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
const res = await fetch(CLAUDE_USAGE_URL, {
|
|
610
|
+
headers: {
|
|
611
|
+
Authorization: `Bearer ${credentials2.accessToken}`,
|
|
612
|
+
"Content-Type": "application/json",
|
|
613
|
+
"anthropic-version": "2023-06-01"
|
|
614
|
+
}
|
|
615
|
+
});
|
|
616
|
+
if (!res.ok) {
|
|
617
|
+
throw new ClaudeUsageError(`Claude usage request failed: HTTP ${res.status}`, "request_failed");
|
|
618
|
+
}
|
|
619
|
+
const body = await res.json();
|
|
620
|
+
return {
|
|
621
|
+
fiveHour: toWindow(body.five_hour),
|
|
622
|
+
sevenDay: toWindow(body.seven_day)
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// src/commands/claude-usage.ts
|
|
627
|
+
function formatWindow(label, window) {
|
|
628
|
+
if (!window) {
|
|
629
|
+
return keyValue(label, "not available for this plan");
|
|
630
|
+
}
|
|
631
|
+
const resetsAt = new Date(window.resetsAt);
|
|
632
|
+
return keyValue(label, `${window.utilization}% used, resets ${resetsAt.toLocaleString()}`);
|
|
633
|
+
}
|
|
634
|
+
async function claudeUsage() {
|
|
635
|
+
try {
|
|
636
|
+
const usage = await getClaudeUsage();
|
|
637
|
+
blank();
|
|
638
|
+
console.log(formatWindow("5-hour session", usage.fiveHour));
|
|
639
|
+
console.log(formatWindow("7-day", usage.sevenDay));
|
|
640
|
+
blank();
|
|
641
|
+
} catch (err) {
|
|
642
|
+
if (err instanceof ClaudeUsageError) {
|
|
643
|
+
printError(err.message);
|
|
644
|
+
process.exit(1);
|
|
645
|
+
}
|
|
646
|
+
throw err;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
470
650
|
// src/commands/run.ts
|
|
471
651
|
import { homedir as homedir3 } from "os";
|
|
472
652
|
import { isAbsolute as isAbsolute2, join as join3, parse, resolve as resolvePath } from "path";
|
|
@@ -479,7 +659,10 @@ var TelemetryEventTypes = {
|
|
|
479
659
|
AGENT_DISCONNECTED: "agent.disconnected",
|
|
480
660
|
AGENT_MESSAGE_PROCESSING: "agent.message_processing",
|
|
481
661
|
AGENT_MESSAGE_DONE: "agent.message_done",
|
|
482
|
-
AGENT_MESSAGE_FAILED: "agent.message_failed"
|
|
662
|
+
AGENT_MESSAGE_FAILED: "agent.message_failed",
|
|
663
|
+
// A `warn`/`error` runner-side log line forwarded server-side for
|
|
664
|
+
// observability (issue #916) — see `apps/cli/src/lib/runner-activity-telemetry.ts`.
|
|
665
|
+
RUNNER_ACTIVITY: "runner.activity"
|
|
483
666
|
};
|
|
484
667
|
|
|
485
668
|
// ../../packages/types/src/tunnel/index.ts
|
|
@@ -534,6 +717,13 @@ var isShuttingDown = false;
|
|
|
534
717
|
var FLUSH_INTERVAL_MS = 5e3;
|
|
535
718
|
var MAX_BUFFER_SIZE = 50;
|
|
536
719
|
var FLUSH_TIMEOUT_MS = 3e3;
|
|
720
|
+
var authProvider = null;
|
|
721
|
+
function setTelemetryAuthProvider(provider) {
|
|
722
|
+
authProvider = provider;
|
|
723
|
+
}
|
|
724
|
+
var FLUSH_FAILURE_LOG_INTERVAL_MS = 6e4;
|
|
725
|
+
var lastFlushFailureLoggedAt = 0;
|
|
726
|
+
var suppressedFlushFailureCount = 0;
|
|
537
727
|
function logEvent(eventType, options = {}) {
|
|
538
728
|
const event = {
|
|
539
729
|
event_type: eventType,
|
|
@@ -568,9 +758,16 @@ async function flushEvents() {
|
|
|
568
758
|
flushTimeout = null;
|
|
569
759
|
}
|
|
570
760
|
try {
|
|
571
|
-
const
|
|
572
|
-
|
|
573
|
-
|
|
761
|
+
const providerContext = authProvider?.();
|
|
762
|
+
let authHeader;
|
|
763
|
+
if (providerContext?.authHeader) {
|
|
764
|
+
authHeader = providerContext.authHeader;
|
|
765
|
+
} else {
|
|
766
|
+
const credentials2 = await getToken();
|
|
767
|
+
if (!credentials2) {
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
authHeader = `Bearer ${credentials2.token}`;
|
|
574
771
|
}
|
|
575
772
|
const apiUrl = getApiUrlConfig();
|
|
576
773
|
const controller = new AbortController();
|
|
@@ -585,7 +782,7 @@ async function flushEvents() {
|
|
|
585
782
|
method: "POST",
|
|
586
783
|
headers: {
|
|
587
784
|
"Content-Type": "application/json",
|
|
588
|
-
Authorization:
|
|
785
|
+
Authorization: authHeader
|
|
589
786
|
},
|
|
590
787
|
body: JSON.stringify(request),
|
|
591
788
|
signal: controller.signal
|
|
@@ -597,8 +794,15 @@ async function flushEvents() {
|
|
|
597
794
|
clearTimeout(timeout);
|
|
598
795
|
}
|
|
599
796
|
} catch (error2) {
|
|
600
|
-
|
|
601
|
-
|
|
797
|
+
const now = Date.now();
|
|
798
|
+
if (now - lastFlushFailureLoggedAt >= FLUSH_FAILURE_LOG_INTERVAL_MS) {
|
|
799
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
800
|
+
const suffix = suppressedFlushFailureCount > 0 ? ` (${suppressedFlushFailureCount} more suppressed in the last ${FLUSH_FAILURE_LOG_INTERVAL_MS / 1e3}s)` : "";
|
|
801
|
+
console.error(`Telemetry flush error: ${message}${suffix}`);
|
|
802
|
+
lastFlushFailureLoggedAt = now;
|
|
803
|
+
suppressedFlushFailureCount = 0;
|
|
804
|
+
} else {
|
|
805
|
+
suppressedFlushFailureCount++;
|
|
602
806
|
}
|
|
603
807
|
}
|
|
604
808
|
}
|
|
@@ -667,6 +871,69 @@ var EventTypes = {
|
|
|
667
871
|
DEPRECATED_AGENT_KEY_ENV_USED: "cli.deprecated_agent_key_env_used"
|
|
668
872
|
};
|
|
669
873
|
|
|
874
|
+
// src/lib/runner-activity-telemetry.ts
|
|
875
|
+
var FORWARDED_LEVELS = /* @__PURE__ */ new Set(["warn", "error"]);
|
|
876
|
+
var SEVERITY_BY_LEVEL = {
|
|
877
|
+
warn: "warning",
|
|
878
|
+
error: "error"
|
|
879
|
+
};
|
|
880
|
+
var MAX_MESSAGE_LENGTH = 500;
|
|
881
|
+
var TRUNCATION_MARKER = "\u2026";
|
|
882
|
+
function redact(message) {
|
|
883
|
+
return message.replace(/esk_[A-Za-z0-9_-]+/g, "esk_***").replace(/ct_[A-Za-z0-9_-]+/g, "ct_***").replace(/https?:\/\/\S+/g, "<url>");
|
|
884
|
+
}
|
|
885
|
+
function truncate(message) {
|
|
886
|
+
if (message.length <= MAX_MESSAGE_LENGTH) return message;
|
|
887
|
+
return message.slice(0, MAX_MESSAGE_LENGTH - TRUNCATION_MARKER.length) + TRUNCATION_MARKER;
|
|
888
|
+
}
|
|
889
|
+
var RATE_LIMIT_WINDOW_MS = 6e4;
|
|
890
|
+
var RATE_LIMIT_MAX_EVENTS = 30;
|
|
891
|
+
var windowStartedAt = 0;
|
|
892
|
+
var windowCount = 0;
|
|
893
|
+
var windowDroppedCount = 0;
|
|
894
|
+
function admitUnderRateLimit(now) {
|
|
895
|
+
if (now - windowStartedAt >= RATE_LIMIT_WINDOW_MS) {
|
|
896
|
+
if (windowDroppedCount > 0) {
|
|
897
|
+
console.error(
|
|
898
|
+
`[runner-activity-telemetry] rate cap reached: dropped ${windowDroppedCount} ${windowDroppedCount === 1 ? "entry" : "entries"} in the last ${RATE_LIMIT_WINDOW_MS / 1e3}s (cap ${RATE_LIMIT_MAX_EVENTS}/min)`
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
windowStartedAt = now;
|
|
902
|
+
windowCount = 0;
|
|
903
|
+
windowDroppedCount = 0;
|
|
904
|
+
}
|
|
905
|
+
if (windowCount >= RATE_LIMIT_MAX_EVENTS) {
|
|
906
|
+
windowDroppedCount++;
|
|
907
|
+
if (windowDroppedCount === 1) {
|
|
908
|
+
console.error(
|
|
909
|
+
`[runner-activity-telemetry] rate cap reached (${RATE_LIMIT_MAX_EVENTS} per ${RATE_LIMIT_WINDOW_MS / 1e3}s) \u2014 dropping further entries this window`
|
|
910
|
+
);
|
|
911
|
+
}
|
|
912
|
+
return false;
|
|
913
|
+
}
|
|
914
|
+
windowCount++;
|
|
915
|
+
return true;
|
|
916
|
+
}
|
|
917
|
+
function forwardRunnerActivity(entry, context) {
|
|
918
|
+
try {
|
|
919
|
+
if (!FORWARDED_LEVELS.has(entry.level)) return;
|
|
920
|
+
if (!context.agentId || !context.authHeader) return;
|
|
921
|
+
if (!admitUnderRateLimit(Date.now())) return;
|
|
922
|
+
const rawMessage = entry.error ?? entry.message ?? "";
|
|
923
|
+
const message = truncate(redact(rawMessage));
|
|
924
|
+
logEvent(TelemetryEventTypes.RUNNER_ACTIVITY, {
|
|
925
|
+
severity: SEVERITY_BY_LEVEL[entry.level],
|
|
926
|
+
message,
|
|
927
|
+
metadata: { source: "cli.run" },
|
|
928
|
+
agentId: context.agentId
|
|
929
|
+
});
|
|
930
|
+
} catch (err) {
|
|
931
|
+
console.error(
|
|
932
|
+
`[runner-activity-telemetry] failed to forward runner activity: ${err instanceof Error ? err.message : String(err)}`
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
670
937
|
// src/lib/auth.ts
|
|
671
938
|
async function getAuthCredentials() {
|
|
672
939
|
const runnerKey = process.env.EVIDENT_RUNNER_KEY;
|
|
@@ -1501,6 +1768,9 @@ function isPreamblePinnedRunning(messages, userMessageId) {
|
|
|
1501
1768
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1502
1769
|
return completedOf(reply) != null && finishOf(reply) === "tool-calls";
|
|
1503
1770
|
}
|
|
1771
|
+
function isB2AbandonmentConfirmed(params) {
|
|
1772
|
+
return params.pinnedForMs >= params.minPinnedMs && params.descendantOngoing === false;
|
|
1773
|
+
}
|
|
1504
1774
|
function messageError(messages, userMessageId) {
|
|
1505
1775
|
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1506
1776
|
const error2 = errorOf(reply);
|
|
@@ -1514,6 +1784,42 @@ function messageError(messages, userMessageId) {
|
|
|
1514
1784
|
}
|
|
1515
1785
|
return "The agent run failed.";
|
|
1516
1786
|
}
|
|
1787
|
+
function messageFailure(messages, userMessageId) {
|
|
1788
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
1789
|
+
const error2 = errorOf(reply);
|
|
1790
|
+
if (error2 == null || typeof error2 !== "object") return null;
|
|
1791
|
+
const e = error2;
|
|
1792
|
+
const replyProviderId = reply?.info?.providerID ?? null;
|
|
1793
|
+
const replyModelId = reply?.info?.modelID ?? null;
|
|
1794
|
+
if (e.name === "ProviderAuthError") {
|
|
1795
|
+
const data = e.data;
|
|
1796
|
+
const providerId = typeof data?.providerID === "string" && data.providerID || replyProviderId;
|
|
1797
|
+
return { kind: "model_auth", providerId, modelId: replyModelId, reason: "missing" };
|
|
1798
|
+
}
|
|
1799
|
+
if (e.name === "APIError") {
|
|
1800
|
+
const data = e.data;
|
|
1801
|
+
const statusCode = data?.statusCode;
|
|
1802
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
1803
|
+
return {
|
|
1804
|
+
kind: "model_auth",
|
|
1805
|
+
providerId: replyProviderId,
|
|
1806
|
+
modelId: replyModelId,
|
|
1807
|
+
reason: "rejected"
|
|
1808
|
+
};
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
return null;
|
|
1812
|
+
}
|
|
1813
|
+
function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
|
|
1814
|
+
if (classified != null) return classified;
|
|
1815
|
+
if (hasConfiguredProvider !== false) return null;
|
|
1816
|
+
return {
|
|
1817
|
+
kind: "model_auth",
|
|
1818
|
+
providerId: replyProviderId,
|
|
1819
|
+
modelId: replyModelId,
|
|
1820
|
+
reason: "missing"
|
|
1821
|
+
};
|
|
1822
|
+
}
|
|
1517
1823
|
function hasRunningAssistantExcept(messages, exceptUserMessageId) {
|
|
1518
1824
|
if (!messages || messages.length === 0) return false;
|
|
1519
1825
|
return messages.some(
|
|
@@ -2042,13 +2348,52 @@ var RunnerConnection = class {
|
|
|
2042
2348
|
}
|
|
2043
2349
|
};
|
|
2044
2350
|
|
|
2351
|
+
// src/lib/tunnel/ready-marker.ts
|
|
2352
|
+
import { writeFileSync } from "fs";
|
|
2353
|
+
function writeTunnelReadyMarker(path, agentId) {
|
|
2354
|
+
try {
|
|
2355
|
+
writeFileSync(path, `${agentId}
|
|
2356
|
+
`);
|
|
2357
|
+
return { ok: true };
|
|
2358
|
+
} catch (error2) {
|
|
2359
|
+
return { ok: false, error: error2 instanceof Error ? error2.message : String(error2) };
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
// src/lib/claude-usage-reporting.ts
|
|
2364
|
+
var VALID_MODES = ["auto", "on", "off"];
|
|
2365
|
+
function resolveClaudeUsageReportingMode(flagValue, env) {
|
|
2366
|
+
const raw = flagValue ?? env.EVIDENT_CLAUDE_USAGE_REPORTING;
|
|
2367
|
+
if (raw === void 0 || raw === "") {
|
|
2368
|
+
return { mode: "auto", warnings: [] };
|
|
2369
|
+
}
|
|
2370
|
+
const normalized = raw.trim().toLowerCase();
|
|
2371
|
+
if (VALID_MODES.includes(normalized)) {
|
|
2372
|
+
return { mode: normalized, warnings: [] };
|
|
2373
|
+
}
|
|
2374
|
+
const source = flagValue !== void 0 ? "--claude-usage-reporting" : "EVIDENT_CLAUDE_USAGE_REPORTING";
|
|
2375
|
+
return {
|
|
2376
|
+
mode: "auto",
|
|
2377
|
+
warnings: [
|
|
2378
|
+
`Ignoring invalid ${source} "${raw}": expected one of ${VALID_MODES.join(", ")}; using auto`
|
|
2379
|
+
]
|
|
2380
|
+
};
|
|
2381
|
+
}
|
|
2382
|
+
var BASE_REPORT_DELAY_MS = 10 * 6e4;
|
|
2383
|
+
var REPORT_DELAY_JITTER_FRACTION = 0.2;
|
|
2384
|
+
function nextReportDelayMs(random = Math.random) {
|
|
2385
|
+
const jitterRangeMs = BASE_REPORT_DELAY_MS * REPORT_DELAY_JITTER_FRACTION;
|
|
2386
|
+
return BASE_REPORT_DELAY_MS - jitterRangeMs + random() * (2 * jitterRangeMs);
|
|
2387
|
+
}
|
|
2388
|
+
var FIRST_REPORT_DELAY_MS = 5e3 + Math.random() * 1e4;
|
|
2389
|
+
|
|
2045
2390
|
// src/lib/channels/driver.ts
|
|
2046
2391
|
import { homedir as homedir2 } from "os";
|
|
2047
2392
|
|
|
2048
2393
|
// src/lib/file-push.ts
|
|
2049
2394
|
import { randomUUID } from "crypto";
|
|
2050
2395
|
import { chmod, mkdir, open as open2, realpath, rename, unlink } from "fs/promises";
|
|
2051
|
-
import { basename, dirname, isAbsolute, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
2396
|
+
import { basename, dirname as dirname2, isAbsolute, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
2052
2397
|
var FILE_MODE = 384;
|
|
2053
2398
|
var DIRECTORY_MODE = 448;
|
|
2054
2399
|
async function writePushedFile(request) {
|
|
@@ -2079,7 +2424,7 @@ async function writePushedFile(request) {
|
|
|
2079
2424
|
}
|
|
2080
2425
|
try {
|
|
2081
2426
|
const { existingAncestor, missingSegments } = await resolveNearestExistingAncestor(
|
|
2082
|
-
|
|
2427
|
+
dirname2(candidate)
|
|
2083
2428
|
);
|
|
2084
2429
|
const realTarget = join2(existingAncestor, ...missingSegments, basename(candidate));
|
|
2085
2430
|
const allowedDirectory = await findContainingAllowedDirectory(allowedDirectories, realTarget);
|
|
@@ -2091,8 +2436,8 @@ async function writePushedFile(request) {
|
|
|
2091
2436
|
}
|
|
2092
2437
|
if (missingSegments.length > 0) {
|
|
2093
2438
|
await createMissingDirectories(existingAncestor, missingSegments);
|
|
2094
|
-
const realParent = await realpath(
|
|
2095
|
-
if (realParent !==
|
|
2439
|
+
const realParent = await realpath(dirname2(realTarget));
|
|
2440
|
+
if (realParent !== dirname2(realTarget) || !contains(allowedDirectory, realTarget)) {
|
|
2096
2441
|
return refuse("path_not_allowed", "The runner does not allow writing to that location.", {
|
|
2097
2442
|
path: realTarget,
|
|
2098
2443
|
bytes,
|
|
@@ -2135,7 +2480,7 @@ async function resolveNearestExistingAncestor(directory) {
|
|
|
2135
2480
|
try {
|
|
2136
2481
|
return { existingAncestor: await realpath(current), missingSegments };
|
|
2137
2482
|
} catch (err) {
|
|
2138
|
-
const parent =
|
|
2483
|
+
const parent = dirname2(current);
|
|
2139
2484
|
if (err.code !== "ENOENT" || parent === current) {
|
|
2140
2485
|
throw err;
|
|
2141
2486
|
}
|
|
@@ -2196,7 +2541,7 @@ async function createMissingDirectories(existingAncestor, missingSegments) {
|
|
|
2196
2541
|
}
|
|
2197
2542
|
}
|
|
2198
2543
|
async function writeAtomically(realTarget, content) {
|
|
2199
|
-
const temporaryPath = join2(
|
|
2544
|
+
const temporaryPath = join2(dirname2(realTarget), `.evident-push-${randomUUID()}.tmp`);
|
|
2200
2545
|
let handle;
|
|
2201
2546
|
try {
|
|
2202
2547
|
handle = await open2(temporaryPath, "wx", FILE_MODE);
|
|
@@ -2468,6 +2813,8 @@ var DEFAULT_PAUSED_MAX_WAIT_MS = 10 * 60 * 1e3;
|
|
|
2468
2813
|
var DEFAULT_STUCK_QUEUED_MS = 6e4;
|
|
2469
2814
|
var HEARTBEAT_MS = 6e4;
|
|
2470
2815
|
var ABSOLUTE_MAX_PROCESSING_MS = 6 * 60 * 60 * 1e3;
|
|
2816
|
+
var B2_ABANDONMENT_MIN_PINNED_MS = 3 * 6e4;
|
|
2817
|
+
var B2_ABANDONMENT_RECHECK_MS = HEARTBEAT_MS;
|
|
2471
2818
|
var POLL_MISS_GRACE_MS = HEARTBEAT_MS;
|
|
2472
2819
|
var MAX_SUPERSEDED_CONVERSATIONS = 256;
|
|
2473
2820
|
var ChannelAuthError = class extends Error {
|
|
@@ -2693,23 +3040,23 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
2693
3040
|
* and stops opencode.
|
|
2694
3041
|
*/
|
|
2695
3042
|
stopped = false;
|
|
2696
|
-
constructor(
|
|
2697
|
-
this.agentId =
|
|
2698
|
-
this.port =
|
|
2699
|
-
this.apiUrl =
|
|
2700
|
-
this.getAuthHeader =
|
|
2701
|
-
this.conversationFilter =
|
|
2702
|
-
this.retry = { ...DEFAULT_RETRY_POLICY, ...
|
|
2703
|
-
this.log =
|
|
3043
|
+
constructor(config) {
|
|
3044
|
+
this.agentId = config.agentId;
|
|
3045
|
+
this.port = config.port;
|
|
3046
|
+
this.apiUrl = config.apiUrl.replace(/\/$/, "");
|
|
3047
|
+
this.getAuthHeader = config.getAuthHeader;
|
|
3048
|
+
this.conversationFilter = config.conversationFilter ?? null;
|
|
3049
|
+
this.retry = { ...DEFAULT_RETRY_POLICY, ...config.retry };
|
|
3050
|
+
this.log = config.log ?? (() => {
|
|
2704
3051
|
});
|
|
2705
|
-
this.fetchImpl =
|
|
2706
|
-
this.sleep =
|
|
2707
|
-
this.pausedPollIntervalMs =
|
|
2708
|
-
this.pausedMaxWaitMs =
|
|
2709
|
-
this.stuckQueuedMs =
|
|
2710
|
-
this.now =
|
|
2711
|
-
this.fileSyncDirectories =
|
|
2712
|
-
this.homeDir =
|
|
3052
|
+
this.fetchImpl = config.fetchImpl ?? fetch;
|
|
3053
|
+
this.sleep = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3054
|
+
this.pausedPollIntervalMs = config.pausedPollIntervalMs ?? DEFAULT_PAUSED_POLL_INTERVAL_MS;
|
|
3055
|
+
this.pausedMaxWaitMs = config.pausedMaxWaitMs ?? DEFAULT_PAUSED_MAX_WAIT_MS;
|
|
3056
|
+
this.stuckQueuedMs = config.stuckQueuedMs ?? DEFAULT_STUCK_QUEUED_MS;
|
|
3057
|
+
this.now = config.now ?? (() => Date.now());
|
|
3058
|
+
this.fileSyncDirectories = config.fileSyncDirectories ?? [];
|
|
3059
|
+
this.homeDir = config.homeDir ?? homedir2();
|
|
2713
3060
|
}
|
|
2714
3061
|
/** The IPv4-loopback base URL for the local `opencode serve`. */
|
|
2715
3062
|
get opencodeBase() {
|
|
@@ -3094,7 +3441,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3094
3441
|
const directory = await this.resolveOpenCodeDirectory();
|
|
3095
3442
|
const sessionId = await createOpenCodeSession(this.port, directory);
|
|
3096
3443
|
this.sessions.set(conversationId, sessionId);
|
|
3097
|
-
await this.persistSession(conversationId, sessionId).catch(() => {
|
|
3444
|
+
await this.persistSession(conversationId, sessionId).catch((err) => {
|
|
3445
|
+
this.log({
|
|
3446
|
+
level: "warn",
|
|
3447
|
+
message: `Persisting the OpenCode session binding ${sessionId.slice(0, 8)} for conversation ${conversationId.slice(0, 8)} failed (best-effort, not retried) \u2014 the completion PATCH also carries opencode_session_id, so the binding is repaired when the turn finishes: ${err instanceof Error ? err.message : String(err)}`,
|
|
3448
|
+
conversation_id: conversationId
|
|
3449
|
+
});
|
|
3098
3450
|
});
|
|
3099
3451
|
return sessionId;
|
|
3100
3452
|
}
|
|
@@ -3286,12 +3638,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3286
3638
|
stuckReported: false,
|
|
3287
3639
|
lastAliveAt: 0,
|
|
3288
3640
|
aliveInFlight: false,
|
|
3641
|
+
titleSynced: false,
|
|
3642
|
+
titleSyncInFlight: false,
|
|
3289
3643
|
awaitingHumanLatched: false,
|
|
3290
3644
|
pausedOnQuestion: false,
|
|
3291
3645
|
pausedOnPermission: false,
|
|
3292
3646
|
pausedClearConfirmed: false,
|
|
3293
3647
|
pausedInFlight: false,
|
|
3294
|
-
deliveryDeadlineAnchored: false
|
|
3648
|
+
deliveryDeadlineAnchored: false,
|
|
3649
|
+
b2PinnedSinceMs: 0,
|
|
3650
|
+
b2LastDescendantCheckMs: 0,
|
|
3651
|
+
b2AbandonedSignalled: false
|
|
3295
3652
|
});
|
|
3296
3653
|
}
|
|
3297
3654
|
/**
|
|
@@ -3359,12 +3716,17 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3359
3716
|
// with no extra `re_adopted` signal needed (folds old WI-6).
|
|
3360
3717
|
lastAliveAt: 0,
|
|
3361
3718
|
aliveInFlight: false,
|
|
3719
|
+
titleSynced: false,
|
|
3720
|
+
titleSyncInFlight: false,
|
|
3362
3721
|
awaitingHumanLatched: false,
|
|
3363
3722
|
pausedOnQuestion: false,
|
|
3364
3723
|
pausedOnPermission: false,
|
|
3365
3724
|
pausedClearConfirmed: false,
|
|
3366
3725
|
pausedInFlight: false,
|
|
3367
|
-
deliveryDeadlineAnchored: false
|
|
3726
|
+
deliveryDeadlineAnchored: false,
|
|
3727
|
+
b2PinnedSinceMs: 0,
|
|
3728
|
+
b2LastDescendantCheckMs: 0,
|
|
3729
|
+
b2AbandonedSignalled: false
|
|
3368
3730
|
});
|
|
3369
3731
|
}
|
|
3370
3732
|
/**
|
|
@@ -3526,58 +3888,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3526
3888
|
}
|
|
3527
3889
|
}
|
|
3528
3890
|
if (state === "done") {
|
|
3529
|
-
this.
|
|
3530
|
-
if (!inFlight.done) {
|
|
3531
|
-
this.log({
|
|
3532
|
-
level: "info",
|
|
3533
|
-
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
3534
|
-
conversation_id: conv.id,
|
|
3535
|
-
message_id: inFlight.evidentMessageId
|
|
3536
|
-
});
|
|
3537
|
-
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
3538
|
-
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3539
|
-
try {
|
|
3540
|
-
await this.markDone(
|
|
3541
|
-
conv.id,
|
|
3542
|
-
inFlight.evidentMessageId,
|
|
3543
|
-
sessionId,
|
|
3544
|
-
inFlight.opencodeMessageId,
|
|
3545
|
-
title,
|
|
3546
|
-
usage
|
|
3547
|
-
);
|
|
3548
|
-
} catch (err) {
|
|
3549
|
-
if (err instanceof ChannelAuthError) throw err;
|
|
3550
|
-
if (err instanceof ChannelTerminalError) {
|
|
3551
|
-
this.log({
|
|
3552
|
-
level: "warn",
|
|
3553
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
3554
|
-
conversation_id: conv.id,
|
|
3555
|
-
message_id: inFlight.evidentMessageId
|
|
3556
|
-
});
|
|
3557
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3558
|
-
return;
|
|
3559
|
-
}
|
|
3560
|
-
if (this.now() >= inFlight.deadline) {
|
|
3561
|
-
this.log({
|
|
3562
|
-
level: "warn",
|
|
3563
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
3564
|
-
conversation_id: conv.id,
|
|
3565
|
-
message_id: inFlight.evidentMessageId
|
|
3566
|
-
});
|
|
3567
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3568
|
-
return;
|
|
3569
|
-
}
|
|
3570
|
-
this.log({
|
|
3571
|
-
level: "warn",
|
|
3572
|
-
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
3573
|
-
conversation_id: conv.id,
|
|
3574
|
-
message_id: inFlight.evidentMessageId
|
|
3575
|
-
});
|
|
3576
|
-
return;
|
|
3577
|
-
}
|
|
3578
|
-
inFlight.done = true;
|
|
3579
|
-
}
|
|
3580
|
-
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3891
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3581
3892
|
return;
|
|
3582
3893
|
}
|
|
3583
3894
|
if (state === "failed") {
|
|
@@ -3591,8 +3902,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3591
3902
|
message_id: inFlight.evidentMessageId
|
|
3592
3903
|
});
|
|
3593
3904
|
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
3905
|
+
const failure = await this.classifyModelAuthFailure(messages, inFlight.opencodeMessageId);
|
|
3594
3906
|
try {
|
|
3595
|
-
await this.markFailed(
|
|
3907
|
+
await this.markFailed(
|
|
3908
|
+
conv.id,
|
|
3909
|
+
inFlight.evidentMessageId,
|
|
3910
|
+
sessionId,
|
|
3911
|
+
error2,
|
|
3912
|
+
usage,
|
|
3913
|
+
failure
|
|
3914
|
+
);
|
|
3596
3915
|
} catch (err) {
|
|
3597
3916
|
if (err instanceof ChannelAuthError) throw err;
|
|
3598
3917
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -3637,6 +3956,44 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3637
3956
|
});
|
|
3638
3957
|
}
|
|
3639
3958
|
const activelyRunning = state === "running" && !awaitingHuman;
|
|
3959
|
+
const pinnedNow = activelyRunning && isPreamblePinnedRunning(messages, inFlight.opencodeMessageId);
|
|
3960
|
+
const snapshotReadable = messages != null && messages.length > 0;
|
|
3961
|
+
if (!pinnedNow) {
|
|
3962
|
+
if (snapshotReadable) {
|
|
3963
|
+
inFlight.b2PinnedSinceMs = 0;
|
|
3964
|
+
inFlight.b2LastDescendantCheckMs = 0;
|
|
3965
|
+
inFlight.b2AbandonedSignalled = false;
|
|
3966
|
+
}
|
|
3967
|
+
} else {
|
|
3968
|
+
if (inFlight.b2AbandonedSignalled) {
|
|
3969
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3970
|
+
return;
|
|
3971
|
+
}
|
|
3972
|
+
if (inFlight.b2PinnedSinceMs === 0) inFlight.b2PinnedSinceMs = this.now();
|
|
3973
|
+
const pinnedForMs = this.now() - inFlight.b2PinnedSinceMs;
|
|
3974
|
+
if (pinnedForMs >= B2_ABANDONMENT_MIN_PINNED_MS && this.now() - inFlight.b2LastDescendantCheckMs >= B2_ABANDONMENT_RECHECK_MS) {
|
|
3975
|
+
inFlight.b2LastDescendantCheckMs = this.now();
|
|
3976
|
+
const descendantOngoing = await this.isAnyDescendantSessionOngoing(sessionId);
|
|
3977
|
+
if (isB2AbandonmentConfirmed({
|
|
3978
|
+
pinnedForMs,
|
|
3979
|
+
minPinnedMs: B2_ABANDONMENT_MIN_PINNED_MS,
|
|
3980
|
+
descendantOngoing
|
|
3981
|
+
})) {
|
|
3982
|
+
inFlight.b2AbandonedSignalled = true;
|
|
3983
|
+
this.log({
|
|
3984
|
+
level: "warn",
|
|
3985
|
+
message: `Message ${id.slice(0, 8)} b2-pinned for ${Math.round(pinnedForMs / 1e3)}s with no ongoing descendant sub-agent session (status-map confirmed) \u2014 treating the delegated/tool turn as abandoned, resolving done`,
|
|
3986
|
+
conversation_id: conv.id,
|
|
3987
|
+
message_id: id
|
|
3988
|
+
});
|
|
3989
|
+
void this.postSignal(conv.id, id, "b2_abandoned_resolved", {
|
|
3990
|
+
watched_for_ms: pinnedForMs
|
|
3991
|
+
});
|
|
3992
|
+
await this.settleMessageDone(sessionId, watcher, inFlight, messages);
|
|
3993
|
+
return;
|
|
3994
|
+
}
|
|
3995
|
+
}
|
|
3996
|
+
}
|
|
3640
3997
|
if (activelyRunning && this.now() - inFlight.processingAnchorMs >= ABSOLUTE_MAX_PROCESSING_MS) {
|
|
3641
3998
|
this.log({
|
|
3642
3999
|
level: "warn",
|
|
@@ -3656,6 +4013,18 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3656
4013
|
inFlight.aliveInFlight = false;
|
|
3657
4014
|
if (ok) inFlight.lastAliveAt = this.now();
|
|
3658
4015
|
});
|
|
4016
|
+
if (!inFlight.titleSynced && !inFlight.titleSyncInFlight) {
|
|
4017
|
+
inFlight.titleSyncInFlight = true;
|
|
4018
|
+
void this.resolveSessionTitle(sessionId, conv.id).then(async (title) => {
|
|
4019
|
+
if (!title) {
|
|
4020
|
+
inFlight.titleSyncInFlight = false;
|
|
4021
|
+
return;
|
|
4022
|
+
}
|
|
4023
|
+
const ok = await this.patchConversationTitle(conv.id, title);
|
|
4024
|
+
inFlight.titleSyncInFlight = false;
|
|
4025
|
+
if (ok) inFlight.titleSynced = true;
|
|
4026
|
+
});
|
|
4027
|
+
}
|
|
3659
4028
|
}
|
|
3660
4029
|
if (awaitingHuman) {
|
|
3661
4030
|
if (!inFlight.awaitingHumanLatched) {
|
|
@@ -3693,6 +4062,70 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3693
4062
|
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
3694
4063
|
}
|
|
3695
4064
|
}
|
|
4065
|
+
/**
|
|
4066
|
+
* Settle a message whose run-state has resolved `'done'` — extracted verbatim
|
|
4067
|
+
* (pure refactor, no behavior change) from `serviceInFlightMessage`'s former
|
|
4068
|
+
* inline `state === 'done'` branch body, so a SECOND caller (the #721
|
|
4069
|
+
* b2-abandonment resolution) can reach the exact same completion behavior
|
|
4070
|
+
* (delivery-deadline anchoring, title resolution, usage extraction, and
|
|
4071
|
+
* `markDone`'s auth/terminal/transient-retry discipline) without duplicating it
|
|
4072
|
+
* and risking the two copies silently drifting apart.
|
|
4073
|
+
*/
|
|
4074
|
+
async settleMessageDone(sessionId, watcher, inFlight, messages) {
|
|
4075
|
+
const conv = watcher.conv;
|
|
4076
|
+
this.anchorDeliveryDeadline(inFlight);
|
|
4077
|
+
if (!inFlight.done) {
|
|
4078
|
+
this.log({
|
|
4079
|
+
level: "info",
|
|
4080
|
+
message: `Message ${inFlight.evidentMessageId.slice(0, 8)} completed \u2014 marking done`,
|
|
4081
|
+
conversation_id: conv.id,
|
|
4082
|
+
message_id: inFlight.evidentMessageId
|
|
4083
|
+
});
|
|
4084
|
+
const title = await this.resolveSessionTitle(sessionId, watcher.conv.id);
|
|
4085
|
+
const usage = messageUsage(messages, inFlight.opencodeMessageId);
|
|
4086
|
+
try {
|
|
4087
|
+
await this.markDone(
|
|
4088
|
+
conv.id,
|
|
4089
|
+
inFlight.evidentMessageId,
|
|
4090
|
+
sessionId,
|
|
4091
|
+
inFlight.opencodeMessageId,
|
|
4092
|
+
title,
|
|
4093
|
+
usage
|
|
4094
|
+
);
|
|
4095
|
+
} catch (err) {
|
|
4096
|
+
if (err instanceof ChannelAuthError) throw err;
|
|
4097
|
+
if (err instanceof ChannelTerminalError) {
|
|
4098
|
+
this.log({
|
|
4099
|
+
level: "warn",
|
|
4100
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (terminal HTTP ${err.status}) \u2014 leaving for the cron safety net: ${err.message}`,
|
|
4101
|
+
conversation_id: conv.id,
|
|
4102
|
+
message_id: inFlight.evidentMessageId
|
|
4103
|
+
});
|
|
4104
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4105
|
+
return;
|
|
4106
|
+
}
|
|
4107
|
+
if (this.now() >= inFlight.deadline) {
|
|
4108
|
+
this.log({
|
|
4109
|
+
level: "warn",
|
|
4110
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done within the watch window \u2014 leaving for the cron safety net: ${err instanceof Error ? err.message : String(err)}`,
|
|
4111
|
+
conversation_id: conv.id,
|
|
4112
|
+
message_id: inFlight.evidentMessageId
|
|
4113
|
+
});
|
|
4114
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4115
|
+
return;
|
|
4116
|
+
}
|
|
4117
|
+
this.log({
|
|
4118
|
+
level: "warn",
|
|
4119
|
+
message: `Failed to mark message ${inFlight.evidentMessageId.slice(0, 8)} done (will retry next tick): ${err instanceof Error ? err.message : String(err)}`,
|
|
4120
|
+
conversation_id: conv.id,
|
|
4121
|
+
message_id: inFlight.evidentMessageId
|
|
4122
|
+
});
|
|
4123
|
+
return;
|
|
4124
|
+
}
|
|
4125
|
+
inFlight.done = true;
|
|
4126
|
+
}
|
|
4127
|
+
this.removeInFlight(watcher, inFlight.evidentMessageId);
|
|
4128
|
+
}
|
|
3696
4129
|
// Restart recovery: re-adopt `processing` messages (ADR-0046, WI-3/4/5)
|
|
3697
4130
|
/**
|
|
3698
4131
|
* Re-adopt this agent's `processing` messages on drain (ADR-0046 Decision §1).
|
|
@@ -3859,6 +4292,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3859
4292
|
if (state === "failed") {
|
|
3860
4293
|
const error2 = messageError(messages, ocId ?? "") ?? void 0;
|
|
3861
4294
|
const usage = messageUsage(messages, ocId ?? "");
|
|
4295
|
+
const failure = await this.classifyModelAuthFailure(messages, ocId ?? "");
|
|
3862
4296
|
this.log({
|
|
3863
4297
|
level: "error",
|
|
3864
4298
|
message: `Re-adopt: message ${row.id.slice(0, 8)} errored while unwatched \u2014 marking failed: ${error2 ?? "(no error text)"}`,
|
|
@@ -3866,7 +4300,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
3866
4300
|
message_id: row.id
|
|
3867
4301
|
});
|
|
3868
4302
|
try {
|
|
3869
|
-
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage);
|
|
4303
|
+
await this.markFailed(row.conversation_id, row.id, sessionId, error2, usage, failure);
|
|
3870
4304
|
} catch (err) {
|
|
3871
4305
|
if (err instanceof ChannelAuthError) throw err;
|
|
3872
4306
|
if (err instanceof ChannelTerminalError) {
|
|
@@ -4272,6 +4706,47 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4272
4706
|
}
|
|
4273
4707
|
return false;
|
|
4274
4708
|
}
|
|
4709
|
+
/**
|
|
4710
|
+
* Tri-state variant of the upward parentID membership walk (#721), used ONLY
|
|
4711
|
+
* by `isAnyDescendantSessionOngoing`. Walks the SAME cached
|
|
4712
|
+
* `resolveSessionParent` chain `sessionBelongsTo` uses above, but — unlike
|
|
4713
|
+
* `sessionBelongsTo`, which deliberately collapses "confirmed not a
|
|
4714
|
+
* descendant" and "the walk's fetch failed" into the same `false` (safe for
|
|
4715
|
+
* its OTHER callers: interaction attribution and the recovery-path
|
|
4716
|
+
* `isAnyDescendantSessionAlive`, both of which just retry next tick with no
|
|
4717
|
+
* safety consequence either way) — this variant keeps those two outcomes
|
|
4718
|
+
* SEPARATE, because `isAnyDescendantSessionOngoing`'s caller
|
|
4719
|
+
* (`isB2AbandonmentConfirmed`) must never treat "couldn't tell" as "confirmed
|
|
4720
|
+
* not ongoing".
|
|
4721
|
+
*
|
|
4722
|
+
* Return contract:
|
|
4723
|
+
* - `true` → the walk reached `rootSessionId` — `sessionId` IS a descendant.
|
|
4724
|
+
* - `false` → the walk reached a definitive, parent-less root session
|
|
4725
|
+
* WITHOUT ever matching `rootSessionId` — `sessionId` is
|
|
4726
|
+
* CONFIRMED NOT a descendant of it.
|
|
4727
|
+
* - `null` → INDETERMINATE: a `GET /session/:id` fetch failed partway
|
|
4728
|
+
* through the walk (`resolveSessionParent` returned `undefined`),
|
|
4729
|
+
* or the depth cap (32) was hit without a definitive answer (a
|
|
4730
|
+
* pathological/cyclic chain proves nothing either way). NEVER
|
|
4731
|
+
* treat this the same as `false` — see `sessionBelongsTo`'s own
|
|
4732
|
+
* doc comment above for why that collapse is safe THERE but not
|
|
4733
|
+
* here.
|
|
4734
|
+
*
|
|
4735
|
+
* `sessionBelongsTo` itself is UNCHANGED — this is an additive helper scoped
|
|
4736
|
+
* to the live-path descendant check, not a modification of shared code used
|
|
4737
|
+
* by interaction attribution or the recovery path.
|
|
4738
|
+
*/
|
|
4739
|
+
async resolveSessionMembership(sessionId, rootSessionId) {
|
|
4740
|
+
let current = sessionId;
|
|
4741
|
+
for (let depth = 0; current && depth < 32; depth++) {
|
|
4742
|
+
if (current === rootSessionId) return true;
|
|
4743
|
+
const parent = await this.resolveSessionParent(current);
|
|
4744
|
+
if (parent === void 0) return null;
|
|
4745
|
+
if (parent === null) return false;
|
|
4746
|
+
current = parent;
|
|
4747
|
+
}
|
|
4748
|
+
return null;
|
|
4749
|
+
}
|
|
4275
4750
|
/**
|
|
4276
4751
|
* Resolve (and cache) a session's `parentID` via `GET /session/:id`. Returns
|
|
4277
4752
|
* `null` for a root session (no parent) and `undefined` when opencode is
|
|
@@ -4356,6 +4831,54 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4356
4831
|
}
|
|
4357
4832
|
return null;
|
|
4358
4833
|
}
|
|
4834
|
+
/**
|
|
4835
|
+
* Best-effort mid-turn title sync (#711 follow-up): PATCH a resolved OpenCode
|
|
4836
|
+
* session title onto the conversation via the PLAIN conversation-update
|
|
4837
|
+
* endpoint (`PATCH /runners/:agentId/conversations/:conversationId`) — NOT the
|
|
4838
|
+
* message-status endpoint `markProcessing`/`markDone` use. Deliberately a
|
|
4839
|
+
* separate, lighter call: it carries no `status`, so it cannot re-trigger the
|
|
4840
|
+
* `processing`/`done` transition side effects (Slack notices, activity-log
|
|
4841
|
+
* rows, delivery jobs) those PATCHes gate on `transitioned` — this call only
|
|
4842
|
+
* ever touches `conversations.title`. That route (`routes/conversations.ts`)
|
|
4843
|
+
* skips a title write matching the stored value, so a redundant call with the
|
|
4844
|
+
* same title is a real no-op — it does not bump `updated_at`, which the
|
|
4845
|
+
* conversation list sorts and paginates on. (Note this is a DIFFERENT guard
|
|
4846
|
+
* from `threads.ts`'s "non-empty AND changed" one, which only covers the
|
|
4847
|
+
* message-status PATCH; the non-empty half is enforced here instead, by
|
|
4848
|
+
* `resolveSessionTitle` never returning an empty/placeholder title.)
|
|
4849
|
+
*
|
|
4850
|
+
* Telemetry-only / never blocks the caller, mirroring `postSignal`: a failure
|
|
4851
|
+
* is logged and the title is simply retried on the next heartbeat tick (the
|
|
4852
|
+
* caller only latches `titleSynced` on `true`).
|
|
4853
|
+
*/
|
|
4854
|
+
async patchConversationTitle(conversationId, title) {
|
|
4855
|
+
try {
|
|
4856
|
+
const res = await this.fetchImpl(
|
|
4857
|
+
`${this.apiUrl}/runners/${this.agentId}/conversations/${conversationId}`,
|
|
4858
|
+
{
|
|
4859
|
+
method: "PATCH",
|
|
4860
|
+
headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
|
|
4861
|
+
body: JSON.stringify({ title })
|
|
4862
|
+
}
|
|
4863
|
+
);
|
|
4864
|
+
if (!res.ok) {
|
|
4865
|
+
this.log({
|
|
4866
|
+
level: "debug",
|
|
4867
|
+
message: `Mid-turn title sync PATCH for conversation ${conversationId.slice(0, 8)} returned HTTP ${res.status} (best-effort, will retry next heartbeat)`,
|
|
4868
|
+
conversation_id: conversationId
|
|
4869
|
+
});
|
|
4870
|
+
return false;
|
|
4871
|
+
}
|
|
4872
|
+
return true;
|
|
4873
|
+
} catch (err) {
|
|
4874
|
+
this.log({
|
|
4875
|
+
level: "debug",
|
|
4876
|
+
message: `Best-effort mid-turn title sync PATCH failed for conversation ${conversationId.slice(0, 8)} (will retry next heartbeat): ${err instanceof Error ? err.message : String(err)}`,
|
|
4877
|
+
conversation_id: conversationId
|
|
4878
|
+
});
|
|
4879
|
+
return false;
|
|
4880
|
+
}
|
|
4881
|
+
}
|
|
4359
4882
|
/**
|
|
4360
4883
|
* DEFENSIVE cross-check for the restart-recovery path (WI-2): is any descendant
|
|
4361
4884
|
* (`task` sub-agent) session under `rootSessionId` still genuinely doing work?
|
|
@@ -4414,6 +4937,84 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4414
4937
|
}
|
|
4415
4938
|
return false;
|
|
4416
4939
|
}
|
|
4940
|
+
/**
|
|
4941
|
+
* LIVE-PATH descendant-liveness check (#721): is any descendant (`task`
|
|
4942
|
+
* sub-agent) session under `rootSessionId` currently ONGOING per OpenCode's own
|
|
4943
|
+
* in-memory status map (`isSessionOngoing` — `busy`/`retry`)?
|
|
4944
|
+
*
|
|
4945
|
+
* Deliberately NOT `isAnyDescendantSessionAlive` (the RECOVERY-path
|
|
4946
|
+
* cross-check above): that method judges liveness from the child's OWN
|
|
4947
|
+
* TRANSCRIPT (`isSessionActivelyGenerating`), which is the right (only) option
|
|
4948
|
+
* on the recovery path because a restart WIPES `SessionStatus`. On the LIVE
|
|
4949
|
+
* path the local opencode server IS running, so its in-memory status map is
|
|
4950
|
+
* live and authoritative — and per ADR-0047 §4a ("the child has its own entry
|
|
4951
|
+
* [in the map]"), a `task` descendant's OWN busy/retry entry reflects its
|
|
4952
|
+
* ENTIRE turn (including any tool call it is itself executing), not a
|
|
4953
|
+
* per-message transcript snapshot. This sidesteps the "child's own tool is
|
|
4954
|
+
* executing, between its step's completion and the next generation step"
|
|
4955
|
+
* transcript gap that a transcript-based check would need a second,
|
|
4956
|
+
* sustained-window bound to guard against — it is simply not derived from
|
|
4957
|
+
* message timestamps at all.
|
|
4958
|
+
*
|
|
4959
|
+
* Why not just check `isSessionOngoing(port, rootSessionId)` (the ROOT's own
|
|
4960
|
+
* status, as the recovery path does per §4a)? Because on the LIVE path the
|
|
4961
|
+
* root session can be shared: a SECOND, unrelated user message can land on the
|
|
4962
|
+
* SAME session (issue #721's own root cause) and keep the root `busy` for a
|
|
4963
|
+
* reason that has nothing to do with THIS message's delegation. A `task`
|
|
4964
|
+
* descendant session is spawned for exactly one delegated turn and never
|
|
4965
|
+
* reused, so its OWN status-map entry is unambiguous evidence about that one
|
|
4966
|
+
* delegation — which the root's status is not.
|
|
4967
|
+
*
|
|
4968
|
+
* Why membership is checked via `resolveSessionMembership`, NOT
|
|
4969
|
+
* `sessionBelongsTo`: `sessionBelongsTo` collapses a transient
|
|
4970
|
+
* `GET /session/:id` fetch failure into "not a descendant", which would
|
|
4971
|
+
* silently drop a genuinely-live candidate from consideration on the one
|
|
4972
|
+
* unlucky tick its membership-walk fetch hiccups (#721).
|
|
4973
|
+
* `resolveSessionMembership` keeps that failure mode as a distinct `null`
|
|
4974
|
+
* (indeterminate) so it is folded into THIS method's own `indeterminate` flag
|
|
4975
|
+
* instead.
|
|
4976
|
+
*
|
|
4977
|
+
* Return contract (note the DIFFERENT judge vs. `isAnyDescendantSessionAlive`):
|
|
4978
|
+
* - `true` → some descendant session is `busy`/`retry` (genuinely ongoing).
|
|
4979
|
+
* - `false` → enumeration succeeded, EVERY candidate's MEMBERSHIP was
|
|
4980
|
+
* confirmed either way (`resolveSessionMembership` never
|
|
4981
|
+
* returned `null`), and every CONFIRMED descendant's status read
|
|
4982
|
+
* succeeded and is not ongoing (includes "no descendant session
|
|
4983
|
+
* exists at all" — e.g. a plain, non-`task` tool call).
|
|
4984
|
+
* - `null` → INDETERMINATE: `listSessions` failed, OR at least one
|
|
4985
|
+
* candidate's MEMBERSHIP could not be confirmed
|
|
4986
|
+
* (`resolveSessionMembership` returned `null` — a fetch failure
|
|
4987
|
+
* or pathological chain partway through the parent walk), OR at
|
|
4988
|
+
* least one CONFIRMED descendant's `isSessionOngoing` read
|
|
4989
|
+
* failed — and no OTHER candidate was already confirmed `true`.
|
|
4990
|
+
* The caller MUST NOT treat `null` the same as `false` here
|
|
4991
|
+
* (unlike the recovery cross-check's contract) — see
|
|
4992
|
+
* `isB2AbandonmentConfirmed`.
|
|
4993
|
+
*/
|
|
4994
|
+
async isAnyDescendantSessionOngoing(rootSessionId) {
|
|
4995
|
+
const sessions = await listSessions(this.port);
|
|
4996
|
+
if (!sessions) {
|
|
4997
|
+
this.log({
|
|
4998
|
+
level: "warn",
|
|
4999
|
+
message: `Could not enumerate sessions to check descendant liveness for root ${rootSessionId} (listSessions failed) \u2014 treating descendant liveness as indeterminate`
|
|
5000
|
+
});
|
|
5001
|
+
return null;
|
|
5002
|
+
}
|
|
5003
|
+
let indeterminate = false;
|
|
5004
|
+
for (const candidate of sessions) {
|
|
5005
|
+
if (!candidate?.id || candidate.id === rootSessionId) continue;
|
|
5006
|
+
const membership = await this.resolveSessionMembership(candidate.id, rootSessionId);
|
|
5007
|
+
if (membership === null) {
|
|
5008
|
+
indeterminate = true;
|
|
5009
|
+
continue;
|
|
5010
|
+
}
|
|
5011
|
+
if (membership === false) continue;
|
|
5012
|
+
const ongoing = await isSessionOngoing(this.port, candidate.id);
|
|
5013
|
+
if (ongoing === true) return true;
|
|
5014
|
+
if (ongoing === null) indeterminate = true;
|
|
5015
|
+
}
|
|
5016
|
+
return indeterminate ? null : false;
|
|
5017
|
+
}
|
|
4417
5018
|
/**
|
|
4418
5019
|
* Cheap decision-telemetry label for a running row's LAST correlated reply
|
|
4419
5020
|
* (WI-2 Task 2.2): which running SHAPE it is, for the status-gated recovery log.
|
|
@@ -4677,7 +5278,7 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4677
5278
|
* exists but is wedged, so the next attempt must get a fresh one
|
|
4678
5279
|
* instead of reusing it — see WI-1's server-side null-clearing PATCH).
|
|
4679
5280
|
*/
|
|
4680
|
-
async markFailed(conversationId, messageId, sessionId, error2, usage) {
|
|
5281
|
+
async markFailed(conversationId, messageId, sessionId, error2, usage, failure) {
|
|
4681
5282
|
const body = { status: "failed" };
|
|
4682
5283
|
if (sessionId === null) {
|
|
4683
5284
|
body.opencode_session_id = null;
|
|
@@ -4686,6 +5287,12 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4686
5287
|
}
|
|
4687
5288
|
if (error2 !== void 0) body.error = error2;
|
|
4688
5289
|
if (usage) Object.assign(body, usage);
|
|
5290
|
+
if (failure) {
|
|
5291
|
+
body.failure_kind = failure.kind;
|
|
5292
|
+
body.failure_provider_id = failure.providerId;
|
|
5293
|
+
body.failure_model_id = failure.modelId;
|
|
5294
|
+
body.failure_reason = failure.reason;
|
|
5295
|
+
}
|
|
4689
5296
|
await this.callWithRetry(
|
|
4690
5297
|
"marking message as failed",
|
|
4691
5298
|
() => this.fetchImpl(
|
|
@@ -4698,6 +5305,29 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4698
5305
|
)
|
|
4699
5306
|
);
|
|
4700
5307
|
}
|
|
5308
|
+
/**
|
|
5309
|
+
* Classify an errored turn as a model-auth failure (#736 P1-4), for `markFailed`.
|
|
5310
|
+
*
|
|
5311
|
+
* `messageFailure` alone (structured OpenCode error → `model_auth`) covers
|
|
5312
|
+
* most cases; when it returns `null` on this ALREADY-FAILED turn, fall back
|
|
5313
|
+
* to the P1-2b zero-provider check — one extra loopback call to
|
|
5314
|
+
* `hasAnyConfiguredProvider`, only reached when the structured classifier
|
|
5315
|
+
* couldn't place it. Fails open (never throws): a fallback probe failure
|
|
5316
|
+
* (`null`/indeterminate) leaves the classification `null`, which produces
|
|
5317
|
+
* today's byte-identical PATCH body via `markFailed`'s `if (failure)` guard.
|
|
5318
|
+
*/
|
|
5319
|
+
async classifyModelAuthFailure(messages, userMessageId) {
|
|
5320
|
+
const classified = messageFailure(messages, userMessageId);
|
|
5321
|
+
if (classified != null) return classified;
|
|
5322
|
+
const reply = findLastAssistantReplyFor(messages, userMessageId);
|
|
5323
|
+
const hasProvider = await hasAnyConfiguredProvider(this.port);
|
|
5324
|
+
return applyZeroProviderFallback(
|
|
5325
|
+
classified,
|
|
5326
|
+
hasProvider,
|
|
5327
|
+
reply?.info?.providerID ?? null,
|
|
5328
|
+
reply?.info?.modelID ?? null
|
|
5329
|
+
);
|
|
5330
|
+
}
|
|
4701
5331
|
/**
|
|
4702
5332
|
* Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
|
|
4703
5333
|
* (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
|
|
@@ -4850,10 +5480,16 @@ var ChannelDriver = class _ChannelDriver {
|
|
|
4850
5480
|
import chalk5 from "chalk";
|
|
4851
5481
|
import ora2 from "ora";
|
|
4852
5482
|
import { select as select2 } from "@inquirer/prompts";
|
|
5483
|
+
var INTERACTIVE_START_TIMEOUT_MS = 3e4;
|
|
4853
5484
|
async function ensureOpenCodeRunning(ctx) {
|
|
4854
5485
|
const healthCheck = await checkOpenCodeHealth(ctx.port);
|
|
4855
5486
|
if (healthCheck.healthy) {
|
|
4856
|
-
return {
|
|
5487
|
+
return {
|
|
5488
|
+
port: ctx.port,
|
|
5489
|
+
process: null,
|
|
5490
|
+
version: healthCheck.version ?? null,
|
|
5491
|
+
notReadyReason: null
|
|
5492
|
+
};
|
|
4857
5493
|
}
|
|
4858
5494
|
const runningInstances = await findHealthyOpenCodeInstances();
|
|
4859
5495
|
if (runningInstances.length > 0) {
|
|
@@ -4874,7 +5510,7 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4874
5510
|
console.log(chalk5.yellow("Tip: Run with the correct port:"));
|
|
4875
5511
|
console.log(
|
|
4876
5512
|
chalk5.dim(
|
|
4877
|
-
` ${getCliName()} run --
|
|
5513
|
+
` ${getCliName()} run --runner ${ctx.agentId} --port ${runningInstances[0].port}`
|
|
4878
5514
|
)
|
|
4879
5515
|
);
|
|
4880
5516
|
}
|
|
@@ -4894,14 +5530,22 @@ async function ensureOpenCodeRunning(ctx) {
|
|
|
4894
5530
|
if (!ctx.interactive) {
|
|
4895
5531
|
ctx.log(`OpenCode is not running on port ${ctx.port}. Starting it automatically...`);
|
|
4896
5532
|
const proc = await startOpenCode(ctx.port);
|
|
4897
|
-
const health = await waitForOpenCodeHealth(ctx.port,
|
|
5533
|
+
const health = await waitForOpenCodeHealth(ctx.port, ctx.startTimeoutMs);
|
|
4898
5534
|
if (!health.healthy) {
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
|
|
5535
|
+
return {
|
|
5536
|
+
port: ctx.port,
|
|
5537
|
+
process: proc,
|
|
5538
|
+
version: null,
|
|
5539
|
+
notReadyReason: `it did not become healthy within ${Math.round(ctx.startTimeoutMs / 1e3)}s`
|
|
5540
|
+
};
|
|
4902
5541
|
}
|
|
4903
5542
|
ctx.log(`OpenCode started on port ${ctx.port}${health.version ? ` (v${health.version})` : ""}`);
|
|
4904
|
-
return {
|
|
5543
|
+
return {
|
|
5544
|
+
port: ctx.port,
|
|
5545
|
+
process: proc,
|
|
5546
|
+
version: health.version ?? null,
|
|
5547
|
+
notReadyReason: null
|
|
5548
|
+
};
|
|
4905
5549
|
}
|
|
4906
5550
|
let port = ctx.port;
|
|
4907
5551
|
if (isPortInUse(port)) {
|
|
@@ -4954,15 +5598,15 @@ Port ${port} is already in use.`));
|
|
|
4954
5598
|
if (action === "start") {
|
|
4955
5599
|
const spinner = ora2("Starting OpenCode...").start();
|
|
4956
5600
|
const proc = await startOpenCode(port);
|
|
4957
|
-
const health = await waitForOpenCodeHealth(port,
|
|
5601
|
+
const health = await waitForOpenCodeHealth(port, INTERACTIVE_START_TIMEOUT_MS);
|
|
4958
5602
|
if (!health.healthy) {
|
|
4959
5603
|
spinner.fail("Failed to start OpenCode");
|
|
4960
5604
|
throw new Error("OpenCode failed to start");
|
|
4961
5605
|
}
|
|
4962
5606
|
spinner.stop();
|
|
4963
|
-
return { port, process: proc, version: health.version ?? null };
|
|
5607
|
+
return { port, process: proc, version: health.version ?? null, notReadyReason: null };
|
|
4964
5608
|
}
|
|
4965
|
-
return { port, process: null, version: null };
|
|
5609
|
+
return { port, process: null, version: null, notReadyReason: "you chose to continue without it" };
|
|
4966
5610
|
}
|
|
4967
5611
|
|
|
4968
5612
|
// src/commands/agent-lookup.ts
|
|
@@ -5004,19 +5648,21 @@ async function resolveAgentIdFromKey(authHeader) {
|
|
|
5004
5648
|
return { agent_id: data.agent_id };
|
|
5005
5649
|
}
|
|
5006
5650
|
return {
|
|
5007
|
-
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --
|
|
5651
|
+
error: "Cannot resolve runner ID: auth type is not agent_key. Please provide --runner explicitly."
|
|
5008
5652
|
};
|
|
5009
5653
|
} catch (error2) {
|
|
5010
5654
|
const message = error2 instanceof Error ? error2.message : "Unknown error";
|
|
5011
5655
|
return { error: `Failed to resolve runner from key: ${message}` };
|
|
5012
5656
|
}
|
|
5013
5657
|
}
|
|
5658
|
+
var BEST_EFFORT_NOTIFY_TIMEOUT_MS = 2e3;
|
|
5014
5659
|
async function notifyAgentDisconnected(agentId, authHeader) {
|
|
5015
5660
|
const apiUrl = getApiUrlConfig();
|
|
5016
5661
|
try {
|
|
5017
5662
|
const response = await fetch(`${apiUrl}/runners/${agentId}/disconnect`, {
|
|
5018
5663
|
method: "POST",
|
|
5019
|
-
headers: { Authorization: authHeader }
|
|
5664
|
+
headers: { Authorization: authHeader },
|
|
5665
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5020
5666
|
});
|
|
5021
5667
|
if (!response.ok) {
|
|
5022
5668
|
const serverMessage = await readErrorMessage(response);
|
|
@@ -5027,7 +5673,63 @@ async function notifyAgentDisconnected(agentId, authHeader) {
|
|
|
5027
5673
|
}
|
|
5028
5674
|
return { ok: true };
|
|
5029
5675
|
} catch (error2) {
|
|
5030
|
-
return { ok: false, error:
|
|
5676
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5677
|
+
}
|
|
5678
|
+
}
|
|
5679
|
+
function describeBestEffortError(error2) {
|
|
5680
|
+
const name = error2?.name;
|
|
5681
|
+
if (name === "TimeoutError" || name === "AbortError") {
|
|
5682
|
+
return `timed out after ${BEST_EFFORT_NOTIFY_TIMEOUT_MS}ms`;
|
|
5683
|
+
}
|
|
5684
|
+
return error2 instanceof Error ? error2.message : String(error2);
|
|
5685
|
+
}
|
|
5686
|
+
async function reportMicrovmId(agentId, authHeader, microvmId) {
|
|
5687
|
+
try {
|
|
5688
|
+
const apiUrl = getApiUrlConfig();
|
|
5689
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/microvm`, {
|
|
5690
|
+
method: "POST",
|
|
5691
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
5692
|
+
body: JSON.stringify({ microvm_id: microvmId }),
|
|
5693
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5694
|
+
});
|
|
5695
|
+
if (!response.ok) {
|
|
5696
|
+
const serverMessage = await readErrorMessage(response);
|
|
5697
|
+
return {
|
|
5698
|
+
ok: false,
|
|
5699
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5700
|
+
};
|
|
5701
|
+
}
|
|
5702
|
+
return { ok: true };
|
|
5703
|
+
} catch (error2) {
|
|
5704
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5705
|
+
}
|
|
5706
|
+
}
|
|
5707
|
+
function toReportedWindow(window) {
|
|
5708
|
+
if (!window) return null;
|
|
5709
|
+
return { utilization: window.utilization, resets_at: window.resetsAt };
|
|
5710
|
+
}
|
|
5711
|
+
async function reportClaudeUsage(agentId, authHeader, snapshot) {
|
|
5712
|
+
try {
|
|
5713
|
+
const apiUrl = getApiUrlConfig();
|
|
5714
|
+
const response = await fetch(`${apiUrl}/runners/${agentId}/claude-usage`, {
|
|
5715
|
+
method: "POST",
|
|
5716
|
+
headers: { Authorization: authHeader, "Content-Type": "application/json" },
|
|
5717
|
+
body: JSON.stringify({
|
|
5718
|
+
five_hour: toReportedWindow(snapshot.fiveHour),
|
|
5719
|
+
seven_day: toReportedWindow(snapshot.sevenDay)
|
|
5720
|
+
}),
|
|
5721
|
+
signal: AbortSignal.timeout(BEST_EFFORT_NOTIFY_TIMEOUT_MS)
|
|
5722
|
+
});
|
|
5723
|
+
if (!response.ok) {
|
|
5724
|
+
const serverMessage = await readErrorMessage(response);
|
|
5725
|
+
return {
|
|
5726
|
+
ok: false,
|
|
5727
|
+
error: `HTTP ${response.status}${serverMessage ? `: ${serverMessage}` : ""}`
|
|
5728
|
+
};
|
|
5729
|
+
}
|
|
5730
|
+
return { ok: true };
|
|
5731
|
+
} catch (error2) {
|
|
5732
|
+
return { ok: false, error: describeBestEffortError(error2) };
|
|
5031
5733
|
}
|
|
5032
5734
|
}
|
|
5033
5735
|
async function getAgentInfo(agentId, authHeader) {
|
|
@@ -5077,6 +5779,7 @@ var MAX_ACTIVITY_LOG_ENTRIES = 10;
|
|
|
5077
5779
|
var CHANNEL_POLL_INTERVAL_MS = Number(process.env.EVIDENT_CHANNEL_POLL_INTERVAL_MS) || 2e3;
|
|
5078
5780
|
var CHANNEL_STUCK_QUEUED_MS = Number(process.env.EVIDENT_STUCK_QUEUED_MS) || void 0;
|
|
5079
5781
|
var SHUTDOWN_DRAIN_TIMEOUT_MS = Number(process.env.EVIDENT_SHUTDOWN_DRAIN_MS) || 25e3;
|
|
5782
|
+
var TELEMETRY_SHUTDOWN_TIMEOUT_MS = 5e3;
|
|
5080
5783
|
function resolveLogLevel(options) {
|
|
5081
5784
|
const accepted = Object.keys(LOG_LEVELS);
|
|
5082
5785
|
const validate = (value, source) => {
|
|
@@ -5128,6 +5831,35 @@ function resolveFileSyncDirectories(raw, homeDir) {
|
|
|
5128
5831
|
}
|
|
5129
5832
|
return directories;
|
|
5130
5833
|
}
|
|
5834
|
+
var DEFAULT_OPENCODE_START_TIMEOUT_SECONDS = 180;
|
|
5835
|
+
var MAX_OPENCODE_START_TIMEOUT_SECONDS = 3600;
|
|
5836
|
+
var OPENCODE_START_TIMEOUT_ENV = "EVIDENT_OPENCODE_START_TIMEOUT";
|
|
5837
|
+
function resolveOpenCodeStartTimeoutMs(options, env = process.env) {
|
|
5838
|
+
const defaultMs = DEFAULT_OPENCODE_START_TIMEOUT_SECONDS * 1e3;
|
|
5839
|
+
let raw;
|
|
5840
|
+
let source;
|
|
5841
|
+
if (options.opencodeStartTimeout !== void 0) {
|
|
5842
|
+
raw = options.opencodeStartTimeout;
|
|
5843
|
+
source = "--opencode-start-timeout";
|
|
5844
|
+
} else if (env[OPENCODE_START_TIMEOUT_ENV] !== void 0 && env[OPENCODE_START_TIMEOUT_ENV] !== "") {
|
|
5845
|
+
raw = env[OPENCODE_START_TIMEOUT_ENV];
|
|
5846
|
+
source = OPENCODE_START_TIMEOUT_ENV;
|
|
5847
|
+
} else {
|
|
5848
|
+
return { timeoutMs: defaultMs, warnings: [] };
|
|
5849
|
+
}
|
|
5850
|
+
const trimmed = raw.trim();
|
|
5851
|
+
const seconds = Number(trimmed);
|
|
5852
|
+
const isPositiveInteger = /^\d+$/.test(trimmed) && Number.isInteger(seconds) && seconds > 0;
|
|
5853
|
+
if (!isPositiveInteger || seconds > MAX_OPENCODE_START_TIMEOUT_SECONDS) {
|
|
5854
|
+
return {
|
|
5855
|
+
timeoutMs: defaultMs,
|
|
5856
|
+
warnings: [
|
|
5857
|
+
`Ignoring invalid ${source} "${raw}": expected a positive integer number of seconds (at most ${MAX_OPENCODE_START_TIMEOUT_SECONDS}); using the default ${DEFAULT_OPENCODE_START_TIMEOUT_SECONDS}s`
|
|
5858
|
+
]
|
|
5859
|
+
};
|
|
5860
|
+
}
|
|
5861
|
+
return { timeoutMs: seconds * 1e3, warnings: [] };
|
|
5862
|
+
}
|
|
5131
5863
|
function meetsThreshold(state, level) {
|
|
5132
5864
|
return LOG_LEVELS[level] >= LOG_LEVELS[state.logLevel];
|
|
5133
5865
|
}
|
|
@@ -5149,6 +5881,10 @@ function log2(state, message, level = "info") {
|
|
|
5149
5881
|
function logActivity(state, entry) {
|
|
5150
5882
|
const level = entry.level ?? (entry.type === "error" ? "error" : "info");
|
|
5151
5883
|
if (!meetsThreshold(state, level)) return;
|
|
5884
|
+
forwardRunnerActivity(
|
|
5885
|
+
{ level, message: entry.message, error: entry.error },
|
|
5886
|
+
{ agentId: state.agentId, authHeader: state.authHeader }
|
|
5887
|
+
);
|
|
5152
5888
|
const fullEntry = {
|
|
5153
5889
|
...entry,
|
|
5154
5890
|
level,
|
|
@@ -5312,8 +6048,8 @@ async function driveChannels(state, driver) {
|
|
|
5312
6048
|
}
|
|
5313
6049
|
}
|
|
5314
6050
|
var SESSION_CLEANUP_FIRST_SWEEP_MS = 1e4;
|
|
5315
|
-
async function runSweep(state, driver,
|
|
5316
|
-
const mode = `age=${
|
|
6051
|
+
async function runSweep(state, driver, config) {
|
|
6052
|
+
const mode = `age=${config.maxAgeMs ?? "\u2014"} count=${config.maxCount ?? "\u2014"}`;
|
|
5317
6053
|
try {
|
|
5318
6054
|
const sessions = await listSessions(state.port);
|
|
5319
6055
|
if (sessions === null) {
|
|
@@ -5326,8 +6062,8 @@ async function runSweep(state, driver, config2) {
|
|
|
5326
6062
|
const toDelete = selectSessionsToDelete(
|
|
5327
6063
|
sessions.map((s) => ({ id: s.id, lastActivityMs: sessionLastActivityMs(s) })),
|
|
5328
6064
|
{
|
|
5329
|
-
maxAgeMs:
|
|
5330
|
-
maxCount:
|
|
6065
|
+
maxAgeMs: config.maxAgeMs,
|
|
6066
|
+
maxCount: config.maxCount,
|
|
5331
6067
|
nowMs: Date.now(),
|
|
5332
6068
|
protectedIds: driver.protectedSessionIds()
|
|
5333
6069
|
}
|
|
@@ -5363,7 +6099,7 @@ async function runSweep(state, driver, config2) {
|
|
|
5363
6099
|
}
|
|
5364
6100
|
}
|
|
5365
6101
|
function scheduleSessionCleanup(state, driver, options) {
|
|
5366
|
-
const
|
|
6102
|
+
const config = resolveSessionCleanupConfig(
|
|
5367
6103
|
{
|
|
5368
6104
|
maxAge: options.sessionCleanupMaxAge,
|
|
5369
6105
|
maxCount: options.sessionCleanupMaxCount,
|
|
@@ -5371,21 +6107,109 @@ function scheduleSessionCleanup(state, driver, options) {
|
|
|
5371
6107
|
},
|
|
5372
6108
|
process.env
|
|
5373
6109
|
);
|
|
5374
|
-
for (const warning2 of
|
|
6110
|
+
for (const warning2 of config.warnings) {
|
|
5375
6111
|
logActivity(state, { type: "info", level: "warn", message: `Session cleanup: ${warning2}` });
|
|
5376
6112
|
}
|
|
5377
|
-
if (!
|
|
6113
|
+
if (!config.enabled) return;
|
|
5378
6114
|
logActivity(state, {
|
|
5379
6115
|
type: "info",
|
|
5380
|
-
message: `Session cleanup enabled (age=${
|
|
6116
|
+
message: `Session cleanup enabled (age=${config.maxAgeMs ?? "\u2014"}, count=${config.maxCount ?? "\u2014"}, interval=${config.intervalMs}ms)`
|
|
5381
6117
|
});
|
|
5382
|
-
const interval = setInterval(() => void runSweep(state, driver,
|
|
6118
|
+
const interval = setInterval(() => void runSweep(state, driver, config), config.intervalMs);
|
|
5383
6119
|
const firstSweep = setTimeout(
|
|
5384
|
-
() => void runSweep(state, driver,
|
|
6120
|
+
() => void runSweep(state, driver, config),
|
|
5385
6121
|
SESSION_CLEANUP_FIRST_SWEEP_MS
|
|
5386
6122
|
);
|
|
5387
6123
|
state.sessionCleanupTimers.push(interval, firstSweep);
|
|
5388
6124
|
}
|
|
6125
|
+
function scheduleClaudeUsageReporting(state, options) {
|
|
6126
|
+
const { mode, warnings } = resolveClaudeUsageReportingMode(
|
|
6127
|
+
options.claudeUsageReporting,
|
|
6128
|
+
process.env
|
|
6129
|
+
);
|
|
6130
|
+
for (const warning2 of warnings) {
|
|
6131
|
+
logActivity(state, {
|
|
6132
|
+
type: "info",
|
|
6133
|
+
level: "warn",
|
|
6134
|
+
message: `Claude usage reporting: ${warning2}`
|
|
6135
|
+
});
|
|
6136
|
+
}
|
|
6137
|
+
if (mode === "off") {
|
|
6138
|
+
logActivity(state, {
|
|
6139
|
+
type: "info",
|
|
6140
|
+
level: "debug",
|
|
6141
|
+
message: "Claude usage reporting is off (--claude-usage-reporting off)"
|
|
6142
|
+
});
|
|
6143
|
+
return;
|
|
6144
|
+
}
|
|
6145
|
+
let consecutiveFailures = 0;
|
|
6146
|
+
const scheduleNextTick = () => {
|
|
6147
|
+
state.claudeUsageTimer = setTimeout(() => void tick(false), nextReportDelayMs());
|
|
6148
|
+
};
|
|
6149
|
+
const tick = async (isFirst) => {
|
|
6150
|
+
try {
|
|
6151
|
+
const usage = await getClaudeUsage();
|
|
6152
|
+
const result = await reportClaudeUsage(state.agentId, state.authHeader, usage);
|
|
6153
|
+
if (result.ok) {
|
|
6154
|
+
if (consecutiveFailures > 0) {
|
|
6155
|
+
logActivity(state, {
|
|
6156
|
+
type: "info",
|
|
6157
|
+
level: "info",
|
|
6158
|
+
message: "Claude usage reporting recovered"
|
|
6159
|
+
});
|
|
6160
|
+
}
|
|
6161
|
+
consecutiveFailures = 0;
|
|
6162
|
+
logActivity(state, {
|
|
6163
|
+
type: "info",
|
|
6164
|
+
level: "debug",
|
|
6165
|
+
message: "Reported Claude usage to Evident"
|
|
6166
|
+
});
|
|
6167
|
+
} else {
|
|
6168
|
+
consecutiveFailures++;
|
|
6169
|
+
logActivity(state, {
|
|
6170
|
+
type: "info",
|
|
6171
|
+
level: consecutiveFailures === 1 ? "warn" : "debug",
|
|
6172
|
+
message: `Failed to report Claude usage: ${result.error}`
|
|
6173
|
+
});
|
|
6174
|
+
}
|
|
6175
|
+
scheduleNextTick();
|
|
6176
|
+
} catch (error2) {
|
|
6177
|
+
if (error2 instanceof ClaudeUsageError && isLocalCredentialProblem(error2)) {
|
|
6178
|
+
if (mode === "on") {
|
|
6179
|
+
logActivity(state, {
|
|
6180
|
+
type: "info",
|
|
6181
|
+
level: "warn",
|
|
6182
|
+
message: "Claude usage reporting is forced on but no usable Claude Code login was found \u2014 run `claude` to sign in; reporting will keep retrying"
|
|
6183
|
+
});
|
|
6184
|
+
scheduleNextTick();
|
|
6185
|
+
} else if (isFirst) {
|
|
6186
|
+
logActivity(state, {
|
|
6187
|
+
type: "info",
|
|
6188
|
+
level: "debug",
|
|
6189
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
6190
|
+
});
|
|
6191
|
+
} else {
|
|
6192
|
+
logActivity(state, {
|
|
6193
|
+
type: "info",
|
|
6194
|
+
level: "debug",
|
|
6195
|
+
message: `Claude usage reporting: ${error2.message}`
|
|
6196
|
+
});
|
|
6197
|
+
scheduleNextTick();
|
|
6198
|
+
}
|
|
6199
|
+
} else {
|
|
6200
|
+
consecutiveFailures++;
|
|
6201
|
+
const message = error2 instanceof Error ? error2.message : String(error2);
|
|
6202
|
+
logActivity(state, {
|
|
6203
|
+
type: "info",
|
|
6204
|
+
level: consecutiveFailures === 1 ? "warn" : "debug",
|
|
6205
|
+
message: `Claude usage reporting failed: ${message}`
|
|
6206
|
+
});
|
|
6207
|
+
scheduleNextTick();
|
|
6208
|
+
}
|
|
6209
|
+
}
|
|
6210
|
+
};
|
|
6211
|
+
state.claudeUsageTimer = setTimeout(() => void tick(true), FIRST_REPORT_DELAY_MS);
|
|
6212
|
+
}
|
|
5389
6213
|
async function notifyOffline(state) {
|
|
5390
6214
|
if (!state.agentId || !state.authHeader) return;
|
|
5391
6215
|
if (!state.connected) {
|
|
@@ -5403,13 +6227,28 @@ async function notifyOffline(state) {
|
|
|
5403
6227
|
if (state.interactive) displayStatus(state);
|
|
5404
6228
|
}
|
|
5405
6229
|
}
|
|
6230
|
+
async function timeShutdownPhase(state, durations, name, run2) {
|
|
6231
|
+
const startedAt = Date.now();
|
|
6232
|
+
try {
|
|
6233
|
+
return await run2();
|
|
6234
|
+
} finally {
|
|
6235
|
+
const elapsedMs = Date.now() - startedAt;
|
|
6236
|
+
durations[name] = elapsedMs;
|
|
6237
|
+
log2(state, `Shutdown phase ${name}: ${elapsedMs}ms`);
|
|
6238
|
+
}
|
|
6239
|
+
}
|
|
5406
6240
|
async function cleanup(state, opts = {}) {
|
|
6241
|
+
const durations = {};
|
|
5407
6242
|
state.running = false;
|
|
5408
6243
|
for (const timer of state.sessionCleanupTimers) {
|
|
5409
6244
|
clearInterval(timer);
|
|
5410
6245
|
clearTimeout(timer);
|
|
5411
6246
|
}
|
|
5412
6247
|
state.sessionCleanupTimers = [];
|
|
6248
|
+
if (state.claudeUsageTimer) {
|
|
6249
|
+
clearTimeout(state.claudeUsageTimer);
|
|
6250
|
+
state.claudeUsageTimer = null;
|
|
6251
|
+
}
|
|
5413
6252
|
if (opts.graceful && state.channelDriver) {
|
|
5414
6253
|
state.channelDriver.stop();
|
|
5415
6254
|
log2(state, "Draining in-flight channel work before shutdown...");
|
|
@@ -5417,7 +6256,13 @@ async function cleanup(state, opts = {}) {
|
|
|
5417
6256
|
logActivity(state, { type: "info", message: "Draining in-flight work before shutdown..." });
|
|
5418
6257
|
displayStatus(state);
|
|
5419
6258
|
}
|
|
5420
|
-
const
|
|
6259
|
+
const driver = state.channelDriver;
|
|
6260
|
+
const settled = await timeShutdownPhase(
|
|
6261
|
+
state,
|
|
6262
|
+
durations,
|
|
6263
|
+
"drain",
|
|
6264
|
+
() => driver.waitForInFlight(SHUTDOWN_DRAIN_TIMEOUT_MS)
|
|
6265
|
+
);
|
|
5421
6266
|
if (!settled) {
|
|
5422
6267
|
logActivity(state, {
|
|
5423
6268
|
type: "info",
|
|
@@ -5426,13 +6271,15 @@ async function cleanup(state, opts = {}) {
|
|
|
5426
6271
|
if (state.interactive) displayStatus(state);
|
|
5427
6272
|
}
|
|
5428
6273
|
}
|
|
5429
|
-
await notifyOffline(state);
|
|
6274
|
+
await timeShutdownPhase(state, durations, "offline_notify", () => notifyOffline(state));
|
|
5430
6275
|
if (state.connection) {
|
|
5431
|
-
state.connection
|
|
6276
|
+
const connection = state.connection;
|
|
6277
|
+
await timeShutdownPhase(state, durations, "tunnel_close", () => connection.close());
|
|
5432
6278
|
state.connection = null;
|
|
5433
6279
|
}
|
|
5434
6280
|
if (state.opencodeProcess) {
|
|
5435
|
-
|
|
6281
|
+
const opencodeProcess = state.opencodeProcess;
|
|
6282
|
+
await timeShutdownPhase(state, durations, "opencode_stop", () => stopOpenCode(opencodeProcess));
|
|
5436
6283
|
if (state.interactive) {
|
|
5437
6284
|
logActivity(state, { type: "info", message: "Stopped OpenCode process" });
|
|
5438
6285
|
displayStatus(state);
|
|
@@ -5441,6 +6288,7 @@ async function cleanup(state, opts = {}) {
|
|
|
5441
6288
|
}
|
|
5442
6289
|
state.opencodeProcess = null;
|
|
5443
6290
|
}
|
|
6291
|
+
return durations;
|
|
5444
6292
|
}
|
|
5445
6293
|
async function run(options) {
|
|
5446
6294
|
const interactive = isInteractive(options.json);
|
|
@@ -5481,8 +6329,10 @@ async function run(options) {
|
|
|
5481
6329
|
messageCount: 0,
|
|
5482
6330
|
lastProxiedActivityAt: null,
|
|
5483
6331
|
sessionCleanupTimers: [],
|
|
6332
|
+
claudeUsageTimer: null,
|
|
5484
6333
|
authHeader: ""
|
|
5485
6334
|
};
|
|
6335
|
+
setTelemetryAuthProvider(() => ({ authHeader: state.authHeader, agentId: state.agentId }));
|
|
5486
6336
|
if (fileSyncDirectories.length > 0) {
|
|
5487
6337
|
log2(state, `File sync enabled for: ${fileSyncDirectories.join(", ")}`);
|
|
5488
6338
|
} else {
|
|
@@ -5511,14 +6361,38 @@ async function run(options) {
|
|
|
5511
6361
|
const handleSignal = async () => {
|
|
5512
6362
|
if (state.shuttingDown) return;
|
|
5513
6363
|
state.shuttingDown = true;
|
|
6364
|
+
const shutdownStartedAt = Date.now();
|
|
5514
6365
|
if (state.interactive) {
|
|
5515
6366
|
logActivity(state, { type: "info", message: "Shutting down..." });
|
|
5516
6367
|
displayStatus(state);
|
|
5517
6368
|
} else {
|
|
5518
6369
|
log2(state, "Shutting down...");
|
|
5519
6370
|
}
|
|
5520
|
-
await cleanup(state, { graceful: true });
|
|
5521
|
-
|
|
6371
|
+
const durations = await cleanup(state, { graceful: true });
|
|
6372
|
+
const telemetryBudgetMs = Number(process.env.EVIDENT_TELEMETRY_SHUTDOWN_MS) || TELEMETRY_SHUTDOWN_TIMEOUT_MS;
|
|
6373
|
+
await timeShutdownPhase(state, durations, "telemetry_flush", async () => {
|
|
6374
|
+
let timer;
|
|
6375
|
+
const flushed = shutdownTelemetry().then(
|
|
6376
|
+
() => true,
|
|
6377
|
+
(error2) => {
|
|
6378
|
+
log2(
|
|
6379
|
+
state,
|
|
6380
|
+
`Telemetry flush failed during shutdown: ${error2 instanceof Error ? error2.message : String(error2)}`,
|
|
6381
|
+
"warn"
|
|
6382
|
+
);
|
|
6383
|
+
return true;
|
|
6384
|
+
}
|
|
6385
|
+
);
|
|
6386
|
+
const timedOut = new Promise((resolve3) => {
|
|
6387
|
+
timer = setTimeout(() => resolve3(false), telemetryBudgetMs);
|
|
6388
|
+
});
|
|
6389
|
+
if (!await Promise.race([flushed, timedOut])) {
|
|
6390
|
+
log2(state, `Telemetry flush exceeded ${telemetryBudgetMs}ms \u2014 exiting anyway`, "warn");
|
|
6391
|
+
}
|
|
6392
|
+
clearTimeout(timer);
|
|
6393
|
+
});
|
|
6394
|
+
const breakdown = Object.entries(durations).map(([phase, ms]) => `${phase}=${ms}ms`).join(" ");
|
|
6395
|
+
log2(state, `Shutdown complete in ${Date.now() - shutdownStartedAt}ms (${breakdown})`);
|
|
5522
6396
|
process.exit(0);
|
|
5523
6397
|
};
|
|
5524
6398
|
process.on("SIGINT", handleSignal);
|
|
@@ -5632,40 +6506,67 @@ async function run(options) {
|
|
|
5632
6506
|
}
|
|
5633
6507
|
spinner?.succeed(`Runner: ${validation.agent.name || state.agentId}`);
|
|
5634
6508
|
state.agentName = validation.agent.name;
|
|
6509
|
+
const microvmId = process.env.MICROVM_ID?.trim();
|
|
6510
|
+
if (microvmId) {
|
|
6511
|
+
const reported = await reportMicrovmId(state.agentId, state.authHeader, microvmId);
|
|
6512
|
+
if (reported.ok) {
|
|
6513
|
+
log2(state, "Reported MicroVM identity so this runner can be resumed rather than restarted");
|
|
6514
|
+
} else {
|
|
6515
|
+
const message = `Could not report MicroVM identity (future wakes will cold-start): ${reported.error}`;
|
|
6516
|
+
log2(state, message, "warn");
|
|
6517
|
+
if (state.interactive && !state.json) {
|
|
6518
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6519
|
+
}
|
|
6520
|
+
}
|
|
6521
|
+
} else {
|
|
6522
|
+
log2(state, "Not running in a MicroVM (MICROVM_ID unset) \u2014 nothing to report", "debug");
|
|
6523
|
+
}
|
|
6524
|
+
const { timeoutMs: opencodeStartTimeoutMs, warnings: opencodeStartTimeoutWarnings } = resolveOpenCodeStartTimeoutMs(options, process.env);
|
|
6525
|
+
for (const warning2 of opencodeStartTimeoutWarnings) {
|
|
6526
|
+
logActivity(state, { type: "info", level: "warn", message: warning2 });
|
|
6527
|
+
}
|
|
5635
6528
|
const ocSpinner = interactive && !state.json ? ora3("Checking OpenCode...").start() : null;
|
|
5636
6529
|
try {
|
|
5637
6530
|
const oc = await ensureOpenCodeRunning({
|
|
5638
6531
|
port: state.port,
|
|
5639
6532
|
interactive: state.interactive,
|
|
5640
6533
|
agentId: state.agentId,
|
|
5641
|
-
log: (message) => log2(state, message)
|
|
6534
|
+
log: (message) => log2(state, message),
|
|
6535
|
+
startTimeoutMs: opencodeStartTimeoutMs
|
|
5642
6536
|
});
|
|
5643
6537
|
state.port = oc.port;
|
|
5644
6538
|
state.opencodeProcess = oc.process;
|
|
5645
6539
|
state.opencodeVersion = oc.version;
|
|
5646
|
-
state.opencodeConnected = oc.
|
|
6540
|
+
state.opencodeConnected = oc.notReadyReason === null;
|
|
5647
6541
|
const version2 = state.opencodeVersion ? ` (v${state.opencodeVersion})` : "";
|
|
5648
6542
|
ocSpinner?.succeed(`OpenCode running on port ${state.port}${version2}`);
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
6543
|
+
if (!state.interactive && oc.notReadyReason !== null) {
|
|
6544
|
+
const message = `OpenCode is not ready on port ${state.port}: ${oc.notReadyReason}. The runner will still come online, but messages will fail until opencode answers \u2014 raise the wait with --opencode-start-timeout <seconds> (env ${OPENCODE_START_TIMEOUT_ENV}).`;
|
|
6545
|
+
logActivity(state, { type: "info", level: "warn", message });
|
|
6546
|
+
} else {
|
|
6547
|
+
const versionWarning = buildOpenCodeVersionWarning(state.opencodeVersion);
|
|
6548
|
+
if (versionWarning) {
|
|
6549
|
+
log2(state, versionWarning, "warn");
|
|
6550
|
+
if (state.interactive && !state.json) {
|
|
6551
|
+
logActivity(state, { type: "info", level: "warn", message: versionWarning });
|
|
6552
|
+
}
|
|
5654
6553
|
}
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5662
|
-
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
5668
|
-
|
|
6554
|
+
const noProviderWarning = buildNoProviderWarning(
|
|
6555
|
+
await hasAnyConfiguredProvider(state.port)
|
|
6556
|
+
);
|
|
6557
|
+
if (noProviderWarning) {
|
|
6558
|
+
log2(state, noProviderWarning, "warn");
|
|
6559
|
+
if (state.interactive && !state.json) {
|
|
6560
|
+
logActivity(state, { type: "info", level: "warn", message: noProviderWarning });
|
|
6561
|
+
blank();
|
|
6562
|
+
console.log(chalk6.yellow("\u26A0 No OpenCode model provider is configured."));
|
|
6563
|
+
console.log(
|
|
6564
|
+
chalk6.dim(
|
|
6565
|
+
`Run ${chalk6.cyan("opencode auth login")} to set one up \u2014 messages will fail until then.`
|
|
6566
|
+
)
|
|
6567
|
+
);
|
|
6568
|
+
blank();
|
|
6569
|
+
}
|
|
5669
6570
|
}
|
|
5670
6571
|
}
|
|
5671
6572
|
} catch (error2) {
|
|
@@ -5710,6 +6611,18 @@ async function run(options) {
|
|
|
5710
6611
|
type: "info",
|
|
5711
6612
|
message: `Tunnel ${isReconnect ? "reconnected" : "connected"} (runner: ${agentId})`
|
|
5712
6613
|
});
|
|
6614
|
+
if (options.tunnelReadyFile) {
|
|
6615
|
+
const marker = writeTunnelReadyMarker(options.tunnelReadyFile, agentId);
|
|
6616
|
+
if (marker.ok) {
|
|
6617
|
+
log2(state, `Wrote tunnel readiness marker to ${options.tunnelReadyFile}`, "debug");
|
|
6618
|
+
} else {
|
|
6619
|
+
log2(
|
|
6620
|
+
state,
|
|
6621
|
+
`Failed to write tunnel readiness marker to ${options.tunnelReadyFile}: ${marker.error}`,
|
|
6622
|
+
"error"
|
|
6623
|
+
);
|
|
6624
|
+
}
|
|
6625
|
+
}
|
|
5713
6626
|
emitAgentConnected(state.agentId, {
|
|
5714
6627
|
port: state.port,
|
|
5715
6628
|
cli_version: getCliVersion(),
|
|
@@ -5800,6 +6713,7 @@ async function run(options) {
|
|
|
5800
6713
|
throw error2;
|
|
5801
6714
|
}
|
|
5802
6715
|
scheduleSessionCleanup(state, channelDriver, options);
|
|
6716
|
+
scheduleClaudeUsageReporting(state, options);
|
|
5803
6717
|
if (!interactive || state.json) {
|
|
5804
6718
|
log2(state, "Driving channel messages...");
|
|
5805
6719
|
}
|
|
@@ -5854,13 +6768,17 @@ program.name("evident").description("Run OpenCode locally and connect it to Evid
|
|
|
5854
6768
|
program.command("login").description("Authenticate with Evident").option("--token", "Use token-based authentication (for CI/CD)").option("--no-browser", "Do not open the browser automatically").action(login);
|
|
5855
6769
|
program.command("logout").description("Remove stored credentials for the current endpoint").option("--all", "Remove stored credentials for all endpoints").action((options) => logout({ all: options.all }));
|
|
5856
6770
|
program.command("whoami").description("Show the currently logged in user").action(whoami);
|
|
6771
|
+
program.command("claude-usage").description("[spike] Show Claude subscription usage (requires a local `claude login`)").action(claudeUsage);
|
|
5857
6772
|
program.command("run").description("Connect to Evident and process messages").option("--runner [id]", "Runner ID to connect to (optional when EVIDENT_RUNNER_KEY is set)").option(
|
|
5858
6773
|
"-a, --agent [id]",
|
|
5859
6774
|
"Deprecated alias for --runner (still supported; --runner wins if both are given)"
|
|
5860
6775
|
).option("-p, --port <port>", "OpenCode port (default: 4096)", "4096").option(
|
|
5861
6776
|
"--log-level <level>",
|
|
5862
6777
|
"Log verbosity: debug | info | warn | error (default: info). Env: EVIDENT_LOG_LEVEL"
|
|
5863
|
-
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
6778
|
+
).option("-v, --verbose", "Alias for --log-level debug (ignored if --log-level is set)").option("-c, --conversation <id>", "Process only this specific conversation").option("--idle-timeout <seconds>", "Exit after N seconds idle").option(
|
|
6779
|
+
"--opencode-start-timeout <seconds>",
|
|
6780
|
+
"Seconds to wait for OpenCode to become healthy when the runner starts it (default: 180). Env: EVIDENT_OPENCODE_START_TIMEOUT"
|
|
6781
|
+
).option("--json", "Output in JSON format").option(
|
|
5864
6782
|
"--session-cleanup-max-age <duration>",
|
|
5865
6783
|
"Delete OpenCode sessions idle longer than this (e.g. 7d, 24h). Enables cleanup. Env: EVIDENT_SESSION_CLEANUP_MAX_AGE"
|
|
5866
6784
|
).option(
|
|
@@ -5869,11 +6787,17 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5869
6787
|
).option(
|
|
5870
6788
|
"--session-cleanup-interval <duration>",
|
|
5871
6789
|
"How often the cleanup sweep runs (default: 1h). Env: EVIDENT_SESSION_CLEANUP_INTERVAL"
|
|
6790
|
+
).option(
|
|
6791
|
+
"--claude-usage-reporting <mode>",
|
|
6792
|
+
"Report Claude subscription usage to Evident: auto | on | off (default: auto). Env: EVIDENT_CLAUDE_USAGE_REPORTING"
|
|
5872
6793
|
).option(
|
|
5873
6794
|
"--enable-file-sync-to <dir>",
|
|
5874
6795
|
"Allow Evident to write files into this directory (repeatable). Omit to disable file sync entirely.",
|
|
5875
6796
|
(value, previous) => previous.concat([value]),
|
|
5876
6797
|
[]
|
|
6798
|
+
).option(
|
|
6799
|
+
"--tunnel-ready-file <path>",
|
|
6800
|
+
"Path to write once the tunnel is connected (set by the MicroVM hooks; unused on a developer machine)"
|
|
5877
6801
|
).action(
|
|
5878
6802
|
(options) => {
|
|
5879
6803
|
run({
|
|
@@ -5886,14 +6810,21 @@ program.command("run").description("Connect to Evident and process messages").op
|
|
|
5886
6810
|
verbose: options.verbose,
|
|
5887
6811
|
conversation: options.conversation,
|
|
5888
6812
|
idleTimeout: options.idleTimeout ? parseInt(options.idleTimeout, 10) : void 0,
|
|
6813
|
+
// Raw string — validation/precedence is single-sourced in run.ts's
|
|
6814
|
+
// resolveOpenCodeStartTimeoutMs (flag > EVIDENT_OPENCODE_START_TIMEOUT > 180s default).
|
|
6815
|
+
opencodeStartTimeout: options.opencodeStartTimeout,
|
|
5889
6816
|
json: options.json,
|
|
5890
6817
|
// Raw strings — the resolver in run.ts single-sources parsing (M1).
|
|
5891
6818
|
sessionCleanupMaxAge: options.sessionCleanupMaxAge,
|
|
5892
6819
|
sessionCleanupMaxCount: options.sessionCleanupMaxCount,
|
|
5893
6820
|
sessionCleanupInterval: options.sessionCleanupInterval,
|
|
6821
|
+
// Raw string — the resolver in run.ts single-sources parsing
|
|
6822
|
+
// (resolveClaudeUsageReportingMode).
|
|
6823
|
+
claudeUsageReporting: options.claudeUsageReporting,
|
|
5894
6824
|
// Raw values — expansion/validation is single-sourced in run.ts's
|
|
5895
6825
|
// resolveFileSyncDirectories.
|
|
5896
|
-
enableFileSyncTo: options.enableFileSyncTo
|
|
6826
|
+
enableFileSyncTo: options.enableFileSyncTo,
|
|
6827
|
+
tunnelReadyFile: options.tunnelReadyFile
|
|
5897
6828
|
});
|
|
5898
6829
|
}
|
|
5899
6830
|
);
|