@garuda-agent/garuda 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +1 -1
- package/dist/{acpCommand-JAYNMIMP.js → acpCommand-SIFV5XBS.js} +12 -13
- package/dist/advise-EA7OIYSV.js +433 -0
- package/dist/{anthropic-NQIZLD64.js → anthropic-6DKONNQG.js} +4 -2
- package/dist/{buildCheck-HB46NWLX.js → buildCheck-JNWGER5R.js} +4 -4
- package/dist/buildInfo-APQVLWI7.js +8 -0
- package/dist/{chunk-TWQ7V2L6.js → chunk-2THG7PEC.js} +2 -2
- package/dist/chunk-2TX3KY4M.js +88 -0
- package/dist/{chunk-TLA2PDGM.js → chunk-4BELTWP4.js} +5 -5
- package/dist/{chunk-QXTHEFKP.js → chunk-5GF2VINR.js} +2 -2
- package/dist/{chunk-JMH5W5DG.js → chunk-65NUEOI7.js} +44 -1
- package/dist/{chunk-WK5NE6RY.js → chunk-A4SSBJ2P.js} +2 -2
- package/dist/{chunk-DQSCCE77.js → chunk-AX3A5KPH.js} +5 -5
- package/dist/{chunk-5UDYQ5MK.js → chunk-BFK432BK.js} +12 -14
- package/dist/{chunk-GKU62MHQ.js → chunk-CHAWBYB4.js} +2 -2
- package/dist/chunk-CKVF67VJ.js +1780 -0
- package/dist/{chunk-IKPGHM54.js → chunk-DRAZNXNP.js} +3 -3
- package/dist/{chunk-ZKT4P2PC.js → chunk-FALYVAFB.js} +2 -2
- package/dist/chunk-FBCVU2P2.js +9 -0
- package/dist/{chunk-LMKGSYJC.js → chunk-L5TZTKUV.js} +3 -1
- package/dist/{chunk-34WBDQBL.js → chunk-MTJT3D3X.js} +19 -19
- package/dist/{chunk-QLBCM2YQ.js → chunk-QE7E77IR.js} +4 -4
- package/dist/chunk-U6Z4WH3B.js +9 -0
- package/dist/{chunk-R7RCW4BQ.js → chunk-WXNIMQOA.js} +5 -5
- package/dist/{chunk-YAQ6QLKB.js → chunk-YIU5QGMV.js} +3 -3
- package/dist/cli/index.js +50 -51
- package/dist/{create-OACSGJ37.js → create-GHGJR3AY.js} +5 -5
- package/dist/{evalCommand-BLJJ4G76.js → evalCommand-GQWNI2YJ.js} +73 -45
- package/dist/{evalTune-W53KAD4V.js → evalTune-LKEIFX7F.js} +20 -21
- package/dist/{export-E677HIQS.js → export-ZPF3AMO6.js} +5 -5
- package/dist/{exportHtml-BMV73YRO.js → exportHtml-ISISG5HG.js} +4 -4
- package/dist/{inkChat-EVLYDC3K.js → inkChat-3YNJC7KE.js} +12 -13
- package/dist/{install-4ST5BNRG.js → install-BQVAOQHD.js} +3 -3
- package/dist/{jobCommand-ILOTYOX3.js → jobCommand-WPHZYBXM.js} +6 -6
- package/dist/{launchd-QG6R5RAK.js → launchd-JZ6Y6JIH.js} +4 -4
- package/dist/{lspCommand-X4FTTONR.js → lspCommand-4LYQRYXL.js} +3 -3
- package/dist/{manager-L25IUXHW.js → manager-7MA6APCK.js} +3 -3
- package/dist/{nightCommand-XPWT6CQZ.js → nightCommand-RQKQGDTB.js} +6 -6
- package/dist/{remove-STX5UO7X.js → remove-S45S7ICN.js} +3 -3
- package/dist/runner-EUXA5UCH.js +97 -0
- package/dist/{setupCommand-OSOMFHDQ.js → setupCommand-YHDRJTPZ.js} +8 -9
- package/dist/{tune-VX2WQTHS.js → tune-XPBOEWU6.js} +12 -13
- package/package.json +5 -5
- package/dist/advise-HDKJGNBL.js +0 -195
- package/dist/buildInfo-K73FFI5S.js +0 -8
- package/dist/chunk-CWVXOSAA.js +0 -894
- package/dist/chunk-DUYLVGPY.js +0 -821
- package/dist/chunk-H7LIBGZT.js +0 -9
- package/dist/chunk-IYGJWX65.js +0 -9
- package/dist/chunk-NH72AELP.js +0 -48
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,78 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.19.0 (2026-11-06)
|
|
4
|
+
|
|
5
|
+
A security fix and the dependency majors. Sandboxed commands can no longer read the keys that the
|
|
6
|
+
`ant` CLI stores. The Anthropic SDK 1.x moves Garuda's requests to API version `2026-10-21` and
|
|
7
|
+
sends the key as `Authorization: Bearer` instead of `X-Api-Key`. Most setups notice nothing. If
|
|
8
|
+
your requests go through a gateway or proxy (for example with `ANTHROPIC_BASE_URL`) that reads
|
|
9
|
+
`X-Api-Key`, change it to read `Authorization`: the API takes either. This change is why the
|
|
10
|
+
version is 0.19.0, not 0.18.3.
|
|
11
|
+
|
|
12
|
+
- Security: sandboxed commands can no longer read `~/.config/anthropic/credentials`, where the `ant`
|
|
13
|
+
CLI stores keys. The SDK 1.x reads them too when it has no key.
|
|
14
|
+
- The Homebrew formula in the repository names 0.18.2 and its tarball hash.
|
|
15
|
+
- The Anthropic SDK is 1.6 (was 0.128). It uses API version `2026-10-21` and sends the key as
|
|
16
|
+
`Authorization: Bearer`. Garuda's requests needed no change: they send no `temperature`,
|
|
17
|
+
`top_p` or `top_k`, which that version refuses. A proxy in front of the API that reads
|
|
18
|
+
`X-Api-Key` must read `Authorization` now.
|
|
19
|
+
- undici is 8.12 (was 7.30). Its HTTP/2 default stays off for model requests: Garuda's
|
|
20
|
+
20-minute wait for quiet streams was checked on HTTP/1.1. undici 8 needs Node.js 22.19 or newer;
|
|
21
|
+
Garuda already needs 22.23.4.
|
|
22
|
+
- The ACP SDK is 2.4 (was 1.7). Garuda speaks the same protocol, ACP version 1, and uses only its
|
|
23
|
+
stable parts. The SDK now refuses `session/new` before `initialize`, as the protocol says;
|
|
24
|
+
editors always send `initialize` first.
|
|
25
|
+
|
|
26
|
+
## 0.18.2 (2026-11-06)
|
|
27
|
+
|
|
28
|
+
A fix release for repo evals (`garuda eval -s repo`, `garuda advise`). On macOS, 0.18.1 failed
|
|
29
|
+
every check of a project whose tools live in `~/Library` (pnpm does), and Vite projects failed
|
|
30
|
+
before a test ran. It also makes `garuda advise` faster and cheaper to repeat: runs in parallel,
|
|
31
|
+
reuse of earlier runs, a cost estimate from real requests, and the command to go on after a stop
|
|
32
|
+
at the cap. A provider outage no longer counts against a model.
|
|
33
|
+
|
|
34
|
+
- Fix: repo-task evals (`-s repo`, `garuda advise`) on macOS failed every check in 0.18.1 when the
|
|
35
|
+
project's tools live in `~/Library`. pnpm runs the version that a project asks for from
|
|
36
|
+
`~/Library/pnpm`, and the 0.18.1 rules denied all of the home folder except hidden entries. The
|
|
37
|
+
check stopped with "EPERM: process.cwd failed", so both models passed 0 of 30 in the user's run.
|
|
38
|
+
`~/Library` is readable again in these runs; Mail, Messages, Safari, iCloud Drive, the other
|
|
39
|
+
personal parts and the keychains stay denied for every sandboxed command. Run 0.18.1 repo evals
|
|
40
|
+
again.
|
|
41
|
+
- `advise` gives no advice when neither model passes a task: the check itself may fail.
|
|
42
|
+
- Evals: a timeout while the model's API is silent is a new failure kind, `provider`. It applies
|
|
43
|
+
when a request was open at the limit, and the stream had broken and been retried or the API had
|
|
44
|
+
sent nothing for 2 minutes. Such a run is not scored, and it runs once more by itself; its
|
|
45
|
+
spending is kept. In the user's Sonnet run, 4 of 5 timeouts were such stalls, and they counted
|
|
46
|
+
against the model. A timeout also shows its steps now, not 0.
|
|
47
|
+
- Fix: in repo-task evals, Vite could not write its compiled `vitest.config.ts` to
|
|
48
|
+
`node_modules/.vite-temp`, because the task's `node_modules` links to the project's and was
|
|
49
|
+
read-only. In the user's suite 10 of 30 tasks failed for both models before a test ran.
|
|
50
|
+
Commands may now write `.vite-temp`, `.vite` and `.cache` there, and nothing else.
|
|
51
|
+
- `garuda eval --from-git` checks each kept commit a third time, as an eval runs it (a copy, in the
|
|
52
|
+
sandbox), and leaves out a commit whose tests fail there. Before, it checked only on the host,
|
|
53
|
+
and a task could pass the build but never an eval (2 of 30 in the user's suite: tests that start
|
|
54
|
+
their own sandbox). Build your suite again.
|
|
55
|
+
- The Homebrew formula in the repository names 0.18.1 and its tarball hash.
|
|
56
|
+
- `garuda advise` runs 4 tasks at a time (`--parallel <n>`); each result line starts with the
|
|
57
|
+
model and a count, as in `[haiku-5-5 12/30] ✓ fix-parser: pass`. The first real run, one at a
|
|
58
|
+
time, took 64 minutes for 30 tasks.
|
|
59
|
+
- `advise` reuses an earlier run of a model when the model, the task hashes, `--repeat`, `--seed`
|
|
60
|
+
and the Garuda version and build commit are the same and every run is there. Runs of 0.18.0 and
|
|
61
|
+
before are not reused: they could read your checkout (the 0.18.1 fix).
|
|
62
|
+
- The cost estimate comes from the mean cost of earlier runs of the model on the same tasks. With
|
|
63
|
+
none, it prices each request of the other model's runs (from their session files) at this
|
|
64
|
+
model's prices; else the token guess. Each line says which. (A first version scaled by the ratio
|
|
65
|
+
of list prices: 20 for Sonnet 5.5 over Haiku 5.5, but Haiku costs 5 times as much for a prompt
|
|
66
|
+
over 100 000 tokens. On a real run it estimated $19.16 where the requests give $12.62.)
|
|
67
|
+
- The repriced estimate is scaled by what both models really cost on shared repo tasks before:
|
|
68
|
+
Sonnet 5.5 needs fewer steps and tokens than Haiku 5.5, and cost $6.30 where Haiku's requests at
|
|
69
|
+
its prices gave $12.62 (× 0.52). Without the factor, the user's 90-run estimate was $33.62; with
|
|
70
|
+
it, $17.37. Runs where no task passed, and built-in suites, are left out.
|
|
71
|
+
- When the estimate or the first model's real cost passes `--max-cost`, `advise` prints the
|
|
72
|
+
command that goes on (`garuda advise --models a,b --repeat R --seed S --max-cost N`). The first
|
|
73
|
+
model's run is kept and reused.
|
|
74
|
+
- `advise` stops when a run fails to start (for example a refused key), not at a missing report.
|
|
75
|
+
|
|
3
76
|
## 0.18.1 (2026-11-06)
|
|
4
77
|
|
|
5
78
|
A security release. In 0.18.0 and earlier, a command in the OS sandbox could read your personal
|
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Website and docs: https://madhu-sv.github.io/garuda/
|
|
|
7
7
|
Garuda is a coding agent for the terminal, and for editors over the Agent Client Protocol. It reads
|
|
8
8
|
and edits your code, runs commands in an OS sandbox, and keeps an audit log of its permission
|
|
9
9
|
decisions. It works with Claude and with OpenAI-compatible models, local ones too. This is version
|
|
10
|
-
0.
|
|
10
|
+
0.19.0. Release notes: [CHANGELOG.md](CHANGELOG.md).
|
|
11
11
|
|
|
12
12
|
## What Garuda does on your machine
|
|
13
13
|
|
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
Runtime,
|
|
4
|
-
auditDirFor
|
|
5
|
-
} from "./chunk-5UDYQ5MK.js";
|
|
6
|
-
import "./chunk-G2AX7MGS.js";
|
|
7
|
-
import "./chunk-ULAJPMY7.js";
|
|
8
|
-
import "./chunk-JMH5W5DG.js";
|
|
9
2
|
import {
|
|
10
3
|
chooseModel,
|
|
11
4
|
keepProviderKeys,
|
|
12
5
|
loadModelsConfig,
|
|
13
6
|
providerKeySource,
|
|
14
7
|
resolveModel
|
|
15
|
-
} from "./chunk-
|
|
16
|
-
import
|
|
8
|
+
} from "./chunk-QE7E77IR.js";
|
|
9
|
+
import {
|
|
10
|
+
Runtime,
|
|
11
|
+
auditDirFor
|
|
12
|
+
} from "./chunk-BFK432BK.js";
|
|
13
|
+
import "./chunk-G2AX7MGS.js";
|
|
14
|
+
import "./chunk-ULAJPMY7.js";
|
|
15
|
+
import "./chunk-65NUEOI7.js";
|
|
17
16
|
import "./chunk-4QRTTRVK.js";
|
|
18
17
|
import "./chunk-A2QAIRQL.js";
|
|
19
18
|
import {
|
|
@@ -26,13 +25,13 @@ import {
|
|
|
26
25
|
import "./chunk-MNE7HHX2.js";
|
|
27
26
|
import {
|
|
28
27
|
VERSION
|
|
29
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-U6Z4WH3B.js";
|
|
30
29
|
import "./chunk-5DEWP3YX.js";
|
|
31
30
|
import "./chunk-PZS3V6P3.js";
|
|
32
31
|
import "./chunk-QANRZW7W.js";
|
|
33
32
|
import "./chunk-ATW7BNIG.js";
|
|
34
33
|
import "./chunk-AOGFYUKG.js";
|
|
35
|
-
import "./chunk-
|
|
34
|
+
import "./chunk-L5TZTKUV.js";
|
|
36
35
|
import "./chunk-BHZ6G65V.js";
|
|
37
36
|
import "./chunk-P6XWWIEK.js";
|
|
38
37
|
import "./chunk-UIRJOQQS.js";
|
|
@@ -50,7 +49,7 @@ async function acpCommand(options) {
|
|
|
50
49
|
`);
|
|
51
50
|
return 1;
|
|
52
51
|
}
|
|
53
|
-
const { setupCommand, terminalIO } = await import("./setupCommand-
|
|
52
|
+
const { setupCommand, terminalIO } = await import("./setupCommand-YHDRJTPZ.js");
|
|
54
53
|
return setupCommand({}, { io: await terminalIO() });
|
|
55
54
|
}
|
|
56
55
|
const output = protocolOutput(process.stdout, process.stderr);
|
|
@@ -193,4 +192,4 @@ export {
|
|
|
193
192
|
prepare,
|
|
194
193
|
protocolOutput
|
|
195
194
|
};
|
|
196
|
-
//# sourceMappingURL=acpCommand-
|
|
195
|
+
//# sourceMappingURL=acpCommand-SIFV5XBS.js.map
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
taskHash
|
|
4
|
+
} from "./chunk-2TX3KY4M.js";
|
|
5
|
+
import {
|
|
6
|
+
garudaCommit
|
|
7
|
+
} from "./chunk-YIU5QGMV.js";
|
|
8
|
+
import {
|
|
9
|
+
escapeHtml
|
|
10
|
+
} from "./chunk-4BELTWP4.js";
|
|
11
|
+
import {
|
|
12
|
+
BUILD_COMMIT
|
|
13
|
+
} from "./chunk-FBCVU2P2.js";
|
|
14
|
+
import {
|
|
15
|
+
compareReports,
|
|
16
|
+
costPerPass,
|
|
17
|
+
formatComparison,
|
|
18
|
+
loadReport,
|
|
19
|
+
repoEvalTasks
|
|
20
|
+
} from "./chunk-CKVF67VJ.js";
|
|
21
|
+
import {
|
|
22
|
+
loadModelsConfig,
|
|
23
|
+
resolveModel
|
|
24
|
+
} from "./chunk-QE7E77IR.js";
|
|
25
|
+
import "./chunk-AX3A5KPH.js";
|
|
26
|
+
import "./chunk-2OYTNNUJ.js";
|
|
27
|
+
import "./chunk-NXJ4XLB5.js";
|
|
28
|
+
import "./chunk-BFK432BK.js";
|
|
29
|
+
import "./chunk-G2AX7MGS.js";
|
|
30
|
+
import "./chunk-ULAJPMY7.js";
|
|
31
|
+
import "./chunk-65NUEOI7.js";
|
|
32
|
+
import "./chunk-4QRTTRVK.js";
|
|
33
|
+
import "./chunk-A2QAIRQL.js";
|
|
34
|
+
import {
|
|
35
|
+
newSessionId
|
|
36
|
+
} from "./chunk-NLS7GMF7.js";
|
|
37
|
+
import {
|
|
38
|
+
costOf,
|
|
39
|
+
responseCost
|
|
40
|
+
} from "./chunk-7KBOIPUT.js";
|
|
41
|
+
import "./chunk-B25VL4RU.js";
|
|
42
|
+
import "./chunk-MNE7HHX2.js";
|
|
43
|
+
import {
|
|
44
|
+
VERSION
|
|
45
|
+
} from "./chunk-U6Z4WH3B.js";
|
|
46
|
+
import "./chunk-5DEWP3YX.js";
|
|
47
|
+
import "./chunk-PZS3V6P3.js";
|
|
48
|
+
import "./chunk-QANRZW7W.js";
|
|
49
|
+
import "./chunk-ATW7BNIG.js";
|
|
50
|
+
import "./chunk-A4SSBJ2P.js";
|
|
51
|
+
import "./chunk-AOGFYUKG.js";
|
|
52
|
+
import "./chunk-5GF2VINR.js";
|
|
53
|
+
import "./chunk-L5TZTKUV.js";
|
|
54
|
+
import "./chunk-BHZ6G65V.js";
|
|
55
|
+
import "./chunk-P6XWWIEK.js";
|
|
56
|
+
import "./chunk-UIRJOQQS.js";
|
|
57
|
+
import "./chunk-4LFJFDCF.js";
|
|
58
|
+
import "./chunk-3NJNVZTK.js";
|
|
59
|
+
|
|
60
|
+
// src/cli/advise.ts
|
|
61
|
+
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "fs";
|
|
62
|
+
import { basename, join } from "path";
|
|
63
|
+
import { createInterface } from "readline/promises";
|
|
64
|
+
var ADVISE_MODELS = ["claude-haiku-5-5", "claude-sonnet-5-5"];
|
|
65
|
+
var ADVISE_REPEAT = 2;
|
|
66
|
+
var ADVISE_MAX_COST = 2;
|
|
67
|
+
var GUESS_INPUT_TOKENS = 6e4;
|
|
68
|
+
var GUESS_OUTPUT_TOKENS = 1500;
|
|
69
|
+
function estimateUsd(price, runs) {
|
|
70
|
+
if (price === void 0) return void 0;
|
|
71
|
+
return runs * (GUESS_INPUT_TOKENS * price.input + GUESS_OUTPUT_TOKENS * price.output) / 1e6;
|
|
72
|
+
}
|
|
73
|
+
function adviceText(a, b, c, tasks) {
|
|
74
|
+
const rough = tasks < 10 ? ` With ${tasks} task${tasks === 1 ? "" : "s"}, this is a rough guide.` : "";
|
|
75
|
+
if (c.verdict === "B passes more")
|
|
76
|
+
return `Advice: ${b} for this repo. It passes more often.${rough}`;
|
|
77
|
+
if (c.verdict === "B passes less")
|
|
78
|
+
return `Advice: ${a} for this repo. It passes more often.${rough}`;
|
|
79
|
+
if (c.rateA === 0 && c.rateB === 0) {
|
|
80
|
+
return `Advice: none. Neither model passed a task, so the check itself may fail: read a run's "reason" in report.json before you trust this.`;
|
|
81
|
+
}
|
|
82
|
+
const ratio = c.costRatio;
|
|
83
|
+
if (ratio === void 0) {
|
|
84
|
+
return `Advice: no clear difference in passes, and the cost could not be compared.${rough}`;
|
|
85
|
+
}
|
|
86
|
+
const times = (x) => `${x.toFixed(1)} times`;
|
|
87
|
+
if (ratio.verdict === "B costs less per accepted change") {
|
|
88
|
+
return `Advice: ${b} for this repo. It passes as often, and ${a} costs ${times(1 / ratio.value)} as much per accepted change.${rough}`;
|
|
89
|
+
}
|
|
90
|
+
if (ratio.verdict === "B costs more per accepted change") {
|
|
91
|
+
return `Advice: ${a} for this repo. It passes as often, and ${b} costs ${times(ratio.value)} as much per accepted change.${rough}`;
|
|
92
|
+
}
|
|
93
|
+
const cheaper = ratio.value <= 1 ? b : a;
|
|
94
|
+
return `Advice: either. No clear difference in passes or in cost; ${cheaper} was cheaper in these runs.${rough}`;
|
|
95
|
+
}
|
|
96
|
+
function adviceHtml(comparison, advice) {
|
|
97
|
+
return [
|
|
98
|
+
"<!doctype html>",
|
|
99
|
+
'<html lang="en"><head><meta charset="utf-8">',
|
|
100
|
+
`<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'">`,
|
|
101
|
+
"<title>Garuda advise</title>",
|
|
102
|
+
"<style>body{font:15px/1.5 system-ui,sans-serif;max-width:900px;margin:32px auto;padding:0 16px}",
|
|
103
|
+
"pre{font:13px/1.45 ui-monospace,monospace;white-space:pre-wrap}.advice{font-weight:600}</style>",
|
|
104
|
+
"</head><body><h1>Which model for this repository</h1>",
|
|
105
|
+
`<p class="advice">${escapeHtml(advice)}</p>`,
|
|
106
|
+
`<pre>${escapeHtml(comparison)}</pre>`,
|
|
107
|
+
"</body></html>",
|
|
108
|
+
""
|
|
109
|
+
].join("\n");
|
|
110
|
+
}
|
|
111
|
+
var ADVISE_PARALLEL = 4;
|
|
112
|
+
function pastRuns(evalsDir) {
|
|
113
|
+
const folders = [];
|
|
114
|
+
try {
|
|
115
|
+
for (const name of readdirSync(evalsDir)) {
|
|
116
|
+
const path = join(evalsDir, name);
|
|
117
|
+
folders.push(path, join(path, "a"), join(path, "b"));
|
|
118
|
+
}
|
|
119
|
+
} catch {
|
|
120
|
+
return [];
|
|
121
|
+
}
|
|
122
|
+
const out = [];
|
|
123
|
+
for (const folder of folders) {
|
|
124
|
+
const file = join(folder, "report.json");
|
|
125
|
+
try {
|
|
126
|
+
const data = JSON.parse(readFileSync(file, "utf8"));
|
|
127
|
+
const garuda = data.garuda;
|
|
128
|
+
out.push({
|
|
129
|
+
path: folder,
|
|
130
|
+
model: typeof data.spec === "string" ? data.spec : typeof data.model === "string" ? data.model : void 0,
|
|
131
|
+
version: typeof garuda?.version === "string" ? garuda.version : void 0,
|
|
132
|
+
commit: typeof garuda?.commit === "string" ? garuda.commit : void 0,
|
|
133
|
+
suite: typeof data.suite === "string" ? data.suite : void 0,
|
|
134
|
+
repeat: typeof data.repeat === "number" ? data.repeat : void 0,
|
|
135
|
+
seed: typeof data.seed === "number" ? data.seed : void 0,
|
|
136
|
+
tasks: typeof data.tasks === "object" && data.tasks !== null ? data.tasks : void 0,
|
|
137
|
+
results: Array.isArray(data.results) ? data.results : [],
|
|
138
|
+
mtimeMs: statSync(file).mtimeMs
|
|
139
|
+
});
|
|
140
|
+
} catch {
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return out.sort((x, y) => y.mtimeMs - x.mtimeMs);
|
|
144
|
+
}
|
|
145
|
+
var sameTasks = (a, b) => a !== void 0 && Object.keys(a).length === Object.keys(b).length && Object.entries(b).every(([id, hash]) => a[id] === hash);
|
|
146
|
+
function reusableRun(runs, want) {
|
|
147
|
+
const count = Object.keys(want.tasks).length * want.repeat;
|
|
148
|
+
return runs.find(
|
|
149
|
+
(r) => r.model === want.model && r.version === want.version && r.commit === want.commit && r.suite === "repo" && r.repeat === want.repeat && r.seed === want.seed && sameTasks(r.tasks, want.tasks) && r.results.length === count
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
function pastCostPerRun(runs, model, tasks) {
|
|
153
|
+
const results = runs.filter((r) => r.model === model && sameTasks(r.tasks, tasks)).flatMap((r) => r.results);
|
|
154
|
+
if (results.length === 0 || results.some((r) => r.costUsd === void 0)) return void 0;
|
|
155
|
+
return results.reduce((sum, r) => sum + (r.costUsd ?? 0), 0) / results.length;
|
|
156
|
+
}
|
|
157
|
+
function repricedSessionCost(file, price) {
|
|
158
|
+
let text;
|
|
159
|
+
try {
|
|
160
|
+
text = readFileSync(file, "utf8");
|
|
161
|
+
} catch {
|
|
162
|
+
return void 0;
|
|
163
|
+
}
|
|
164
|
+
let usd = 0;
|
|
165
|
+
for (const line of text.split("\n")) {
|
|
166
|
+
if (line.trim() === "") continue;
|
|
167
|
+
let record;
|
|
168
|
+
try {
|
|
169
|
+
record = JSON.parse(line);
|
|
170
|
+
} catch {
|
|
171
|
+
return void 0;
|
|
172
|
+
}
|
|
173
|
+
if (record.type === "unreported" || record.type === "child_debit") return void 0;
|
|
174
|
+
if (line.includes('"subagent":{')) return void 0;
|
|
175
|
+
if (record.type === "assistant") {
|
|
176
|
+
const cost = responseCost(record.response, price);
|
|
177
|
+
if (cost === void 0) return void 0;
|
|
178
|
+
usd += cost;
|
|
179
|
+
}
|
|
180
|
+
if (record.type === "side_usage") usd += costOf(record.usage, price);
|
|
181
|
+
}
|
|
182
|
+
return usd;
|
|
183
|
+
}
|
|
184
|
+
function repricedCostPerRun(runs, model, tasks, price) {
|
|
185
|
+
if (price === void 0) return void 0;
|
|
186
|
+
const costs = [];
|
|
187
|
+
for (const run of runs.filter((r) => r.model === model && sameTasks(r.tasks, tasks))) {
|
|
188
|
+
for (const result of run.results) {
|
|
189
|
+
const file = result.sessionFile;
|
|
190
|
+
if (file === void 0) {
|
|
191
|
+
costs.push(void 0);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const near = join(run.path, basename(file));
|
|
195
|
+
costs.push(repricedSessionCost(near, price) ?? repricedSessionCost(file, price));
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const known = costs.filter((c) => c !== void 0);
|
|
199
|
+
if (known.length === 0 || known.length * 2 < costs.length) return void 0;
|
|
200
|
+
return known.reduce((sum, c) => sum + c, 0) / known.length;
|
|
201
|
+
}
|
|
202
|
+
function repriceFactor(runs, source, target, price) {
|
|
203
|
+
const useful = runs.filter((r) => r.suite === "repo" && r.results.some((x) => x.passed));
|
|
204
|
+
const sets = [];
|
|
205
|
+
for (const run of useful.filter((r) => r.model === source && r.tasks !== void 0)) {
|
|
206
|
+
const tasks = run.tasks;
|
|
207
|
+
if (sets.some((t) => sameTasks(t, tasks))) continue;
|
|
208
|
+
if (useful.some((r) => r.model === target && sameTasks(r.tasks, tasks))) sets.push(tasks);
|
|
209
|
+
}
|
|
210
|
+
const ratios = [];
|
|
211
|
+
for (const tasks of sets) {
|
|
212
|
+
const real = pastCostPerRun(useful, target, tasks);
|
|
213
|
+
const repriced = repricedCostPerRun(useful, source, tasks, price);
|
|
214
|
+
if (real !== void 0 && repriced !== void 0 && repriced > 0) ratios.push(real / repriced);
|
|
215
|
+
}
|
|
216
|
+
if (ratios.length === 0) return void 0;
|
|
217
|
+
return { value: ratios.reduce((sum, r) => sum + r, 0) / ratios.length, sets: ratios.length };
|
|
218
|
+
}
|
|
219
|
+
function costEstimate(runs, own, repriced, price) {
|
|
220
|
+
if (own !== void 0) return { usd: own * runs, how: "from earlier runs" };
|
|
221
|
+
if (repriced.perRun !== void 0) {
|
|
222
|
+
const f = repriced.factor;
|
|
223
|
+
return {
|
|
224
|
+
usd: repriced.perRun * (f?.value ?? 1) * runs,
|
|
225
|
+
how: f === void 0 ? `from ${repriced.from}'s requests at this model's prices` : `from ${repriced.from}'s requests at this model's prices, \xD7 ${f.value.toFixed(2)} as in ${f.sets} earlier comparison${f.sets === 1 ? "" : "s"}`
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
return { usd: estimateUsd(price, runs), how: "a guess" };
|
|
229
|
+
}
|
|
230
|
+
var sumKnown = (values) => values.some((v) => v === void 0) ? void 0 : values.reduce((sum, v) => sum + (v ?? 0), 0);
|
|
231
|
+
function continueCommand(a, b, repeat, seed, maxCost) {
|
|
232
|
+
return `garuda advise --models ${a},${b} --repeat ${repeat} --seed ${seed} --max-cost ${Math.ceil(maxCost)}`;
|
|
233
|
+
}
|
|
234
|
+
async function runAdvise(options) {
|
|
235
|
+
const [a, b] = options.models ?? ADVISE_MODELS;
|
|
236
|
+
if (a === void 0 || b === void 0 || options.models?.length === 1 || (options.models?.length ?? 2) > 2) {
|
|
237
|
+
process.stderr.write("Give two models: --models <a>,<b>.\n");
|
|
238
|
+
return 1;
|
|
239
|
+
}
|
|
240
|
+
const repeat = options.repeat ?? ADVISE_REPEAT;
|
|
241
|
+
const maxCost = options.maxCost ?? ADVISE_MAX_COST;
|
|
242
|
+
const seed = options.seed ?? 1;
|
|
243
|
+
const parallel = options.parallel ?? ADVISE_PARALLEL;
|
|
244
|
+
if (!Number.isInteger(repeat) || repeat < 1 || !(maxCost > 0) || !Number.isInteger(parallel) || parallel < 1) {
|
|
245
|
+
process.stderr.write(
|
|
246
|
+
"--repeat and --parallel must be whole numbers of 1 or more, and --max-cost above 0.\n"
|
|
247
|
+
);
|
|
248
|
+
return 1;
|
|
249
|
+
}
|
|
250
|
+
const { repoSuite, runEvalCommand } = await import("./evalCommand-GQWNI2YJ.js");
|
|
251
|
+
const suite = await repoSuite();
|
|
252
|
+
if (typeof suite === "string") {
|
|
253
|
+
process.stderr.write(`${suite}
|
|
254
|
+
`);
|
|
255
|
+
return 1;
|
|
256
|
+
}
|
|
257
|
+
const repoTasks = repoEvalTasks(suite);
|
|
258
|
+
const tasks = repoTasks.length;
|
|
259
|
+
const hashes = Object.fromEntries(repoTasks.map((t) => [t.id, taskHash(t)]));
|
|
260
|
+
const models = await loadModelsConfig();
|
|
261
|
+
if (models.problem !== void 0) {
|
|
262
|
+
process.stderr.write(`${models.problem}
|
|
263
|
+
`);
|
|
264
|
+
return 1;
|
|
265
|
+
}
|
|
266
|
+
let prices;
|
|
267
|
+
try {
|
|
268
|
+
prices = [resolveModel(a, models.config).info.price, resolveModel(b, models.config).info.price];
|
|
269
|
+
} catch (error) {
|
|
270
|
+
process.stderr.write(`${error.message}
|
|
271
|
+
`);
|
|
272
|
+
return 1;
|
|
273
|
+
}
|
|
274
|
+
const evalsDir = join(process.cwd(), ".garuda", "evals");
|
|
275
|
+
const past = pastRuns(evalsDir);
|
|
276
|
+
const commit = BUILD_COMMIT ?? await garudaCommit();
|
|
277
|
+
const want = { repeat, seed, tasks: hashes, version: VERSION, commit };
|
|
278
|
+
const reuse = [
|
|
279
|
+
reusableRun(past, { ...want, model: a }),
|
|
280
|
+
reusableRun(past, { ...want, model: b })
|
|
281
|
+
];
|
|
282
|
+
const runs = tasks * repeat;
|
|
283
|
+
const perRun = [pastCostPerRun(past, a, hashes), pastCostPerRun(past, b, hashes)];
|
|
284
|
+
const repriced = (i, from) => perRun[i] === void 0 ? repricedCostPerRun(past, from, hashes, prices[i]) : void 0;
|
|
285
|
+
const factors = [
|
|
286
|
+
perRun[0] === void 0 ? repriceFactor(past, b, a, prices[0]) : void 0,
|
|
287
|
+
perRun[1] === void 0 ? repriceFactor(past, a, b, prices[1]) : void 0
|
|
288
|
+
];
|
|
289
|
+
const estimates = [
|
|
290
|
+
costEstimate(
|
|
291
|
+
runs,
|
|
292
|
+
perRun[0],
|
|
293
|
+
{ perRun: repriced(0, b), from: b, factor: factors[0] },
|
|
294
|
+
prices[0]
|
|
295
|
+
),
|
|
296
|
+
costEstimate(
|
|
297
|
+
runs,
|
|
298
|
+
perRun[1],
|
|
299
|
+
{ perRun: repriced(1, a), from: a, factor: factors[1] },
|
|
300
|
+
prices[1]
|
|
301
|
+
)
|
|
302
|
+
];
|
|
303
|
+
const line = (model, i) => {
|
|
304
|
+
const r = reuse[i];
|
|
305
|
+
const e = estimates[i];
|
|
306
|
+
return r !== void 0 ? ` ${model}: reuses ${r.path} (Garuda ${r.version}, ${r.results.length} runs)` : ` ${model}: ${runs} runs, ${e?.usd === void 0 ? "cost unknown" : `about $${e.usd.toFixed(2)} (${e.how})`}`;
|
|
307
|
+
};
|
|
308
|
+
const toSpend = sumKnown(
|
|
309
|
+
[0, 1].filter((i) => reuse[i] === void 0).map((i) => estimates[i]?.usd)
|
|
310
|
+
);
|
|
311
|
+
process.stderr.write(
|
|
312
|
+
[
|
|
313
|
+
`Your repo: ${tasks} task${tasks === 1 ? "" : "s"} from its history (tests that fail before the commit and pass after it).`,
|
|
314
|
+
`${repeat} run${repeat === 1 ? "" : "s"} a task, ${parallel} at a time, seed ${seed}:`,
|
|
315
|
+
line(a, 0),
|
|
316
|
+
line(b, 1),
|
|
317
|
+
`Cap: --max-cost $${maxCost}; it stops before a model that would pass it.`,
|
|
318
|
+
""
|
|
319
|
+
].join("\n")
|
|
320
|
+
);
|
|
321
|
+
if (toSpend !== void 0 && toSpend > maxCost) {
|
|
322
|
+
process.stderr.write(
|
|
323
|
+
`The estimate passes the cap. To go on: ${continueCommand(a, b, repeat, seed, toSpend * 1.25)}
|
|
324
|
+
`
|
|
325
|
+
);
|
|
326
|
+
return 1;
|
|
327
|
+
}
|
|
328
|
+
if (options.yes !== true && (reuse[0] === void 0 || reuse[1] === void 0)) {
|
|
329
|
+
if (!process.stdin.isTTY) {
|
|
330
|
+
process.stderr.write("Add --yes to start without a question.\n");
|
|
331
|
+
return 1;
|
|
332
|
+
}
|
|
333
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
334
|
+
const answer = await rl.question("Start? (y/N) ");
|
|
335
|
+
rl.close();
|
|
336
|
+
if (!/^y(es)?$/i.test(answer.trim())) return 1;
|
|
337
|
+
}
|
|
338
|
+
const dir = join(evalsDir, `advise-${newSessionId()}`);
|
|
339
|
+
mkdirSync(dir, { recursive: true });
|
|
340
|
+
const runModel = async (model, label, i) => {
|
|
341
|
+
const earlier = reuse[i];
|
|
342
|
+
if (earlier !== void 0) {
|
|
343
|
+
process.stderr.write(`Reusing ${earlier.path} for ${model}.
|
|
344
|
+
`);
|
|
345
|
+
return earlier.path;
|
|
346
|
+
}
|
|
347
|
+
const outDir = join(dir, label);
|
|
348
|
+
const code = await runEvalCommand({
|
|
349
|
+
model,
|
|
350
|
+
suite: "repo",
|
|
351
|
+
repeat,
|
|
352
|
+
shuffle: true,
|
|
353
|
+
seed,
|
|
354
|
+
parallel,
|
|
355
|
+
outDir,
|
|
356
|
+
progress: model.replace(/^claude-/, "")
|
|
357
|
+
});
|
|
358
|
+
return code === 1 ? void 0 : outDir;
|
|
359
|
+
};
|
|
360
|
+
const pathA = await runModel(a, "a", 0);
|
|
361
|
+
if (pathA === void 0) return 1;
|
|
362
|
+
const first = await loadReport(pathA);
|
|
363
|
+
if (reuse[0] === void 0 && reuse[1] === void 0) {
|
|
364
|
+
const spent = costPerPass(first.results);
|
|
365
|
+
const spentA = spent.kind === "unknown" ? void 0 : spent.totalUsd;
|
|
366
|
+
const nextB = costEstimate(
|
|
367
|
+
runs,
|
|
368
|
+
perRun[1],
|
|
369
|
+
{
|
|
370
|
+
perRun: repricedCostPerRun(pastRuns(evalsDir), a, hashes, prices[1]),
|
|
371
|
+
from: a,
|
|
372
|
+
factor: factors[1]
|
|
373
|
+
},
|
|
374
|
+
prices[1]
|
|
375
|
+
).usd;
|
|
376
|
+
if (spentA !== void 0 && nextB !== void 0 && spentA + nextB > maxCost) {
|
|
377
|
+
process.stderr.write(
|
|
378
|
+
[
|
|
379
|
+
`Stopped before ${b}: ${a} cost $${spentA.toFixed(2)}, and ${b} would cost about $${nextB.toFixed(2)}, past $${maxCost}.`,
|
|
380
|
+
`${a}'s run is kept and will be reused. To go on: ${continueCommand(a, b, repeat, seed, nextB * 1.25)}`,
|
|
381
|
+
""
|
|
382
|
+
].join("\n")
|
|
383
|
+
);
|
|
384
|
+
return 1;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
const pathB = await runModel(b, "b", 1);
|
|
388
|
+
if (pathB === void 0) return 1;
|
|
389
|
+
const second = await loadReport(pathB);
|
|
390
|
+
const result = compareReports(first, second, { seed });
|
|
391
|
+
if ("problems" in result) {
|
|
392
|
+
process.stderr.write(`The two runs cannot be compared:
|
|
393
|
+
${result.problems.join("\n")}
|
|
394
|
+
`);
|
|
395
|
+
return 1;
|
|
396
|
+
}
|
|
397
|
+
const comparison = formatComparison(first, second, result.comparison);
|
|
398
|
+
const advice = adviceText(a, b, result.comparison, tasks);
|
|
399
|
+
writeFileSync(join(dir, "advice.txt"), `${advice}
|
|
400
|
+
|
|
401
|
+
${comparison}
|
|
402
|
+
`);
|
|
403
|
+
writeFileSync(join(dir, "report.html"), adviceHtml(comparison, advice));
|
|
404
|
+
process.stdout.write(`
|
|
405
|
+
${comparison}
|
|
406
|
+
|
|
407
|
+
${advice}
|
|
408
|
+
Report: ${join(dir, "report.html")}
|
|
409
|
+
`);
|
|
410
|
+
return 0;
|
|
411
|
+
}
|
|
412
|
+
export {
|
|
413
|
+
ADVISE_MAX_COST,
|
|
414
|
+
ADVISE_MODELS,
|
|
415
|
+
ADVISE_PARALLEL,
|
|
416
|
+
ADVISE_REPEAT,
|
|
417
|
+
GUESS_INPUT_TOKENS,
|
|
418
|
+
GUESS_OUTPUT_TOKENS,
|
|
419
|
+
adviceHtml,
|
|
420
|
+
adviceText,
|
|
421
|
+
continueCommand,
|
|
422
|
+
costEstimate,
|
|
423
|
+
estimateUsd,
|
|
424
|
+
pastCostPerRun,
|
|
425
|
+
pastRuns,
|
|
426
|
+
repriceFactor,
|
|
427
|
+
repricedCostPerRun,
|
|
428
|
+
repricedSessionCost,
|
|
429
|
+
reusableRun,
|
|
430
|
+
runAdvise,
|
|
431
|
+
sumKnown
|
|
432
|
+
};
|
|
433
|
+
//# sourceMappingURL=advise-EA7OIYSV.js.map
|
|
@@ -42,11 +42,12 @@ function withHiddenThinking(response) {
|
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
var STREAM_IDLE_MS = 20 * 6e4;
|
|
45
|
+
var STREAM_ALLOW_H2 = false;
|
|
45
46
|
var dispatchers = /* @__PURE__ */ new Map();
|
|
46
47
|
function streamFetch(idleMs = STREAM_IDLE_MS) {
|
|
47
48
|
let agent = dispatchers.get(idleMs);
|
|
48
49
|
if (agent === void 0) {
|
|
49
|
-
agent = new Agent({ bodyTimeout: idleMs });
|
|
50
|
+
agent = new Agent({ bodyTimeout: idleMs, allowH2: STREAM_ALLOW_H2 });
|
|
50
51
|
dispatchers.set(idleMs, agent);
|
|
51
52
|
}
|
|
52
53
|
const dispatcher = agent;
|
|
@@ -320,6 +321,7 @@ export {
|
|
|
320
321
|
AnthropicClient,
|
|
321
322
|
BATCH_CHECK_TIMEOUT_MS,
|
|
322
323
|
BATCH_POLL_MS,
|
|
324
|
+
STREAM_ALLOW_H2,
|
|
323
325
|
STREAM_IDLE_MS,
|
|
324
326
|
WEB_SEARCH_TOOL,
|
|
325
327
|
clientOptions,
|
|
@@ -330,4 +332,4 @@ export {
|
|
|
330
332
|
toWireTools,
|
|
331
333
|
withHiddenThinking
|
|
332
334
|
};
|
|
333
|
-
//# sourceMappingURL=anthropic-
|
|
335
|
+
//# sourceMappingURL=anthropic-6DKONNQG.js.map
|
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
CHAT_STALE,
|
|
4
4
|
garudaCommit,
|
|
5
5
|
staleBuildNote
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-YIU5QGMV.js";
|
|
7
|
+
import "./chunk-5GF2VINR.js";
|
|
8
|
+
import "./chunk-L5TZTKUV.js";
|
|
9
9
|
import "./chunk-BHZ6G65V.js";
|
|
10
10
|
import "./chunk-P6XWWIEK.js";
|
|
11
11
|
import "./chunk-UIRJOQQS.js";
|
|
@@ -16,4 +16,4 @@ export {
|
|
|
16
16
|
garudaCommit,
|
|
17
17
|
staleBuildNote
|
|
18
18
|
};
|
|
19
|
-
//# sourceMappingURL=buildCheck-
|
|
19
|
+
//# sourceMappingURL=buildCheck-JNWGER5R.js.map
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import {
|
|
3
3
|
hostCommand,
|
|
4
4
|
shellWord
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-5GF2VINR.js";
|
|
6
6
|
import {
|
|
7
7
|
JOBS_DIR
|
|
8
8
|
} from "./chunk-VTMD5IHD.js";
|
|
@@ -154,4 +154,4 @@ export {
|
|
|
154
154
|
nightAgentTime,
|
|
155
155
|
removeAgent
|
|
156
156
|
};
|
|
157
|
-
//# sourceMappingURL=chunk-
|
|
157
|
+
//# sourceMappingURL=chunk-2THG7PEC.js.map
|