@garuda-agent/garuda 0.18.1 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/README.md +1 -1
- package/dist/{acpCommand-JAYNMIMP.js → acpCommand-PUJHFWYU.js} +11 -12
- package/dist/advise-C777SZ5M.js +433 -0
- package/dist/buildInfo-Z4P7QAQL.js +8 -0
- package/dist/{chunk-QLBCM2YQ.js → chunk-2FIV2VZP.js} +2 -2
- package/dist/{chunk-TLA2PDGM.js → chunk-4BELTWP4.js} +5 -5
- package/dist/{chunk-JMH5W5DG.js → chunk-65NUEOI7.js} +44 -1
- package/dist/chunk-6JQTPW3P.js +88 -0
- package/dist/chunk-A5DLAXB4.js +1780 -0
- package/dist/{chunk-DQSCCE77.js → chunk-AX3A5KPH.js} +5 -5
- package/dist/chunk-CFDRP6XW.js +9 -0
- package/dist/chunk-E6YRCDSF.js +9 -0
- package/dist/{chunk-34WBDQBL.js → chunk-HP6EGFHV.js} +18 -18
- package/dist/{chunk-GKU62MHQ.js → chunk-LWNMEGDF.js} +2 -2
- package/dist/{chunk-5UDYQ5MK.js → chunk-STVFWSIL.js} +5 -7
- package/dist/{chunk-IKPGHM54.js → chunk-YEH2OONM.js} +3 -3
- package/dist/cli/index.js +39 -40
- package/dist/{evalCommand-BLJJ4G76.js → evalCommand-MRO6ENCA.js} +70 -42
- package/dist/{evalTune-W53KAD4V.js → evalTune-VL5RHYOG.js} +17 -18
- package/dist/{export-E677HIQS.js → export-ZPF3AMO6.js} +5 -5
- package/dist/{exportHtml-BMV73YRO.js → exportHtml-ISISG5HG.js} +4 -4
- package/dist/{inkChat-EVLYDC3K.js → inkChat-UDFX6GH6.js} +11 -12
- package/dist/{manager-L25IUXHW.js → manager-YH7722C2.js} +2 -2
- package/dist/runner-EBF6EOYM.js +97 -0
- package/dist/{setupCommand-OSOMFHDQ.js → setupCommand-YMF6I3MD.js} +8 -9
- package/dist/{tune-VX2WQTHS.js → tune-KSNKBPNJ.js} +8 -9
- package/package.json +1 -1
- package/dist/advise-HDKJGNBL.js +0 -195
- package/dist/buildInfo-K73FFI5S.js +0 -8
- package/dist/chunk-CWVXOSAA.js +0 -894
- package/dist/chunk-DUYLVGPY.js +0 -821
- package/dist/chunk-H7LIBGZT.js +0 -9
- package/dist/chunk-IYGJWX65.js +0 -9
- package/dist/chunk-NH72AELP.js +0 -48
|
@@ -4,11 +4,11 @@ import {
|
|
|
4
4
|
callChange,
|
|
5
5
|
escapeHtml,
|
|
6
6
|
sessionHtml
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-4BELTWP4.js";
|
|
8
|
+
import "./chunk-AX3A5KPH.js";
|
|
9
|
+
import "./chunk-2OYTNNUJ.js";
|
|
9
10
|
import "./chunk-4QRTTRVK.js";
|
|
10
11
|
import "./chunk-A2QAIRQL.js";
|
|
11
|
-
import "./chunk-2OYTNNUJ.js";
|
|
12
12
|
import "./chunk-NLS7GMF7.js";
|
|
13
13
|
import "./chunk-7KBOIPUT.js";
|
|
14
14
|
import "./chunk-BHZ6G65V.js";
|
|
@@ -18,4 +18,4 @@ export {
|
|
|
18
18
|
escapeHtml,
|
|
19
19
|
sessionHtml
|
|
20
20
|
};
|
|
21
|
-
//# sourceMappingURL=exportHtml-
|
|
21
|
+
//# sourceMappingURL=exportHtml-ISISG5HG.js.map
|
|
@@ -11,12 +11,7 @@ import {
|
|
|
11
11
|
fuzzyFiles,
|
|
12
12
|
runChat,
|
|
13
13
|
statusOf
|
|
14
|
-
} from "./chunk-
|
|
15
|
-
import "./chunk-5UDYQ5MK.js";
|
|
16
|
-
import "./chunk-G2AX7MGS.js";
|
|
17
|
-
import "./chunk-ULAJPMY7.js";
|
|
18
|
-
import "./chunk-JMH5W5DG.js";
|
|
19
|
-
import "./chunk-NH72AELP.js";
|
|
14
|
+
} from "./chunk-HP6EGFHV.js";
|
|
20
15
|
import {
|
|
21
16
|
colorPreview,
|
|
22
17
|
header,
|
|
@@ -31,21 +26,25 @@ import {
|
|
|
31
26
|
summariseCall,
|
|
32
27
|
summariseResult,
|
|
33
28
|
todoLines
|
|
34
|
-
} from "./chunk-
|
|
35
|
-
import "./chunk-4QRTTRVK.js";
|
|
36
|
-
import "./chunk-A2QAIRQL.js";
|
|
29
|
+
} from "./chunk-AX3A5KPH.js";
|
|
37
30
|
import {
|
|
38
31
|
receiptText,
|
|
39
32
|
screenReaderMode
|
|
40
33
|
} from "./chunk-2OYTNNUJ.js";
|
|
34
|
+
import "./chunk-Y62J7PTD.js";
|
|
35
|
+
import "./chunk-STVFWSIL.js";
|
|
36
|
+
import "./chunk-G2AX7MGS.js";
|
|
37
|
+
import "./chunk-ULAJPMY7.js";
|
|
38
|
+
import "./chunk-65NUEOI7.js";
|
|
39
|
+
import "./chunk-4QRTTRVK.js";
|
|
40
|
+
import "./chunk-A2QAIRQL.js";
|
|
41
41
|
import "./chunk-NLS7GMF7.js";
|
|
42
42
|
import "./chunk-7KBOIPUT.js";
|
|
43
|
-
import "./chunk-Y62J7PTD.js";
|
|
44
43
|
import "./chunk-B25VL4RU.js";
|
|
45
44
|
import "./chunk-MNE7HHX2.js";
|
|
46
45
|
import "./chunk-42DSA4EA.js";
|
|
47
46
|
import "./chunk-EU7PA45O.js";
|
|
48
|
-
import "./chunk-
|
|
47
|
+
import "./chunk-CFDRP6XW.js";
|
|
49
48
|
import "./chunk-5DEWP3YX.js";
|
|
50
49
|
import "./chunk-PZS3V6P3.js";
|
|
51
50
|
import "./chunk-QANRZW7W.js";
|
|
@@ -1410,4 +1409,4 @@ export {
|
|
|
1410
1409
|
clearScreenKeepingScrollback,
|
|
1411
1410
|
runInkChat
|
|
1412
1411
|
};
|
|
1413
|
-
//# sourceMappingURL=inkChat-
|
|
1412
|
+
//# sourceMappingURL=inkChat-UDFX6GH6.js.map
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from "./chunk-4P26OKTP.js";
|
|
12
12
|
import {
|
|
13
13
|
VERSION
|
|
14
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-CFDRP6XW.js";
|
|
15
15
|
import {
|
|
16
16
|
USER_SCOPE
|
|
17
17
|
} from "./chunk-5DEWP3YX.js";
|
|
@@ -846,4 +846,4 @@ export {
|
|
|
846
846
|
changeLines,
|
|
847
847
|
warnings
|
|
848
848
|
};
|
|
849
|
-
//# sourceMappingURL=manager-
|
|
849
|
+
//# sourceMappingURL=manager-YH7722C2.js.map
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
EVAL_BIOME_CONFIG,
|
|
4
|
+
EVAL_DENY_RULES,
|
|
5
|
+
EVAL_TASK_TIMEOUT_MS,
|
|
6
|
+
FAILURE_KINDS,
|
|
7
|
+
MCP_NOT_STARTED,
|
|
8
|
+
MCP_PREPARE_FAILED,
|
|
9
|
+
MCP_PREPARE_TIMEOUT_MS,
|
|
10
|
+
ModelWatch,
|
|
11
|
+
PROTECTED_FILES_CHANGED,
|
|
12
|
+
PROVIDER_STALLED,
|
|
13
|
+
PROVIDER_STALL_MS,
|
|
14
|
+
REPO_CHECK_TIMEOUT_MS,
|
|
15
|
+
checkAsEval,
|
|
16
|
+
failureKind,
|
|
17
|
+
failureLine,
|
|
18
|
+
failureSummary,
|
|
19
|
+
formatProject,
|
|
20
|
+
formatReport,
|
|
21
|
+
idWidth,
|
|
22
|
+
isAuthFailure,
|
|
23
|
+
isConnectionFailure,
|
|
24
|
+
isErrorRun,
|
|
25
|
+
isMcpFailure,
|
|
26
|
+
isProviderStall,
|
|
27
|
+
isSetupFailure,
|
|
28
|
+
repoTaskSandbox,
|
|
29
|
+
runCheck,
|
|
30
|
+
runEvalTask,
|
|
31
|
+
runEvals,
|
|
32
|
+
saveUnique,
|
|
33
|
+
summarize,
|
|
34
|
+
withStalledRun,
|
|
35
|
+
writeFiles
|
|
36
|
+
} from "./chunk-A5DLAXB4.js";
|
|
37
|
+
import "./chunk-NXJ4XLB5.js";
|
|
38
|
+
import "./chunk-STVFWSIL.js";
|
|
39
|
+
import "./chunk-G2AX7MGS.js";
|
|
40
|
+
import "./chunk-ULAJPMY7.js";
|
|
41
|
+
import "./chunk-65NUEOI7.js";
|
|
42
|
+
import "./chunk-4QRTTRVK.js";
|
|
43
|
+
import "./chunk-A2QAIRQL.js";
|
|
44
|
+
import "./chunk-NLS7GMF7.js";
|
|
45
|
+
import "./chunk-7KBOIPUT.js";
|
|
46
|
+
import "./chunk-B25VL4RU.js";
|
|
47
|
+
import "./chunk-MNE7HHX2.js";
|
|
48
|
+
import "./chunk-CFDRP6XW.js";
|
|
49
|
+
import "./chunk-5DEWP3YX.js";
|
|
50
|
+
import "./chunk-PZS3V6P3.js";
|
|
51
|
+
import "./chunk-QANRZW7W.js";
|
|
52
|
+
import "./chunk-ATW7BNIG.js";
|
|
53
|
+
import "./chunk-WK5NE6RY.js";
|
|
54
|
+
import "./chunk-AOGFYUKG.js";
|
|
55
|
+
import "./chunk-QXTHEFKP.js";
|
|
56
|
+
import "./chunk-LMKGSYJC.js";
|
|
57
|
+
import "./chunk-BHZ6G65V.js";
|
|
58
|
+
import "./chunk-P6XWWIEK.js";
|
|
59
|
+
import "./chunk-UIRJOQQS.js";
|
|
60
|
+
import "./chunk-4LFJFDCF.js";
|
|
61
|
+
import "./chunk-3NJNVZTK.js";
|
|
62
|
+
export {
|
|
63
|
+
EVAL_BIOME_CONFIG,
|
|
64
|
+
EVAL_DENY_RULES,
|
|
65
|
+
EVAL_TASK_TIMEOUT_MS,
|
|
66
|
+
FAILURE_KINDS,
|
|
67
|
+
MCP_NOT_STARTED,
|
|
68
|
+
MCP_PREPARE_FAILED,
|
|
69
|
+
MCP_PREPARE_TIMEOUT_MS,
|
|
70
|
+
ModelWatch,
|
|
71
|
+
PROTECTED_FILES_CHANGED,
|
|
72
|
+
PROVIDER_STALLED,
|
|
73
|
+
PROVIDER_STALL_MS,
|
|
74
|
+
REPO_CHECK_TIMEOUT_MS,
|
|
75
|
+
checkAsEval,
|
|
76
|
+
failureKind,
|
|
77
|
+
failureLine,
|
|
78
|
+
failureSummary,
|
|
79
|
+
formatProject,
|
|
80
|
+
formatReport,
|
|
81
|
+
idWidth,
|
|
82
|
+
isAuthFailure,
|
|
83
|
+
isConnectionFailure,
|
|
84
|
+
isErrorRun,
|
|
85
|
+
isMcpFailure,
|
|
86
|
+
isProviderStall,
|
|
87
|
+
isSetupFailure,
|
|
88
|
+
repoTaskSandbox,
|
|
89
|
+
runCheck,
|
|
90
|
+
runEvalTask,
|
|
91
|
+
runEvals,
|
|
92
|
+
saveUnique,
|
|
93
|
+
summarize,
|
|
94
|
+
withStalledRun,
|
|
95
|
+
writeFiles
|
|
96
|
+
};
|
|
97
|
+
//# sourceMappingURL=runner-EBF6EOYM.js.map
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
fetchCatalog,
|
|
4
|
-
loadCatalog,
|
|
5
|
-
mergedModels,
|
|
6
|
-
saveCatalog
|
|
7
|
-
} from "./chunk-JMH5W5DG.js";
|
|
8
2
|
import {
|
|
9
3
|
BUILTIN_PROVIDERS,
|
|
10
4
|
KEY_NAME,
|
|
@@ -17,8 +11,13 @@ import {
|
|
|
17
11
|
resolveModel,
|
|
18
12
|
setDefaultModel,
|
|
19
13
|
writeCredentials
|
|
20
|
-
} from "./chunk-
|
|
21
|
-
import
|
|
14
|
+
} from "./chunk-2FIV2VZP.js";
|
|
15
|
+
import {
|
|
16
|
+
fetchCatalog,
|
|
17
|
+
loadCatalog,
|
|
18
|
+
mergedModels,
|
|
19
|
+
saveCatalog
|
|
20
|
+
} from "./chunk-65NUEOI7.js";
|
|
22
21
|
import "./chunk-7KBOIPUT.js";
|
|
23
22
|
import "./chunk-ATW7BNIG.js";
|
|
24
23
|
import "./chunk-BHZ6G65V.js";
|
|
@@ -333,4 +332,4 @@ export {
|
|
|
333
332
|
setupCommand,
|
|
334
333
|
terminalIO
|
|
335
334
|
};
|
|
336
|
-
//# sourceMappingURL=setupCommand-
|
|
335
|
+
//# sourceMappingURL=setupCommand-YMF6I3MD.js.map
|
|
@@ -8,24 +8,23 @@ import {
|
|
|
8
8
|
readAddendum,
|
|
9
9
|
rejectReason,
|
|
10
10
|
tune
|
|
11
|
-
} from "./chunk-
|
|
12
|
-
import "./chunk-
|
|
13
|
-
import "./chunk-CWVXOSAA.js";
|
|
14
|
-
import "./chunk-H7LIBGZT.js";
|
|
11
|
+
} from "./chunk-YEH2OONM.js";
|
|
12
|
+
import "./chunk-6JQTPW3P.js";
|
|
15
13
|
import "./chunk-YAQ6QLKB.js";
|
|
14
|
+
import "./chunk-E6YRCDSF.js";
|
|
15
|
+
import "./chunk-A5DLAXB4.js";
|
|
16
16
|
import "./chunk-NXJ4XLB5.js";
|
|
17
|
-
import "./chunk-
|
|
17
|
+
import "./chunk-STVFWSIL.js";
|
|
18
18
|
import "./chunk-G2AX7MGS.js";
|
|
19
19
|
import "./chunk-ULAJPMY7.js";
|
|
20
|
-
import "./chunk-
|
|
21
|
-
import "./chunk-NH72AELP.js";
|
|
20
|
+
import "./chunk-65NUEOI7.js";
|
|
22
21
|
import "./chunk-4QRTTRVK.js";
|
|
23
22
|
import "./chunk-A2QAIRQL.js";
|
|
24
23
|
import "./chunk-NLS7GMF7.js";
|
|
25
24
|
import "./chunk-7KBOIPUT.js";
|
|
26
25
|
import "./chunk-B25VL4RU.js";
|
|
27
26
|
import "./chunk-MNE7HHX2.js";
|
|
28
|
-
import "./chunk-
|
|
27
|
+
import "./chunk-CFDRP6XW.js";
|
|
29
28
|
import "./chunk-5DEWP3YX.js";
|
|
30
29
|
import "./chunk-PZS3V6P3.js";
|
|
31
30
|
import "./chunk-QANRZW7W.js";
|
|
@@ -49,4 +48,4 @@ export {
|
|
|
49
48
|
rejectReason,
|
|
50
49
|
tune
|
|
51
50
|
};
|
|
52
|
-
//# sourceMappingURL=tune-
|
|
51
|
+
//# sourceMappingURL=tune-KSNKBPNJ.js.map
|
package/package.json
CHANGED
package/dist/advise-HDKJGNBL.js
DELETED
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
compareReports,
|
|
4
|
-
costPerPass,
|
|
5
|
-
formatComparison,
|
|
6
|
-
loadReport,
|
|
7
|
-
repoEvalTasks
|
|
8
|
-
} from "./chunk-CWVXOSAA.js";
|
|
9
|
-
import {
|
|
10
|
-
escapeHtml
|
|
11
|
-
} from "./chunk-TLA2PDGM.js";
|
|
12
|
-
import {
|
|
13
|
-
loadModelsConfig,
|
|
14
|
-
resolveModel
|
|
15
|
-
} from "./chunk-QLBCM2YQ.js";
|
|
16
|
-
import "./chunk-NH72AELP.js";
|
|
17
|
-
import "./chunk-DQSCCE77.js";
|
|
18
|
-
import "./chunk-4QRTTRVK.js";
|
|
19
|
-
import "./chunk-A2QAIRQL.js";
|
|
20
|
-
import "./chunk-2OYTNNUJ.js";
|
|
21
|
-
import {
|
|
22
|
-
newSessionId
|
|
23
|
-
} from "./chunk-NLS7GMF7.js";
|
|
24
|
-
import "./chunk-7KBOIPUT.js";
|
|
25
|
-
import "./chunk-ATW7BNIG.js";
|
|
26
|
-
import "./chunk-WK5NE6RY.js";
|
|
27
|
-
import "./chunk-AOGFYUKG.js";
|
|
28
|
-
import "./chunk-QXTHEFKP.js";
|
|
29
|
-
import "./chunk-LMKGSYJC.js";
|
|
30
|
-
import "./chunk-BHZ6G65V.js";
|
|
31
|
-
import "./chunk-P6XWWIEK.js";
|
|
32
|
-
import "./chunk-UIRJOQQS.js";
|
|
33
|
-
import "./chunk-4LFJFDCF.js";
|
|
34
|
-
import "./chunk-3NJNVZTK.js";
|
|
35
|
-
|
|
36
|
-
// src/cli/advise.ts
|
|
37
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
38
|
-
import { join } from "path";
|
|
39
|
-
import { createInterface } from "readline/promises";
|
|
40
|
-
var ADVISE_MODELS = ["claude-haiku-5-5", "claude-sonnet-5-5"];
|
|
41
|
-
var ADVISE_REPEAT = 2;
|
|
42
|
-
var ADVISE_MAX_COST = 2;
|
|
43
|
-
var GUESS_INPUT_TOKENS = 6e4;
|
|
44
|
-
var GUESS_OUTPUT_TOKENS = 1500;
|
|
45
|
-
function estimateUsd(price, runs) {
|
|
46
|
-
if (price === void 0) return void 0;
|
|
47
|
-
return runs * (GUESS_INPUT_TOKENS * price.input + GUESS_OUTPUT_TOKENS * price.output) / 1e6;
|
|
48
|
-
}
|
|
49
|
-
function adviceText(a, b, c, tasks) {
|
|
50
|
-
const rough = tasks < 10 ? ` With ${tasks} task${tasks === 1 ? "" : "s"}, this is a rough guide.` : "";
|
|
51
|
-
if (c.verdict === "B passes more")
|
|
52
|
-
return `Advice: ${b} for this repo. It passes more often.${rough}`;
|
|
53
|
-
if (c.verdict === "B passes less")
|
|
54
|
-
return `Advice: ${a} for this repo. It passes more often.${rough}`;
|
|
55
|
-
const ratio = c.costRatio;
|
|
56
|
-
if (ratio === void 0) {
|
|
57
|
-
return `Advice: no clear difference in passes, and the cost could not be compared.${rough}`;
|
|
58
|
-
}
|
|
59
|
-
const times = (x) => `${x.toFixed(1)} times`;
|
|
60
|
-
if (ratio.verdict === "B costs less per accepted change") {
|
|
61
|
-
return `Advice: ${b} for this repo. It passes as often, and ${a} costs ${times(1 / ratio.value)} as much per accepted change.${rough}`;
|
|
62
|
-
}
|
|
63
|
-
if (ratio.verdict === "B costs more per accepted change") {
|
|
64
|
-
return `Advice: ${a} for this repo. It passes as often, and ${b} costs ${times(ratio.value)} as much per accepted change.${rough}`;
|
|
65
|
-
}
|
|
66
|
-
const cheaper = ratio.value <= 1 ? b : a;
|
|
67
|
-
return `Advice: either. No clear difference in passes or in cost; ${cheaper} was cheaper in these runs.${rough}`;
|
|
68
|
-
}
|
|
69
|
-
function adviceHtml(comparison, advice) {
|
|
70
|
-
return [
|
|
71
|
-
"<!doctype html>",
|
|
72
|
-
'<html lang="en"><head><meta charset="utf-8">',
|
|
73
|
-
`<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'">`,
|
|
74
|
-
"<title>Garuda advise</title>",
|
|
75
|
-
"<style>body{font:15px/1.5 system-ui,sans-serif;max-width:900px;margin:32px auto;padding:0 16px}",
|
|
76
|
-
"pre{font:13px/1.45 ui-monospace,monospace;white-space:pre-wrap}.advice{font-weight:600}</style>",
|
|
77
|
-
"</head><body><h1>Which model for this repository</h1>",
|
|
78
|
-
`<p class="advice">${escapeHtml(advice)}</p>`,
|
|
79
|
-
`<pre>${escapeHtml(comparison)}</pre>`,
|
|
80
|
-
"</body></html>",
|
|
81
|
-
""
|
|
82
|
-
].join("\n");
|
|
83
|
-
}
|
|
84
|
-
async function runAdvise(options) {
|
|
85
|
-
const [a, b] = options.models ?? ADVISE_MODELS;
|
|
86
|
-
if (a === void 0 || b === void 0 || options.models?.length === 1 || (options.models?.length ?? 2) > 2) {
|
|
87
|
-
process.stderr.write("Give two models: --models <a>,<b>.\n");
|
|
88
|
-
return 1;
|
|
89
|
-
}
|
|
90
|
-
const repeat = options.repeat ?? ADVISE_REPEAT;
|
|
91
|
-
const maxCost = options.maxCost ?? ADVISE_MAX_COST;
|
|
92
|
-
const seed = options.seed ?? 1;
|
|
93
|
-
if (!Number.isInteger(repeat) || repeat < 1 || !(maxCost > 0)) {
|
|
94
|
-
process.stderr.write("--repeat must be a whole number of 1 or more, and --max-cost above 0.\n");
|
|
95
|
-
return 1;
|
|
96
|
-
}
|
|
97
|
-
const { repoSuite, runEvalCommand } = await import("./evalCommand-BLJJ4G76.js");
|
|
98
|
-
const suite = await repoSuite();
|
|
99
|
-
if (typeof suite === "string") {
|
|
100
|
-
process.stderr.write(`${suite}
|
|
101
|
-
`);
|
|
102
|
-
return 1;
|
|
103
|
-
}
|
|
104
|
-
const tasks = repoEvalTasks(suite).length;
|
|
105
|
-
const models = await loadModelsConfig();
|
|
106
|
-
if (models.problem !== void 0) {
|
|
107
|
-
process.stderr.write(`${models.problem}
|
|
108
|
-
`);
|
|
109
|
-
return 1;
|
|
110
|
-
}
|
|
111
|
-
let prices;
|
|
112
|
-
try {
|
|
113
|
-
prices = [resolveModel(a, models.config).info.price, resolveModel(b, models.config).info.price];
|
|
114
|
-
} catch (error) {
|
|
115
|
-
process.stderr.write(`${error.message}
|
|
116
|
-
`);
|
|
117
|
-
return 1;
|
|
118
|
-
}
|
|
119
|
-
const runs = tasks * repeat;
|
|
120
|
-
const estimates = prices.map((p) => estimateUsd(p, runs));
|
|
121
|
-
const total = estimates[0] === void 0 || estimates[1] === void 0 ? void 0 : estimates[0] + estimates[1];
|
|
122
|
-
process.stderr.write(
|
|
123
|
-
[
|
|
124
|
-
`Your repo: ${tasks} task${tasks === 1 ? "" : "s"} from its history (tests that fail before the commit and pass after it).`,
|
|
125
|
-
`Models: ${a}, ${b} \xB7 ${repeat} run${repeat === 1 ? "" : "s"} a task \xB7 ${2 * runs} runs, ${total === void 0 ? "cost unknown" : `about $${total.toFixed(2)} (a guess)`}.`,
|
|
126
|
-
`Cap: --max-cost $${maxCost}; it stops before a model that would pass it.`,
|
|
127
|
-
""
|
|
128
|
-
].join("\n")
|
|
129
|
-
);
|
|
130
|
-
if (total !== void 0 && total > maxCost) {
|
|
131
|
-
process.stderr.write(`The guess passes the cap: raise --max-cost, or lower --repeat.
|
|
132
|
-
`);
|
|
133
|
-
return 1;
|
|
134
|
-
}
|
|
135
|
-
if (options.yes !== true) {
|
|
136
|
-
if (!process.stdin.isTTY) {
|
|
137
|
-
process.stderr.write("Add --yes to start without a question.\n");
|
|
138
|
-
return 1;
|
|
139
|
-
}
|
|
140
|
-
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
141
|
-
const answer = await rl.question("Start? (y/N) ");
|
|
142
|
-
rl.close();
|
|
143
|
-
if (!/^y(es)?$/i.test(answer.trim())) return 1;
|
|
144
|
-
}
|
|
145
|
-
const dir = join(process.cwd(), ".garuda", "evals", `advise-${newSessionId()}`);
|
|
146
|
-
mkdirSync(dir, { recursive: true });
|
|
147
|
-
const run = (model, label) => runEvalCommand({ model, suite: "repo", repeat, shuffle: true, seed, outDir: join(dir, label) });
|
|
148
|
-
await run(a, "a");
|
|
149
|
-
const first = await loadReport(join(dir, "a"));
|
|
150
|
-
const spent = costPerPass(first.results);
|
|
151
|
-
const spentA = spent.kind === "unknown" ? void 0 : spent.totalUsd;
|
|
152
|
-
const ratio = prices[0] === void 0 || prices[1] === void 0 ? void 0 : prices[1].input / prices[0].input;
|
|
153
|
-
if (spentA !== void 0 && ratio !== void 0 && spentA * (1 + ratio) > maxCost) {
|
|
154
|
-
process.stderr.write(
|
|
155
|
-
`Stopped before ${b}: ${a} cost $${spentA.toFixed(2)}, so ${b} would take the total past $${maxCost}.
|
|
156
|
-
`
|
|
157
|
-
);
|
|
158
|
-
return 1;
|
|
159
|
-
}
|
|
160
|
-
await run(b, "b");
|
|
161
|
-
const second = await loadReport(join(dir, "b"));
|
|
162
|
-
const result = compareReports(first, second, { seed });
|
|
163
|
-
if ("problems" in result) {
|
|
164
|
-
process.stderr.write(`The two runs cannot be compared:
|
|
165
|
-
${result.problems.join("\n")}
|
|
166
|
-
`);
|
|
167
|
-
return 1;
|
|
168
|
-
}
|
|
169
|
-
const comparison = formatComparison(first, second, result.comparison);
|
|
170
|
-
const advice = adviceText(a, b, result.comparison, tasks);
|
|
171
|
-
writeFileSync(join(dir, "advice.txt"), `${advice}
|
|
172
|
-
|
|
173
|
-
${comparison}
|
|
174
|
-
`);
|
|
175
|
-
writeFileSync(join(dir, "report.html"), adviceHtml(comparison, advice));
|
|
176
|
-
process.stdout.write(`
|
|
177
|
-
${comparison}
|
|
178
|
-
|
|
179
|
-
${advice}
|
|
180
|
-
Report: ${join(dir, "report.html")}
|
|
181
|
-
`);
|
|
182
|
-
return 0;
|
|
183
|
-
}
|
|
184
|
-
export {
|
|
185
|
-
ADVISE_MAX_COST,
|
|
186
|
-
ADVISE_MODELS,
|
|
187
|
-
ADVISE_REPEAT,
|
|
188
|
-
GUESS_INPUT_TOKENS,
|
|
189
|
-
GUESS_OUTPUT_TOKENS,
|
|
190
|
-
adviceHtml,
|
|
191
|
-
adviceText,
|
|
192
|
-
estimateUsd,
|
|
193
|
-
runAdvise
|
|
194
|
-
};
|
|
195
|
-
//# sourceMappingURL=advise-HDKJGNBL.js.map
|