@ls-stack/agent-eval 0.42.3 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{app-DPamBr5R.mjs → app-BZmhhSFZ.mjs} +10 -4
- package/dist/apps/web/dist/assets/index-BU3IqUso.css +1 -0
- package/dist/apps/web/dist/assets/index-am2Knit6.js +140 -0
- package/dist/apps/web/dist/index.html +2 -2
- package/dist/bin.mjs +1 -1
- package/dist/{cli-BeJCJMQo.mjs → cli-vdJYkEVk.mjs} +28 -5
- package/dist/index.d.mts +300 -7
- package/dist/index.mjs +3 -3
- package/dist/runChild.mjs +1 -1
- package/dist/{runOrchestration-OVUFw1fL.mjs → runOrchestration-BFdxG9ws.mjs} +259 -250
- package/dist/{runner-BJQq7cpd.mjs → runner--aH0jO4Z.mjs} +1 -1
- package/dist/{runner-B1KygirW.mjs → runner-DJWn_7p0.mjs} +2 -2
- package/dist/{src-D7_xKo7h.mjs → src-BRqs3kSA.mjs} +2 -2
- package/package.json +2 -2
- package/skills/agent-eval/SKILL.md +14 -12
- package/dist/apps/web/dist/assets/index-S3J5Nm0o.css +0 -1
- package/dist/apps/web/dist/assets/index-XLJByNnS.js +0 -140
|
@@ -25,8 +25,8 @@
|
|
|
25
25
|
href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
|
|
26
26
|
rel="stylesheet"
|
|
27
27
|
/>
|
|
28
|
-
<script type="module" crossorigin src="/assets/index-
|
|
29
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
28
|
+
<script type="module" crossorigin src="/assets/index-am2Knit6.js"></script>
|
|
29
|
+
<link rel="stylesheet" crossorigin href="/assets/index-BU3IqUso.css">
|
|
30
30
|
</head>
|
|
31
31
|
<body>
|
|
32
32
|
<div id="root"></div>
|
package/dist/bin.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as validateCharts, At as buildEvalKey, C as deriveEvalFreshness, Ct as getEvalDisplayStatus, D as loadConfig, Dt as runSummarySchema, E as resolveEvalDefaultConfig, Lt as getEvalRegistry, O as buildDeclaredColumnDefs, Ot as resolveApiCallsConfig, S as parseManualInputValues, St as getEvalTitle, T as parseEvalDiscovery, Tt as matchesTagsFilter, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as resolveArtifactPath, bt as applyDerivedCallAttributes, c as getLastRunStatuses, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, j as createFsCacheStore, jt as getCaseRowCaseKey, k as normalizeScoreDef, kt as resolveLlmCallsConfig, l as getLatestRunInfos, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as stripTerminalControlCodes, p as persistCaseDetail, s as generateRunId, u as loadPersistedRunSnapshot, v as runTouchesEval, w as loadEvalModule, wt as deriveScopedSummaryFromCases, x as buildManualInputDescriptor, y as resolveTracePresentation } from "./runOrchestration-
|
|
1
|
+
import { A as validateCharts, At as buildEvalKey, C as deriveEvalFreshness, Ct as getEvalDisplayStatus, D as loadConfig, Dt as runSummarySchema, E as resolveEvalDefaultConfig, Lt as getEvalRegistry, O as buildDeclaredColumnDefs, Ot as resolveApiCallsConfig, S as parseManualInputValues, St as getEvalTitle, T as parseEvalDiscovery, Tt as matchesTagsFilter, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as resolveArtifactPath, bt as applyDerivedCallAttributes, c as getLastRunStatuses, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, j as createFsCacheStore, jt as getCaseRowCaseKey, k as normalizeScoreDef, kt as resolveLlmCallsConfig, l as getLatestRunInfos, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as stripTerminalControlCodes, p as persistCaseDetail, s as generateRunId, u as loadPersistedRunSnapshot, v as runTouchesEval, w as loadEvalModule, wt as deriveScopedSummaryFromCases, x as buildManualInputDescriptor, y as resolveTracePresentation } from "./runOrchestration-BFdxG9ws.mjs";
|
|
2
2
|
import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
@@ -8,6 +8,29 @@ import { existsSync } from "node:fs";
|
|
|
8
8
|
import { resultify } from "t-result";
|
|
9
9
|
import { fileURLToPath } from "node:url";
|
|
10
10
|
import { spawn, spawnSync } from "node:child_process";
|
|
11
|
+
//#region ../runner/src/caseDetailLookup.ts
|
|
12
|
+
function getCaseLookupIds(caseId) {
|
|
13
|
+
const decodedResult = resultify(() => decodeURIComponent(caseId));
|
|
14
|
+
const decodedCaseId = decodedResult.error ? caseId : decodedResult.value;
|
|
15
|
+
const lookupIds = new Set([caseId, decodedCaseId]);
|
|
16
|
+
for (const lookupId of [...lookupIds]) {
|
|
17
|
+
const segments = lookupId.split("#");
|
|
18
|
+
if (segments.length < 3) continue;
|
|
19
|
+
lookupIds.add(segments.map((segment) => encodeURIComponent(segment)).join("#"));
|
|
20
|
+
}
|
|
21
|
+
return [...lookupIds];
|
|
22
|
+
}
|
|
23
|
+
function resolveCaseDetailLookup(run, caseId) {
|
|
24
|
+
const lookupIds = new Set(getCaseLookupIds(caseId));
|
|
25
|
+
for (const lookupId of lookupIds) {
|
|
26
|
+
const caseDetail = run.caseDetails.get(lookupId);
|
|
27
|
+
if (caseDetail) return caseDetail;
|
|
28
|
+
}
|
|
29
|
+
const matchingCaseRow = run.cases.find((caseRow) => lookupIds.has(getCaseRowCaseKey(caseRow)) || lookupIds.has(caseRow.caseId));
|
|
30
|
+
if (matchingCaseRow === void 0) return void 0;
|
|
31
|
+
return run.caseDetails.get(getCaseRowCaseKey(matchingCaseRow));
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
11
34
|
//#region ../runner/src/configReload.ts
|
|
12
35
|
/** Coordinates idle-only reloads for `agent-evals.config.ts` in app mode. */
|
|
13
36
|
function createConfigReloadController({ getActiveRunCount, closeRunnerWatchers, loadRunnerState, emitToDiscoveryListeners }) {
|
|
@@ -1338,7 +1361,7 @@ function createRunner({ watchForChanges = true } = {}) {
|
|
|
1338
1361
|
getCaseDetail(runId, caseId) {
|
|
1339
1362
|
const run = runs.get(runId);
|
|
1340
1363
|
if (!run) return void 0;
|
|
1341
|
-
return
|
|
1364
|
+
return resolveCaseDetailLookup(run, caseId);
|
|
1342
1365
|
},
|
|
1343
1366
|
subscribe(runId, listener) {
|
|
1344
1367
|
const run = runs.get(runId);
|
|
@@ -2072,8 +2095,8 @@ async function commandApp(args) {
|
|
|
2072
2095
|
const { serve } = await import("@hono/node-server");
|
|
2073
2096
|
const bundledWebDist = resolve(currentDir, "apps/web/dist");
|
|
2074
2097
|
if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
|
|
2075
|
-
const appModule = await import("./app-
|
|
2076
|
-
const runnerModule = await import("./runner
|
|
2098
|
+
const appModule = await import("./app-BZmhhSFZ.mjs");
|
|
2099
|
+
const runnerModule = await import("./runner--aH0jO4Z.mjs");
|
|
2077
2100
|
if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
|
|
2078
2101
|
if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
|
|
2079
2102
|
await runnerModule.initRunner();
|
|
@@ -2260,7 +2283,7 @@ async function commandCache(args) {
|
|
|
2260
2283
|
const evalIds = runner.getEvals().filter((ev) => (args.evalIds.length === 0 || args.evalIds.includes(ev.id)) && (args.files.length === 0 || args.files.some((file) => fileMatches(file, ev.filePath)))).map((ev) => ev.id);
|
|
2261
2284
|
for (const evalId of evalIds) {
|
|
2262
2285
|
const entries = await runner.listCache();
|
|
2263
|
-
const prefix = `${evalId}
|
|
2286
|
+
const prefix = `${evalId}.`;
|
|
2264
2287
|
const matching = entries.filter((entry) => entry.namespace.startsWith(prefix));
|
|
2265
2288
|
for (const entry of matching) await runner.clearCache({
|
|
2266
2289
|
namespace: entry.namespace,
|
package/dist/index.d.mts
CHANGED
|
@@ -1486,7 +1486,7 @@ declare function evalAssert(condition: unknown, message: string): asserts condit
|
|
|
1486
1486
|
/** Info accepted by `evalTracer.cache(info, fn)` for spanless value caching. */
|
|
1487
1487
|
type TraceCacheInfo = {
|
|
1488
1488
|
/** Display name used for cache listings and the default namespace. */name: string; /** Arbitrary JSON-safe value used to derive the cache key. */
|
|
1489
|
-
key: unknown; /** Override the default namespace (`${evalId}
|
|
1489
|
+
key: unknown; /** Override the default namespace (`${evalId}.${name}`). */
|
|
1490
1490
|
namespace?: string;
|
|
1491
1491
|
/**
|
|
1492
1492
|
* Include native `Blob`/`File` bytes in the cache key. By default only stable
|
|
@@ -4277,7 +4277,12 @@ declare const cacheEntrySchema: z$1.ZodObject<{
|
|
|
4277
4277
|
}, z$1.core.$strip>;
|
|
4278
4278
|
/** Persisted cache file contents. */
|
|
4279
4279
|
type CacheEntry = z$1.infer<typeof cacheEntrySchema>;
|
|
4280
|
-
/**
|
|
4280
|
+
/**
|
|
4281
|
+
* Debug-only raw key metadata stored outside the reusable cache entry.
|
|
4282
|
+
*
|
|
4283
|
+
* Debug entries mirror the serialized cache entry so inspecting one debug file
|
|
4284
|
+
* shows both the authored raw key and the persisted payload for that key.
|
|
4285
|
+
*/
|
|
4281
4286
|
declare const cacheDebugKeyEntrySchema: z$1.ZodObject<{
|
|
4282
4287
|
version: z$1.ZodLiteral<1>;
|
|
4283
4288
|
key: z$1.ZodString;
|
|
@@ -4289,8 +4294,83 @@ declare const cacheDebugKeyEntrySchema: z$1.ZodObject<{
|
|
|
4289
4294
|
operationName: z$1.ZodString;
|
|
4290
4295
|
storedAt: z$1.ZodString;
|
|
4291
4296
|
rawKey: z$1.ZodUnknown;
|
|
4297
|
+
entry: z$1.ZodObject<{
|
|
4298
|
+
version: z$1.ZodLiteral<1>;
|
|
4299
|
+
key: z$1.ZodString;
|
|
4300
|
+
namespace: z$1.ZodString;
|
|
4301
|
+
operationType: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4302
|
+
value: "value";
|
|
4303
|
+
span: "span";
|
|
4304
|
+
}>>;
|
|
4305
|
+
operationName: z$1.ZodOptional<z$1.ZodString>;
|
|
4306
|
+
spanName: z$1.ZodOptional<z$1.ZodString>;
|
|
4307
|
+
spanKind: z$1.ZodOptional<z$1.ZodString>;
|
|
4308
|
+
storedAt: z$1.ZodString;
|
|
4309
|
+
recording: z$1.ZodObject<{
|
|
4310
|
+
returnValue: z$1.ZodUnknown;
|
|
4311
|
+
finalAttributes: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4312
|
+
finalStatus: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4313
|
+
error: "error";
|
|
4314
|
+
running: "running";
|
|
4315
|
+
ok: "ok";
|
|
4316
|
+
cancelled: "cancelled";
|
|
4317
|
+
}>>;
|
|
4318
|
+
finalError: z$1.ZodOptional<z$1.ZodObject<{
|
|
4319
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4320
|
+
message: z$1.ZodString;
|
|
4321
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4322
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4323
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4324
|
+
finalErrors: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4325
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4326
|
+
message: z$1.ZodString;
|
|
4327
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4328
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4329
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4330
|
+
finalWarning: z$1.ZodOptional<z$1.ZodObject<{
|
|
4331
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4332
|
+
message: z$1.ZodString;
|
|
4333
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4334
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4335
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4336
|
+
finalWarnings: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4337
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4338
|
+
message: z$1.ZodString;
|
|
4339
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4340
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4341
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4342
|
+
ops: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
|
|
4343
|
+
kind: z$1.ZodLiteral<"setOutput">;
|
|
4344
|
+
key: z$1.ZodString;
|
|
4345
|
+
value: z$1.ZodUnknown;
|
|
4346
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4347
|
+
kind: z$1.ZodLiteral<"appendOutput">;
|
|
4348
|
+
key: z$1.ZodString;
|
|
4349
|
+
value: z$1.ZodUnknown;
|
|
4350
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4351
|
+
kind: z$1.ZodLiteral<"mergeOutput">;
|
|
4352
|
+
key: z$1.ZodString;
|
|
4353
|
+
patch: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4354
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4355
|
+
kind: z$1.ZodLiteral<"incrementOutput">;
|
|
4356
|
+
key: z$1.ZodString;
|
|
4357
|
+
delta: z$1.ZodNumber;
|
|
4358
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4359
|
+
kind: z$1.ZodLiteral<"checkpoint">;
|
|
4360
|
+
name: z$1.ZodString;
|
|
4361
|
+
data: z$1.ZodUnknown;
|
|
4362
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4363
|
+
kind: z$1.ZodLiteral<"subSpan">;
|
|
4364
|
+
span: z$1.ZodType<SerializedCacheSpan, unknown, z$1.core.$ZodTypeInternals<SerializedCacheSpan, unknown>>;
|
|
4365
|
+
}, z$1.core.$strip>], "kind">>;
|
|
4366
|
+
}, z$1.core.$strip>;
|
|
4367
|
+
}, z$1.core.$strip>;
|
|
4292
4368
|
}, z$1.core.$strip>;
|
|
4293
|
-
/**
|
|
4369
|
+
/**
|
|
4370
|
+
* Debug-only raw cache key entry.
|
|
4371
|
+
*
|
|
4372
|
+
* May contain sensitive prompt/input data and the full serialized cache entry.
|
|
4373
|
+
*/
|
|
4294
4374
|
type CacheDebugKeyEntry = z$1.infer<typeof cacheDebugKeyEntrySchema>;
|
|
4295
4375
|
/** Cache lookup response with optional debug-only raw key data. */
|
|
4296
4376
|
declare const cacheEntryWithDebugKeySchema$1: z$1.ZodObject<{
|
|
@@ -4374,11 +4454,82 @@ declare const cacheEntryWithDebugKeySchema$1: z$1.ZodObject<{
|
|
|
4374
4454
|
operationName: z$1.ZodString;
|
|
4375
4455
|
storedAt: z$1.ZodString;
|
|
4376
4456
|
rawKey: z$1.ZodUnknown;
|
|
4457
|
+
entry: z$1.ZodObject<{
|
|
4458
|
+
version: z$1.ZodLiteral<1>;
|
|
4459
|
+
key: z$1.ZodString;
|
|
4460
|
+
namespace: z$1.ZodString;
|
|
4461
|
+
operationType: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4462
|
+
value: "value";
|
|
4463
|
+
span: "span";
|
|
4464
|
+
}>>;
|
|
4465
|
+
operationName: z$1.ZodOptional<z$1.ZodString>;
|
|
4466
|
+
spanName: z$1.ZodOptional<z$1.ZodString>;
|
|
4467
|
+
spanKind: z$1.ZodOptional<z$1.ZodString>;
|
|
4468
|
+
storedAt: z$1.ZodString;
|
|
4469
|
+
recording: z$1.ZodObject<{
|
|
4470
|
+
returnValue: z$1.ZodUnknown;
|
|
4471
|
+
finalAttributes: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4472
|
+
finalStatus: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4473
|
+
error: "error";
|
|
4474
|
+
running: "running";
|
|
4475
|
+
ok: "ok";
|
|
4476
|
+
cancelled: "cancelled";
|
|
4477
|
+
}>>;
|
|
4478
|
+
finalError: z$1.ZodOptional<z$1.ZodObject<{
|
|
4479
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4480
|
+
message: z$1.ZodString;
|
|
4481
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4482
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4483
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4484
|
+
finalErrors: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4485
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4486
|
+
message: z$1.ZodString;
|
|
4487
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4488
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4489
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4490
|
+
finalWarning: z$1.ZodOptional<z$1.ZodObject<{
|
|
4491
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4492
|
+
message: z$1.ZodString;
|
|
4493
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4494
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4495
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4496
|
+
finalWarnings: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4497
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4498
|
+
message: z$1.ZodString;
|
|
4499
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4500
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4501
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4502
|
+
ops: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
|
|
4503
|
+
kind: z$1.ZodLiteral<"setOutput">;
|
|
4504
|
+
key: z$1.ZodString;
|
|
4505
|
+
value: z$1.ZodUnknown;
|
|
4506
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4507
|
+
kind: z$1.ZodLiteral<"appendOutput">;
|
|
4508
|
+
key: z$1.ZodString;
|
|
4509
|
+
value: z$1.ZodUnknown;
|
|
4510
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4511
|
+
kind: z$1.ZodLiteral<"mergeOutput">;
|
|
4512
|
+
key: z$1.ZodString;
|
|
4513
|
+
patch: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4514
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4515
|
+
kind: z$1.ZodLiteral<"incrementOutput">;
|
|
4516
|
+
key: z$1.ZodString;
|
|
4517
|
+
delta: z$1.ZodNumber;
|
|
4518
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4519
|
+
kind: z$1.ZodLiteral<"checkpoint">;
|
|
4520
|
+
name: z$1.ZodString;
|
|
4521
|
+
data: z$1.ZodUnknown;
|
|
4522
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4523
|
+
kind: z$1.ZodLiteral<"subSpan">;
|
|
4524
|
+
span: z$1.ZodType<SerializedCacheSpan, unknown, z$1.core.$ZodTypeInternals<SerializedCacheSpan, unknown>>;
|
|
4525
|
+
}, z$1.core.$strip>], "kind">>;
|
|
4526
|
+
}, z$1.core.$strip>;
|
|
4527
|
+
}, z$1.core.$strip>;
|
|
4377
4528
|
}, z$1.core.$strip>>;
|
|
4378
4529
|
}, z$1.core.$strip>;
|
|
4379
4530
|
/** Cache lookup response returned by cache APIs when raw-key debug data exists. */
|
|
4380
4531
|
type CacheEntryWithDebugKey = z$1.infer<typeof cacheEntryWithDebugKeySchema$1>;
|
|
4381
|
-
/**
|
|
4532
|
+
/** Legacy aggregate cache file shape retained for API compatibility. */
|
|
4382
4533
|
declare const cacheFileSchema: z$1.ZodObject<{
|
|
4383
4534
|
version: z$1.ZodLiteral<1>;
|
|
4384
4535
|
owner: z$1.ZodString;
|
|
@@ -4454,9 +4605,9 @@ declare const cacheFileSchema: z$1.ZodObject<{
|
|
|
4454
4605
|
}, z$1.core.$strip>;
|
|
4455
4606
|
}, z$1.core.$strip>>;
|
|
4456
4607
|
}, z$1.core.$strip>;
|
|
4457
|
-
/**
|
|
4608
|
+
/** Legacy aggregate cache file contents retained for API compatibility. */
|
|
4458
4609
|
type CacheFile = z$1.infer<typeof cacheFileSchema>;
|
|
4459
|
-
/**
|
|
4610
|
+
/** Legacy aggregate debug file shape retained for API compatibility. */
|
|
4460
4611
|
declare const cacheDebugKeyFileSchema: z$1.ZodObject<{
|
|
4461
4612
|
version: z$1.ZodLiteral<1>;
|
|
4462
4613
|
owner: z$1.ZodString;
|
|
@@ -4471,9 +4622,80 @@ declare const cacheDebugKeyFileSchema: z$1.ZodObject<{
|
|
|
4471
4622
|
operationName: z$1.ZodString;
|
|
4472
4623
|
storedAt: z$1.ZodString;
|
|
4473
4624
|
rawKey: z$1.ZodUnknown;
|
|
4625
|
+
entry: z$1.ZodObject<{
|
|
4626
|
+
version: z$1.ZodLiteral<1>;
|
|
4627
|
+
key: z$1.ZodString;
|
|
4628
|
+
namespace: z$1.ZodString;
|
|
4629
|
+
operationType: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4630
|
+
value: "value";
|
|
4631
|
+
span: "span";
|
|
4632
|
+
}>>;
|
|
4633
|
+
operationName: z$1.ZodOptional<z$1.ZodString>;
|
|
4634
|
+
spanName: z$1.ZodOptional<z$1.ZodString>;
|
|
4635
|
+
spanKind: z$1.ZodOptional<z$1.ZodString>;
|
|
4636
|
+
storedAt: z$1.ZodString;
|
|
4637
|
+
recording: z$1.ZodObject<{
|
|
4638
|
+
returnValue: z$1.ZodUnknown;
|
|
4639
|
+
finalAttributes: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4640
|
+
finalStatus: z$1.ZodOptional<z$1.ZodEnum<{
|
|
4641
|
+
error: "error";
|
|
4642
|
+
running: "running";
|
|
4643
|
+
ok: "ok";
|
|
4644
|
+
cancelled: "cancelled";
|
|
4645
|
+
}>>;
|
|
4646
|
+
finalError: z$1.ZodOptional<z$1.ZodObject<{
|
|
4647
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4648
|
+
message: z$1.ZodString;
|
|
4649
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4650
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4651
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4652
|
+
finalErrors: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4653
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4654
|
+
message: z$1.ZodString;
|
|
4655
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4656
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4657
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4658
|
+
finalWarning: z$1.ZodOptional<z$1.ZodObject<{
|
|
4659
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4660
|
+
message: z$1.ZodString;
|
|
4661
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4662
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4663
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
4664
|
+
finalWarnings: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
4665
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
4666
|
+
message: z$1.ZodString;
|
|
4667
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
4668
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
4669
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
4670
|
+
ops: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
|
|
4671
|
+
kind: z$1.ZodLiteral<"setOutput">;
|
|
4672
|
+
key: z$1.ZodString;
|
|
4673
|
+
value: z$1.ZodUnknown;
|
|
4674
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4675
|
+
kind: z$1.ZodLiteral<"appendOutput">;
|
|
4676
|
+
key: z$1.ZodString;
|
|
4677
|
+
value: z$1.ZodUnknown;
|
|
4678
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4679
|
+
kind: z$1.ZodLiteral<"mergeOutput">;
|
|
4680
|
+
key: z$1.ZodString;
|
|
4681
|
+
patch: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
4682
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4683
|
+
kind: z$1.ZodLiteral<"incrementOutput">;
|
|
4684
|
+
key: z$1.ZodString;
|
|
4685
|
+
delta: z$1.ZodNumber;
|
|
4686
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4687
|
+
kind: z$1.ZodLiteral<"checkpoint">;
|
|
4688
|
+
name: z$1.ZodString;
|
|
4689
|
+
data: z$1.ZodUnknown;
|
|
4690
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
4691
|
+
kind: z$1.ZodLiteral<"subSpan">;
|
|
4692
|
+
span: z$1.ZodType<SerializedCacheSpan, unknown, z$1.core.$ZodTypeInternals<SerializedCacheSpan, unknown>>;
|
|
4693
|
+
}, z$1.core.$strip>], "kind">>;
|
|
4694
|
+
}, z$1.core.$strip>;
|
|
4695
|
+
}, z$1.core.$strip>;
|
|
4474
4696
|
}, z$1.core.$strip>>;
|
|
4475
4697
|
}, z$1.core.$strip>;
|
|
4476
|
-
/**
|
|
4698
|
+
/** Legacy aggregate raw cache key debug file contents retained for compatibility. */
|
|
4477
4699
|
type CacheDebugKeyFile = z$1.infer<typeof cacheDebugKeyFileSchema>; //#endregion
|
|
4478
4700
|
//#region src/utils/extractCacheHits.d.ts
|
|
4479
4701
|
/**
|
|
@@ -5707,6 +5929,77 @@ declare const cacheEntryWithDebugKeySchema: z$1.ZodObject<{
|
|
|
5707
5929
|
operationName: z$1.ZodString;
|
|
5708
5930
|
storedAt: z$1.ZodString;
|
|
5709
5931
|
rawKey: z$1.ZodUnknown;
|
|
5932
|
+
entry: z$1.ZodObject<{
|
|
5933
|
+
version: z$1.ZodLiteral<1>;
|
|
5934
|
+
key: z$1.ZodString;
|
|
5935
|
+
namespace: z$1.ZodString;
|
|
5936
|
+
operationType: z$1.ZodOptional<z$1.ZodEnum<{
|
|
5937
|
+
span: "span";
|
|
5938
|
+
value: "value";
|
|
5939
|
+
}>>;
|
|
5940
|
+
operationName: z$1.ZodOptional<z$1.ZodString>;
|
|
5941
|
+
spanName: z$1.ZodOptional<z$1.ZodString>;
|
|
5942
|
+
spanKind: z$1.ZodOptional<z$1.ZodString>;
|
|
5943
|
+
storedAt: z$1.ZodString;
|
|
5944
|
+
recording: z$1.ZodObject<{
|
|
5945
|
+
returnValue: z$1.ZodUnknown;
|
|
5946
|
+
finalAttributes: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
5947
|
+
finalStatus: z$1.ZodOptional<z$1.ZodEnum<{
|
|
5948
|
+
error: "error";
|
|
5949
|
+
running: "running";
|
|
5950
|
+
ok: "ok";
|
|
5951
|
+
cancelled: "cancelled";
|
|
5952
|
+
}>>;
|
|
5953
|
+
finalError: z$1.ZodOptional<z$1.ZodObject<{
|
|
5954
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
5955
|
+
message: z$1.ZodString;
|
|
5956
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
5957
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
5958
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
5959
|
+
finalErrors: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
5960
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
5961
|
+
message: z$1.ZodString;
|
|
5962
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
5963
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
5964
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
5965
|
+
finalWarning: z$1.ZodOptional<z$1.ZodObject<{
|
|
5966
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
5967
|
+
message: z$1.ZodString;
|
|
5968
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
5969
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
5970
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>;
|
|
5971
|
+
finalWarnings: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
|
|
5972
|
+
name: z$1.ZodOptional<z$1.ZodString>;
|
|
5973
|
+
message: z$1.ZodString;
|
|
5974
|
+
stack: z$1.ZodOptional<z$1.ZodString>;
|
|
5975
|
+
capturedAt: z$1.ZodOptional<z$1.ZodString>;
|
|
5976
|
+
}, z$1.core.$catchall<z$1.ZodUnknown>>>>;
|
|
5977
|
+
ops: z$1.ZodArray<z$1.ZodDiscriminatedUnion<[z$1.ZodObject<{
|
|
5978
|
+
kind: z$1.ZodLiteral<"setOutput">;
|
|
5979
|
+
key: z$1.ZodString;
|
|
5980
|
+
value: z$1.ZodUnknown;
|
|
5981
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
5982
|
+
kind: z$1.ZodLiteral<"appendOutput">;
|
|
5983
|
+
key: z$1.ZodString;
|
|
5984
|
+
value: z$1.ZodUnknown;
|
|
5985
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
5986
|
+
kind: z$1.ZodLiteral<"mergeOutput">;
|
|
5987
|
+
key: z$1.ZodString;
|
|
5988
|
+
patch: z$1.ZodRecord<z$1.ZodString, z$1.ZodUnknown>;
|
|
5989
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
5990
|
+
kind: z$1.ZodLiteral<"incrementOutput">;
|
|
5991
|
+
key: z$1.ZodString;
|
|
5992
|
+
delta: z$1.ZodNumber;
|
|
5993
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
5994
|
+
kind: z$1.ZodLiteral<"checkpoint">;
|
|
5995
|
+
name: z$1.ZodString;
|
|
5996
|
+
data: z$1.ZodUnknown;
|
|
5997
|
+
}, z$1.core.$strip>, z$1.ZodObject<{
|
|
5998
|
+
kind: z$1.ZodLiteral<"subSpan">;
|
|
5999
|
+
span: z$1.ZodType<SerializedCacheSpan$1, unknown, z$1.core.$ZodTypeInternals<SerializedCacheSpan$1, unknown>>;
|
|
6000
|
+
}, z$1.core.$strip>], "kind">>;
|
|
6001
|
+
}, z$1.core.$strip>;
|
|
6002
|
+
}, z$1.core.$strip>;
|
|
5710
6003
|
}, z$1.core.$strip>>;
|
|
5711
6004
|
}, z$1.core.$strip>;
|
|
5712
6005
|
/** Cache lookup response returned by cache APIs when raw-key debug data exists. */
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as getCurrentScope, B as deserializeCacheValue, F as evalSpan, G as readManualInputFile, H as serializeCacheValue, I as evalTracer, J as appendToEvalOutput, K as evalExpect, L as hashCacheKey, Lt as getEvalRegistry, M as z, N as buildTraceTree, P as captureEvalSpanError, Q as evalTime, R as hashCacheKeySync, U as repoFile, V as serializeCacheRecording, W as manualInputFileValueSchema, X as evalAssert, Z as evalLog, _t as extractLlmCalls, at as nextEvalId, ct as runInExistingEvalScope, dt as startEvalBackgroundJob, et as getEvalCaseInput, gt as extractApiCalls, ht as extractCacheHits, it as mergeEvalOutput, lt as setEvalOutput, mt as extractCacheEntries, nt as isInEvalScope, ot as runInEvalRuntimeScope, q as EvalAssertionError, st as runInEvalScope, tt as incrementEvalOutput, ut as setScopeCacheContext, vt as simulateLlmCallCost, xt as getNestedAttribute, yt as simulateTokenAllocation, z as deserializeCacheRecording } from "./runOrchestration-
|
|
2
|
-
import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-
|
|
3
|
-
import { n as matchesEvalTags, t as defineEval } from "./src-
|
|
1
|
+
import { $ as getCurrentScope, B as deserializeCacheValue, F as evalSpan, G as readManualInputFile, H as serializeCacheValue, I as evalTracer, J as appendToEvalOutput, K as evalExpect, L as hashCacheKey, Lt as getEvalRegistry, M as z, N as buildTraceTree, P as captureEvalSpanError, Q as evalTime, R as hashCacheKeySync, U as repoFile, V as serializeCacheRecording, W as manualInputFileValueSchema, X as evalAssert, Z as evalLog, _t as extractLlmCalls, at as nextEvalId, ct as runInExistingEvalScope, dt as startEvalBackgroundJob, et as getEvalCaseInput, gt as extractApiCalls, ht as extractCacheHits, it as mergeEvalOutput, lt as setEvalOutput, mt as extractCacheEntries, nt as isInEvalScope, ot as runInEvalRuntimeScope, q as EvalAssertionError, st as runInEvalScope, tt as incrementEvalOutput, ut as setScopeCacheContext, vt as simulateLlmCallCost, xt as getNestedAttribute, yt as simulateTokenAllocation, z as deserializeCacheRecording } from "./runOrchestration-BFdxG9ws.mjs";
|
|
2
|
+
import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-vdJYkEVk.mjs";
|
|
3
|
+
import { n as matchesEvalTags, t as defineEval } from "./src-BRqs3kSA.mjs";
|
|
4
4
|
export { EvalAssertionError, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, z };
|
package/dist/runChild.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { At as buildEvalKey, D as loadConfig, Dt as runSummarySchema, Et as runManifestSchema, Ft as columnDefSchema, Mt as evalStatsConfigSchema, Nt as manualInputDescriptorSchema, Pt as evalChartsConfigSchema, T as parseEvalDiscovery, Y as configureEvalRunLogs, ft as createRunRequestSchema, h as persistRunState, j as createFsCacheStore, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-
|
|
1
|
+
import { At as buildEvalKey, D as loadConfig, Dt as runSummarySchema, Et as runManifestSchema, Ft as columnDefSchema, Mt as evalStatsConfigSchema, Nt as manualInputDescriptorSchema, Pt as evalChartsConfigSchema, T as parseEvalDiscovery, Y as configureEvalRunLogs, ft as createRunRequestSchema, h as persistRunState, j as createFsCacheStore, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-BFdxG9ws.mjs";
|
|
2
2
|
import { z } from "zod/v4";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
4
|
import { relative } from "node:path";
|