npm - @ls-stack/agent-eval - Versions diffs - 0.55.1 → 0.55.2 - Mend

@ls-stack/agent-eval 0.55.1 → 0.55.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (16) hide show

package/dist/{app-BD0D9-7k.mjs → app-NI4to6lp.mjs} +4 -4
package/dist/apps/web/dist/assets/{index-CvsPmlHl.js → index-C7QjETk8.js} +54 -54
package/dist/apps/web/dist/index.html +1 -1
package/dist/bin.mjs +1 -1
package/dist/caseChild.mjs +1 -1
package/dist/{cli-BR3wMZMx.mjs → cli-Bu9347r1.mjs} +60 -13
package/dist/index.d.mts +37 -24
package/dist/index.mjs +3 -3
package/dist/runChild.mjs +2 -2
package/dist/{runExecution-Sw38bCaq.mjs → runExecution-C31dpemR.mjs} +363 -114
package/dist/{runOrchestration-DJsdLYeZ.mjs → runOrchestration-3RoHLW4U.mjs} +1 -1
package/dist/{runner-72rsqJRq.mjs → runner-B4EfMn1d.mjs} +2 -2
package/dist/{runner-dB69WsnM.mjs → runner-CTp9zHbM.mjs} +1 -1
package/dist/{src-hBGtzWuA.mjs → src-FR60ZR_4.mjs} +2 -2
package/package.json +3 -3
package/skills/agent-eval/SKILL.md +19 -8

package/dist/apps/web/dist/index.html CHANGED Viewed

@@ -25,7 +25,7 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
       rel="stylesheet"
     />
-    <script type="module" crossorigin src="/assets/index-CvsPmlHl.js"></script>
+    <script type="module" crossorigin src="/assets/index-C7QjETk8.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-2I-eWzVL.css">
   </head>
   <body>

package/dist/bin.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-BR3wMZMx.mjs";
+import { t as runCli } from "./cli-Bu9347r1.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/caseChild.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-import { I as configureEvalRunLogs, Pt as runWithEvalRegistry, St as resolveLlmCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as createFsCacheStore, xt as resolveApiCallsConfig } from "./runExecution-Sw38bCaq.mjs";
+import { I as configureEvalRunLogs, Pt as runWithEvalRegistry, St as resolveLlmCallsConfig, _ as createBufferedCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as createFsCacheStore, xt as resolveApiCallsConfig } from "./runExecution-C31dpemR.mjs";
 //#region ../runner/src/caseChild.ts
 let fatalErrorReported = false;
 let disconnectExpected = false;

package/dist/{cli-BR3wMZMx.mjs → cli-Bu9347r1.mjs} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { Ct as buildEvalKey, Nt as getEvalRegistry, St as resolveLlmCallsConfig, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as normalizeScoreDef, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as createFsCacheStore, wt as getCaseRowCaseKey, xt as resolveApiCallsConfig } from "./runExecution-Sw38bCaq.mjs";
-import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-DJsdLYeZ.mjs";
+import { Ct as buildEvalKey, Nt as getEvalRegistry, St as resolveLlmCallsConfig, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as normalizeScoreDef, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as createFsCacheStore, wt as getCaseRowCaseKey, xt as resolveApiCallsConfig } from "./runExecution-C31dpemR.mjs";
+import { C as validateCharts, S as parseEvalDiscovery, _ as runTouchesEval, a as validateTagsFilters, b as deriveEvalFreshness, c as getLatestRunInfos, d as nextShortIdFromSnapshots, f as persistCaseDetail, g as recomputePersistedCaseStatus, h as recomputeEvalStatusesInRuns, i as resolveEvalTags, l as loadPersistedRunSnapshot, m as persistRunState, n as getTargetEvalKeys, o as generateRunId, p as deleteTemporaryRuns, s as getLastRunStatuses, u as loadPersistedRunSnapshots, v as buildManualInputDescriptor, x as loadIsolatedEvalRegistry, y as parseManualInputValues } from "./runOrchestration-3RoHLW4U.mjs";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
@@ -794,6 +794,7 @@ async function markRunErrored(runState, message, managerContext) {
 		payload: { message }
 	});
 	managerContext.emitDiscoveryEvent();
+	managerContext.onRunTerminal();
 }
 async function markRunTerminalFromChild(runState, event, managerContext) {
 	const snapshot = await loadPersistedRunSnapshot(runState.runDir);
@@ -815,6 +816,7 @@ async function markRunTerminalFromChild(runState, event, managerContext) {
 	}
 	managerContext.emitEvent(runState, event);
 	managerContext.emitDiscoveryEvent();
+	managerContext.onRunTerminal();
 }
 //#endregion
 //#region ../runner/src/runnerStateHydration.ts
@@ -883,6 +885,7 @@ function getWatchRootsForIncludePatterns(params) {
 }
 //#endregion
 //#region ../runner/src/runner.ts
+const defaultCachePruneIdleDelayMs = 5e3;
 /** Create an in-memory eval runner bound to the current workspace config. */
 function createRunner({ watchForChanges = true } = {}) {
 	let config;
@@ -903,6 +906,7 @@ function createRunner({ watchForChanges = true } = {}) {
 	let runHistoryWatcher;
 	let discoveryRefreshTimer;
 	let runHistoryRefreshTimer;
+	let cachePruneIdleTimer;
 	let registryLoadCounter = 0;
 	const configReload = createConfigReloadController({
 		getActiveRunCount,
@@ -942,6 +946,9 @@ function createRunner({ watchForChanges = true } = {}) {
 		async clearCache(filter) {
 			await cacheStore.clear(filter);
 		},
+		async repairCache() {
+			return cacheStore.repair();
+		},
 		async recomputeStatusesForEval(evalKey) {
 			const evalMeta = resolveEvalMeta(evalKey);
 			if (!evalMeta) return { updatedRuns: 0 };
@@ -1252,6 +1259,7 @@ function createRunner({ watchForChanges = true } = {}) {
 			emitDiscoveryEvent();
 		},
 		async startRun(request) {
+			cancelCacheRetentionPrune();
 			const tagsFilterError = validateTagsFilters(request.target.tagsFilter);
 			if (tagsFilterError !== null) throw new Error(tagsFilterError);
 			const deletedTemporaryRuns = await deleteTemporaryRuns({
@@ -1349,7 +1357,8 @@ function createRunner({ watchForChanges = true } = {}) {
 					workspaceRoot,
 					evals,
 					emitEvent,
-					emitDiscoveryEvent
+					emitDiscoveryEvent,
+					onRunTerminal: scheduleCacheRetentionPrune
 				}
 			});
 			if (deletedTemporaryRuns > 0) emitDiscoveryEvent();
@@ -1398,6 +1407,7 @@ function createRunner({ watchForChanges = true } = {}) {
 				payload: run.summary
 			});
 			emitDiscoveryEvent();
+			scheduleCacheRetentionPrune();
 		},
 		getCaseDetail(runId, caseId) {
 			const run = runs.get(runId);
@@ -1419,6 +1429,7 @@ function createRunner({ watchForChanges = true } = {}) {
 			};
 		},
 		async close() {
+			cancelCacheRetentionPrune();
 			await Promise.all([closeWatchers(), configReload.close()]);
 		},
 		getWorkspaceRoot() {
@@ -1522,6 +1533,28 @@ function createRunner({ watchForChanges = true } = {}) {
 	function getActiveRunCount() {
 		return [...runs.values()].filter((run) => run.manifest.status === "running").length;
 	}
+	function getCachePruneIdleDelayMs() {
+		const configured = config.cache?.pruneIdleDelayMs;
+		if (configured === void 0 || !Number.isFinite(configured) || configured <= 0) return defaultCachePruneIdleDelayMs;
+		return Math.floor(configured);
+	}
+	function cancelCacheRetentionPrune() {
+		if (cachePruneIdleTimer === void 0) return;
+		clearTimeout(cachePruneIdleTimer);
+		cachePruneIdleTimer = void 0;
+	}
+	function scheduleCacheRetentionPrune() {
+		cancelCacheRetentionPrune();
+		cachePruneIdleTimer = setTimeout(() => {
+			cachePruneIdleTimer = void 0;
+			pruneCacheRetentionIfIdle();
+		}, getCachePruneIdleDelayMs());
+		cachePruneIdleTimer.unref();
+	}
+	async function pruneCacheRetentionIfIdle() {
+		if (getActiveRunCount() > 0) return;
+		await cacheStore.pruneRetention();
+	}
 	function emitDiscoveryEvent() {
 		const lastRunStatuses = getLastRunStatuses({
 			runs: runs.values(),
@@ -1657,19 +1690,20 @@ Flags:
   `);
 		return;
 	}
-	if (topic === "cache" || topic === "cache list" || topic === "cache clear") {
+	if (topic === "cache" || topic === "cache list" || topic === "cache clear" || topic === "cache repair") {
 		console.info(`
-agent-evals cache - Manage cached operation entries
+agent-evals cache - Manage cached namespace/key entries
 Usage:
   agent-evals cache list [flags]
   agent-evals cache clear --eval <id>
   agent-evals cache clear --all
+  agent-evals cache repair [flags]
 Flags:
   --eval <id>                Clear entries for specific eval(s) (comma-separated)
   --all                      Confirm clearing every cached entry
-  --json                     Output cache listing as JSON
+  --json                     Output cache listing or repair summary as JSON
   --no-env                   Disable automatic .env loading
   --help, -h                 Show this help
   `);
@@ -1683,9 +1717,10 @@ Commands:
   list                       List discovered evals
   run                        Run evals
   show-runs [id|latest]      Show saved run artifact file paths
-  cache list                 List cached operation entries
+  cache list                 List cached namespace/key entries
   cache clear --eval <id>    Clear cache entries for one eval
   cache clear --all          Clear every cached entry
+  cache repair               Remove unindexed/orphaned cache files
   help                       Show this help
 Options:
@@ -1961,7 +1996,7 @@ function parseArgs(argv) {
 	let cursor = 1;
 	if (args.command === "cache") {
 		const sub = normalizedArgv[cursor];
-		if (sub === "list" || sub === "clear") {
+		if (sub === "list" || sub === "clear" || sub === "repair") {
 			args.subcommand = sub;
 			args.helpTopic = `cache ${sub}`;
 			cursor++;
@@ -2136,8 +2171,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-BD0D9-7k.mjs");
-	const runnerModule = await import("./runner-dB69WsnM.mjs");
+	const appModule = await import("./app-NI4to6lp.mjs");
+	const runnerModule = await import("./runner-CTp9zHbM.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner();
@@ -2311,10 +2346,8 @@ async function commandCache(args) {
 		for (const entry of entries) {
 			console.info(`  ${entry.namespace}`);
 			console.info(`    key: ${entry.key}`);
-			const operationLabel = entry.operationType === "span" ? `${entry.operationName} (span ${entry.spanKind ?? "unknown"})` : `${entry.operationName} (value)`;
-			console.info(`    operation: ${operationLabel}`);
 			console.info(`    stored: ${entry.storedAt}`);
-			console.info(`    size: ${String(entry.sizeBytes)} bytes`);
+			console.info(`    last accessed: ${entry.lastAccessedAt}`);
 			console.info("");
 		}
 		return;
@@ -2342,6 +2375,20 @@ async function commandCache(args) {
 		console.info("Refusing to clear cache without --eval <id> or --all. Use one of these flags to confirm.");
 		process.exit(1);
 	}
+	if (args.subcommand === "repair") {
+		const summary = await runner.repairCache();
+		if (args.json) {
+			console.info(JSON.stringify(summary, null, 2));
+			return;
+		}
+		console.info("Cache repair complete.");
+		console.info(`Removed cache files: ${String(summary.removedCacheFiles)}`);
+		console.info(`Removed debug files: ${String(summary.removedDebugFiles)}`);
+		console.info(`Removed blob files: ${String(summary.removedBlobFiles)}`);
+		console.info(`Removed index rows: ${String(summary.removedIndexRows)}`);
+		console.info(`Rewritten indexes: ${String(summary.rewrittenIndexes)}`);
+		return;
+	}
 	printHelp(args.helpTopic);
 }
 function getSortedRunSnapshots(runner) {

package/dist/index.d.mts CHANGED Viewed

@@ -1399,7 +1399,7 @@ type EvalLogLevelInput = RunLogLevel$1 | 'warning';
 declare class EvalAssertionError extends Error {
   constructor(message: string);
 }
-/** Return the shifted wall-clock time for a stored eval clock state. */
+/** Return the host process clock, bypassing the eval Date shim. */
 /**
  * Eval time helpers for reading and moving the active eval clock.
  *
@@ -4042,7 +4042,13 @@ type AgentEvalsConfig$1 = {
      * Exact namespace-specific retention caps. Values override
      * `maxEntriesPerNamespace` for matching namespaces.
      */
-    maxEntriesByNamespace?: Record<string, number>; /** Legacy alias for `maxEntriesPerNamespace`, retained so older config files keep working. */
+    maxEntriesByNamespace?: Record<string, number>;
+    /**
+     * Milliseconds the runner waits after becoming idle before pruning indexed
+     * cache entries. Defaults to `5000`; non-positive or non-finite values use
+     * the default.
+     */
+    pruneIdleDelayMs?: number; /** Legacy alias for `maxEntriesPerNamespace`, retained so older config files keep working. */
     maxEntriesPerEval?: number;
   };
 };
@@ -4304,22 +4310,25 @@ declare const traceCacheRefSchema: z$1.ZodObject<{
 }, z$1.core.$strip>;
 /** Reference to a value-cache lookup performed via `evalTracer.cache(...)`. */
 type TraceCacheRef$1 = z$1.infer<typeof traceCacheRefSchema>;
-/** Summary of a single persisted cache entry, used by list/delete endpoints. */
+/** Minimal index-backed summary of a persisted cache entry. */
 declare const cacheListItemSchema$1: z$1.ZodObject<{
   key: z$1.ZodString;
   namespace: z$1.ZodString;
-  operationType: z$1.ZodEnum<{
-    value: "value";
-    span: "span";
-  }>;
-  operationName: z$1.ZodString;
-  spanName: z$1.ZodOptional<z$1.ZodString>;
-  spanKind: z$1.ZodOptional<z$1.ZodString>;
   storedAt: z$1.ZodString;
-  sizeBytes: z$1.ZodNumber;
+  lastAccessedAt: z$1.ZodString;
 }, z$1.core.$strip>;
-/** Summary row for a single cache entry. */
+/** Minimal summary row for a single cache entry. */
 type CacheListItem = z$1.infer<typeof cacheListItemSchema$1>;
+/** Summary of cleanup performed by manual cache repair. */
+declare const cacheRepairSummarySchema$1: z$1.ZodObject<{
+  removedCacheFiles: z$1.ZodNumber;
+  removedDebugFiles: z$1.ZodNumber;
+  removedBlobFiles: z$1.ZodNumber;
+  removedIndexRows: z$1.ZodNumber;
+  rewrittenIndexes: z$1.ZodNumber;
+}, z$1.core.$strip>;
+/** Stable JSON summary returned by manual cache repair. */
+type CacheRepairSummary = z$1.infer<typeof cacheRepairSummarySchema$1>;
 /** Serialized nested span captured while recording a cached operation. */
 type SerializedCacheSpan = {
   kind: string;
@@ -6097,22 +6106,25 @@ type ResolvedLlmCallCostCurrency$1 = {
   numberFormat?: NumberDisplayOptions$1;
 }; //#endregion
 //#region ../shared/src/schemas/cache.d.ts
-/** Summary of a single persisted cache entry, used by list/delete endpoints. */
+/** Minimal index-backed summary of a persisted cache entry. */
 declare const cacheListItemSchema: z$1.ZodObject<{
   key: z$1.ZodString;
   namespace: z$1.ZodString;
-  operationType: z$1.ZodEnum<{
-    span: "span";
-    value: "value";
-  }>;
-  operationName: z$1.ZodString;
-  spanName: z$1.ZodOptional<z$1.ZodString>;
-  spanKind: z$1.ZodOptional<z$1.ZodString>;
   storedAt: z$1.ZodString;
-  sizeBytes: z$1.ZodNumber;
+  lastAccessedAt: z$1.ZodString;
 }, z$1.core.$strip>;
-/** Summary row for a single cache entry. */
+/** Minimal summary row for a single cache entry. */
 type CacheListItem$1 = z$1.infer<typeof cacheListItemSchema>;
+/** Summary of cleanup performed by manual cache repair. */
+declare const cacheRepairSummarySchema: z$1.ZodObject<{
+  removedCacheFiles: z$1.ZodNumber;
+  removedDebugFiles: z$1.ZodNumber;
+  removedBlobFiles: z$1.ZodNumber;
+  removedIndexRows: z$1.ZodNumber;
+  rewrittenIndexes: z$1.ZodNumber;
+}, z$1.core.$strip>;
+/** Stable JSON summary returned by manual cache repair. */
+type CacheRepairSummary$1 = z$1.infer<typeof cacheRepairSummarySchema>;
 /** Serialized nested span captured while recording a cached operation. */
 type SerializedCacheSpan$1 = {
   kind: string;
@@ -6454,7 +6466,8 @@ type EvalRunner = {
    * Remove cache entries matching `filter`, or all entries when no filter is
    * supplied.
    */
-  clearCache(filter?: CacheClearFilter): Promise<void>;
+  clearCache(filter?: CacheClearFilter): Promise<void>; /** Remove cache/debug/blob files that are not referenced by cache indexes. */
+  repairCache(): Promise<CacheRepairSummary$1>;
   /**
    * Recompute persisted case and run statuses for terminal runs touching one
    * eval. Accepts the exact eval key.
@@ -6635,4 +6648,4 @@ declare function defineEval<TInput = unknown, TOutputs extends EvalOutputs = Eva
 /** Return whether the active eval case has tags matching the typed input. */
 declare function matchesEvalTags(input: EvalTagMatchInput): boolean;
 //#endregion
-export { AgentEvalTagRegistry, AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheActivityEntry, type CacheAdapter, type CacheDebugKeyEntry, type CacheDebugKeyFile, type CacheDebugKeyWrite, type CacheEntry, type CacheEntryWithDebugKey, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheScopeContext, type CacheSerializationOptions, type CacheStatus, type CallDerivedAttribute, type CallDerivedAttributeContext, type CallDerivedAttributesConfig, type CallDerivedAttributesFn, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type ConfigReloadState, type ConfigReloadStatus, type CreateRunRequest, type DefaultConfigKey, type DerivedStatus, type DiscoveryIssue, EvalAssertionError, type EvalCacheConfig, EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, EvalDefinition, type EvalDeriveConfig, type EvalDeriveContext, type EvalDeriveFn, type EvalDeriveMap, type EvalDeriveValueFn, type EvalDisplayStatus, type EvalExecuteContext, type EvalExpectation, type EvalFreshnessStatus, type EvalManualInputConfig, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalRuntimeScope, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStartTime, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, EvalTag, EvalTagMatchInput, type EvalTraceTree, type JsonCell, type LlmCallCostBreakdown, type LlmCallCostCurrency, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallPricing, type LlmCallPricingRate, type LlmCallPricingRegistry, type LlmCallSimulatedTokens, type LlmCallsConfigInput, type LlmCostScenario, type ManualInputDescriptor, type ManualInputFieldDescriptor, type ManualInputFieldKind, type ManualInputFieldOverride, type ManualInputFieldsConfig, type ManualInputFileValue, type ManualInputSelectOption, type MaterializeManualInputFilesResult, type NumberDisplayOptions, type ReadManualInputFileResult, type RemoveDefaultConfig, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedCallDerivedAttribute, type ResolvedLlmCallCostCurrency, type ResolvedLlmCallMetric, type ResolvedLlmCallPricing, type ResolvedLlmCallsConfig, type RunInEvalScopeOptions, type RunLogEntry, type RunLogLevel, type RunLogLocation, type RunLogPhase, type RunLogsConfigInput, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SerializedCacheValue, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, z };
+export { AgentEvalTagRegistry, AgentEvalsConfig, type ApiCallEntry, type ApiCallMetric, type ApiCallMetricFormat, type ApiCallMetricPlacement, type ApiCallMetricValue, type ApiCallsConfigInput, type AssertionFailure, type CacheActivityEntry, type CacheAdapter, type CacheDebugKeyEntry, type CacheDebugKeyFile, type CacheDebugKeyWrite, type CacheEntry, type CacheEntryWithDebugKey, type CacheFile, type CacheHitEntry, type CacheKeyHashInput, type CacheKeyHashOptions, type CacheListItem, type CacheMode, type CacheOperationType, type CacheRecording, type CacheRecordingFrame, type CacheRecordingOp, type CacheRepairSummary, type CacheScopeContext, type CacheSerializationOptions, type CacheStatus, type CallDerivedAttribute, type CallDerivedAttributeContext, type CallDerivedAttributesConfig, type CallDerivedAttributesFn, type CaptureEvalSpanErrorLevel, type CaptureEvalSpanErrorOptions, type CaseDetail, type CaseRow, type CellValue, type ColumnDef, type ColumnFormat, type ColumnKind, type ConfigReloadState, type ConfigReloadStatus, type CreateRunRequest, type DefaultConfigKey, type DerivedStatus, type DiscoveryIssue, EvalAssertionError, type EvalCacheConfig, EvalCase, type EvalCaseScope, type EvalChartAggregate, type EvalChartAxis, type EvalChartBuiltinMetric, type EvalChartColor, type EvalChartConfig, type EvalChartMetric, type EvalChartTooltipExtra, type EvalChartType, type EvalChartsConfig, type EvalColumnOverride, type EvalColumns, EvalDefinition, type EvalDeriveConfig, type EvalDeriveContext, type EvalDeriveFn, type EvalDeriveMap, type EvalDeriveValueFn, type EvalDisplayStatus, type EvalExecuteContext, type EvalExpectation, type EvalFreshnessStatus, type EvalManualInputConfig, type EvalManualScoreDef, type EvalOutputs, type EvalOutputsSchema, type EvalRunner, type EvalRuntimeScope, type EvalScoreContext, type EvalScoreDef, type EvalScoreFn, type EvalSetOutput, type EvalStartTime, type EvalStatAggregate, type EvalStatItem, type EvalStatsConfig, type EvalSummary, EvalTag, EvalTagMatchInput, type EvalTraceTree, type JsonCell, type LlmCallCostBreakdown, type LlmCallCostCurrency, type LlmCallEntry, type LlmCallMetric, type LlmCallMetricFormat, type LlmCallMetricPlacement, type LlmCallMetricValue, type LlmCallPricing, type LlmCallPricingRate, type LlmCallPricingRegistry, type LlmCallSimulatedTokens, type LlmCallsConfigInput, type LlmCostScenario, type ManualInputDescriptor, type ManualInputFieldDescriptor, type ManualInputFieldKind, type ManualInputFieldOverride, type ManualInputFieldsConfig, type ManualInputFileValue, type ManualInputSelectOption, type MaterializeManualInputFilesResult, type NumberDisplayOptions, type ReadManualInputFileResult, type RemoveDefaultConfig, type ResolvedApiCallMetric, type ResolvedApiCallsConfig, type ResolvedCallDerivedAttribute, type ResolvedLlmCallCostCurrency, type ResolvedLlmCallMetric, type ResolvedLlmCallPricing, type ResolvedLlmCallsConfig, type RunInEvalScopeOptions, type RunLogEntry, type RunLogLevel, type RunLogLocation, type RunLogPhase, type RunLogsConfigInput, type RunManifest, type RunSummary, type ScalarCell, type ScopedCaseSummary, type ScoreTrace, type SerializedCacheSpan, type SerializedCacheValue, type SpanCacheOptions, type SseEnvelope, type SseEventType, type TraceActiveSpan, type TraceAttributeDisplay, type TraceAttributeDisplayFormat, type TraceAttributeDisplayInput, type TraceAttributeDisplayPlacement, type TraceAttributeTransform, type TraceAttributeTransformContext, type TraceCacheInfo, type TraceCacheRef, type TraceDisplayConfig, type TraceDisplayInputConfig, type TraceSpanInfo, type TrialSelectionMode, type UpdateManualScoreRequest, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, z };

package/dist/index.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-import { $ as startEvalBackgroundJob, A as repoFile, B as getCurrentScope, C as evalTracer, D as deserializeCacheValue, E as deserializeCacheRecording, F as appendToEvalOutput, G as mergeEvalOutput, H as incrementEvalOutput, J as runInEvalScope, K as nextEvalId, L as evalAssert, M as readManualInputFile, N as evalExpect, Nt as getEvalRegistry, O as serializeCacheRecording, P as EvalAssertionError, Q as setScopeCacheContext, R as evalLog, S as evalSpan, T as hashCacheKeySync, U as isInEvalScope, V as getEvalCaseInput, Y as runInExistingEvalScope, Z as setEvalOutput, at as extractLlmCalls, b as buildTraceTree, it as extractApiCalls, j as manualInputFileValueSchema, k as serializeCacheValue, lt as getNestedAttribute, nt as extractCacheEntries, ot as simulateLlmCallCost, q as runInEvalRuntimeScope, rt as extractCacheHits, st as simulateTokenAllocation, w as hashCacheKey, x as captureEvalSpanError, y as z, z as evalTime } from "./runExecution-Sw38bCaq.mjs";
-import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-BR3wMZMx.mjs";
-import { n as matchesEvalTags, t as defineEval } from "./src-hBGtzWuA.mjs";
+import { $ as startEvalBackgroundJob, A as repoFile, B as getCurrentScope, C as evalTracer, D as deserializeCacheValue, E as deserializeCacheRecording, F as appendToEvalOutput, G as mergeEvalOutput, H as incrementEvalOutput, J as runInEvalScope, K as nextEvalId, L as evalAssert, M as readManualInputFile, N as evalExpect, Nt as getEvalRegistry, O as serializeCacheRecording, P as EvalAssertionError, Q as setScopeCacheContext, R as evalLog, S as evalSpan, T as hashCacheKeySync, U as isInEvalScope, V as getEvalCaseInput, Y as runInExistingEvalScope, Z as setEvalOutput, at as extractLlmCalls, b as buildTraceTree, it as extractApiCalls, j as manualInputFileValueSchema, k as serializeCacheValue, lt as getNestedAttribute, nt as extractCacheEntries, ot as simulateLlmCallCost, q as runInEvalRuntimeScope, rt as extractCacheHits, st as simulateTokenAllocation, w as hashCacheKey, x as captureEvalSpanError, y as z, z as evalTime } from "./runExecution-C31dpemR.mjs";
+import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-Bu9347r1.mjs";
+import { n as matchesEvalTags, t as defineEval } from "./src-FR60ZR_4.mjs";
 export { EvalAssertionError, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob, z };

package/dist/runChild.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
-import { At as evalChartsConfigSchema, Ct as buildEvalKey, Dt as evalStatAggregateSchema, I as configureEvalRunLogs, Ot as evalStatsConfigSchema, bt as runSummarySchema, et as createRunRequestSchema, jt as columnDefSchema, kt as manualInputDescriptorSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, v as createFsCacheStore, yt as runManifestSchema } from "./runExecution-Sw38bCaq.mjs";
-import { S as parseEvalDiscovery, m as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-DJsdLYeZ.mjs";
+import { At as evalChartsConfigSchema, Ct as buildEvalKey, Dt as evalStatAggregateSchema, I as configureEvalRunLogs, Ot as evalStatsConfigSchema, bt as runSummarySchema, et as createRunRequestSchema, jt as columnDefSchema, kt as manualInputDescriptorSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, v as createFsCacheStore, yt as runManifestSchema } from "./runExecution-C31dpemR.mjs";
+import { S as parseEvalDiscovery, m as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-3RoHLW4U.mjs";
 import { z } from "zod/v4";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";