@cursor/july 0.1.16 → 0.1.18

package/dist/internal/eval-runner.d.ts

@@ -7,7 +7,8 @@
  * Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
  * `t.messageIncludes` / `t.check`.
  */
-import type { EvalAssertionResult, EvalCaseDefinition, EvalConfigFile } from "../evals.js";
+import type { EvalJudgeCaller } from "../evals/judge.js";
+import type { EvalCaseDefinition, EvalConfigFile, EvalReporter, EvalReporterTarget, EvalRunResult } from "../evals.js";
 import type { SessionEvent } from "../types.js";
 export interface DiscoveredEval {
     /** Runnable datapoint id (`weather/nyc` or file id for single-test files). */
@@ -17,30 +18,7 @@ export interface DiscoveredEval {
     path: string;
     definition: EvalCaseDefinition;
 }
-export interface EvalRunResult {
-    id: string;
-    path: string;
-    description?: string;
-    ok: boolean;
-    assertions: EvalAssertionResult[];
-    sessionId?: string;
-    /** Messages passed to `t.send` (eval inputs). */
-    inputs: string[];
-    finalText?: string;
-    tools?: string[];
-    /** Tool calls with args/results from the trajectory. */
-    toolCalls?: Array<{
-        toolName: string;
-        args?: unknown;
-        output?: unknown;
-        isError: boolean;
-    }>;
-    error?: string;
-    logs: string[];
-    /** Structured scores from {@link import("../evals.js").EvalTestContext.metric}. */
-    metrics: Record<string, string | number>;
-    durationMs: number;
-}
+export type { EvalRunResult } from "../evals.js";
 export interface EvalSelectionOptions {
     filterIds?: string[];
     tags?: string[];
@@ -49,6 +27,21 @@ export interface RunEvalsOptions extends EvalSelectionOptions {
     projectRoot: string;
     baseUrl: string;
     timeoutMs?: number;
+    /** Override `evals.config.ts` `maxConcurrency` (CLI `--max-concurrency`). */
+    maxConcurrency?: number;
+    /**
+     * Reporters attached by the caller (CLI `--junit` / `--artifacts`), in
+     * addition to the ones declared in `evals.config.ts` and on each eval.
+     */
+    reporters?: readonly EvalReporter[];
+    /** Ignore reporters declared in config and eval files (`--skip-report`). */
+    skipReport?: boolean;
+    /** Reported to reporters so a destination can label the run. */
+    target?: EvalReporterTarget;
+    /** Treat soft threshold misses as failures when summarizing. */
+    strict?: boolean;
+    /** Injected judge model call; defaults to the Cursor SDK caller. */
+    judgeCaller?: EvalJudgeCaller;
     bearerToken?: string;
     /** Extra request headers (e.g. hosted engineAccess). */
     headers?: Record<string, string>;
@@ -76,6 +69,19 @@ export declare function discoverEvals(projectRoot: string, importModule?: (absol
     evals: DiscoveredEval[];
     config: Partial<EvalConfigFile>;
 }>;
+/**
+ * Expand one eval module's default export into runnable datapoints.
+ *
+ * A file normally exports a single `defineEval(...)`. It may instead export an
+ * **array** of them to fan out over a dataset, in which case ids come from the
+ * file name plus a zero-padded index in array order (`sql/0000`, `sql/0001`, …).
+ * Prefer `cases` when the datapoints are hand-written and deserve stable names;
+ * the array form is for generated datasets where the index *is* the name.
+ */
+export declare function expandEvalModule(fileId: string, defaultExport: unknown): Array<{
+    id: string;
+    definition: EvalCaseDefinition;
+}>;
 /** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
 export declare function filterDiscoveredEvals(evals: readonly DiscoveredEval[], options?: EvalSelectionOptions): DiscoveredEval[];
 export declare function runDiscoveredEvals(options: RunEvalsOptions): Promise<EvalRunResult[]>;

package/dist/internal/eval-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/internal/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EAMf,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,eAAgB,SAAQ,oBAAoB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxC,4EAA4E;IAC5E,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iDAAiD;IACjD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,kFAAkF;IAClF,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1E;;;OAGG;IACH,UAAU,CAAC,EAAE;QACX,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;QACjC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;KACjC,CAAC;IACF,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAuCD,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,CACZ,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAiB,GACpD,OAAO,CAAC;IACT,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CACjC,CAAC,CA4CD;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,oBAAyB,GACjC,cAAc,EAAE,CAgBlB;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,aAAa,EAAE,CAAC,CA6D1B;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,cAAc,GAAG,MAAM,CAWvE;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,cAAc,EAAE,GAC/B,cAAc,EAAE,EAAE,CAmBpB"}
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/internal/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkBH,OAAO,KAAK,EAEV,eAAe,EAIhB,MAAM,mBAAmB,CAAC;AAS3B,OAAO,KAAK,EACV,kBAAkB,EAClB,cAAc,EAEd,YAAY,EACZ,kBAAkB,EAClB,aAAa,EAOd,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAgB,YAAY,EAAE,MAAM,aAAa,CAAC;AAW9D,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,eAAgB,SAAQ,oBAAoB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,4EAA4E;IAC5E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gEAAgE;IAChE,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,gEAAgE;IAChE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oEAAoE;IACpE,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxC,4EAA4E;IAC5E,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iDAAiD;IACjD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,kFAAkF;IAClF,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1E;;;OAGG;IACH,UAAU,CAAC,EAAE;QACX,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;QACjC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;KACjC,CAAC;IACF,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAuCD,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,CACZ,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAiB,GACpD,OAAO,CAAC;IACT,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CACjC,CAAC,CA0CD;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,OAAO,GACrB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAA;CAAE,CAAC,CAyBvD;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,oBAAyB,GACjC,cAAc,EAAE,CAgBlB;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,aAAa,EAAE,CAAC,CAqF1B;AAwED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,cAAc,GAAG,MAAM,CAWvE;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,cAAc,EAAE,GAC/B,cAAc,EAAE,EAAE,CAmBpB"}