judgeval 0.9.5 → 1.0.0

package/README.md

@@ -49,7 +49,7 @@ const tracedChat = Tracer.observe(async (userMessage: string) => {
     messages: [{ role: "user", content: userMessage }],
   });
 
-  Tracer.asyncEvaluate("Relevancy");
+  Tracer.asyncEvaluate({ judge: "Relevancy" });
 
   return response.choices[0].message.content || "";
 });

package/dist/Judgeval.d.ts

@@ -1,3 +1,13 @@
+import { EvaluationFactory } from "./evaluation/EvaluationFactory";
+import { DatasetFactory } from "./datasets/DatasetFactory";
+import type { OfflineTracer, OfflineTracerConfig } from "./trace/OfflineTracer";
+/**
+ * Options for {@link Judgeval.offlineTracer}.
+ *
+ * Mirrors `OfflineTracerConfig` minus credentials and `projectName`,
+ * which are reused from the parent `Judgeval` instance.
+ */
+export type JudgevalOfflineTracerOptions = Omit<OfflineTracerConfig, "projectName" | "apiKey" | "organizationId" | "apiUrl">;
 /**
  * Configuration options for the Judgeval client.
  *
@@ -6,6 +16,8 @@
  * `JUDGMENT_API_URL`.
  */
 export interface JudgevalConfig {
+    /** The project name on the Judgment platform. */
+    projectName: string;
     /** Judgment API key. Defaults to `JUDGMENT_API_KEY` env var. */
     apiKey?: string;
     /** Judgment organization ID. Defaults to `JUDGMENT_ORG_ID` env var. */
@@ -17,34 +29,61 @@ export interface JudgevalConfig {
  * The main entry point for interacting with the Judgment platform.
  *
  * `Judgeval` connects to your Judgment project and gives you access to
- * tracing, evaluation, and monitoring through the Judgment platform.
+ * evaluation, datasets, and monitoring through the Judgment platform.
  *
  * @example
  * ```typescript
  * import { Judgeval } from "judgeval";
  *
- * const client = Judgeval.create();
+ * const client = await Judgeval.create({ projectName: "my-project" });
  * ```
  *
  * @throws Error if any required credential is missing.
  */
 export declare class Judgeval {
-    private readonly internalClient;
-    protected constructor(config?: JudgevalConfig);
+    private readonly _client;
+    private readonly _projectName;
+    private readonly _projectId;
+    private constructor();
     /**
      * Create a new Judgeval client instance.
      *
+     * Resolves the `projectName` to a `projectId` via the Judgment API.
+     *
      * @param config - Configuration options. Credentials default to environment variables.
      * @returns A new `Judgeval` instance.
      *
      * @example
      * ```typescript
-     * const client = Judgeval.create({
+     * const client = await Judgeval.create({
+     *   projectName: "my-project",
      *   apiKey: "<your-api-key>",
      *   organizationId: "<your-organization-id>",
      * });
      * ```
      */
-    static create(config?: JudgevalConfig): Judgeval;
+    static create(config: JudgevalConfig): Promise<Judgeval>;
+    /**
+     * Create and activate an `OfflineTracer` for this project.
+     *
+     * Reuses the credentials supplied to this `Judgeval` instance. Each
+     * completed root span appends an `Example` to `dataset`, carrying
+     * the offline trace id and the static `exampleFields`.
+     *
+     * @example
+     * ```typescript
+     * const judgeval = await Judgeval.create({ projectName: "my-project" });
+     * const dataset: Example[] = [];
+     * const tracer = await judgeval.offlineTracer({
+     *   dataset,
+     *   exampleFields: { input: item.input, golden_output: item.goldenOutput },
+     * });
+     * ```
+     */
+    offlineTracer(options: JudgevalOfflineTracerOptions): Promise<OfflineTracer>;
+    /** Access dataset management (create, get, list). */
+    get datasets(): DatasetFactory;
+    /** Access evaluation (create evaluation runs). */
+    get evaluation(): EvaluationFactory;
 }
 //# sourceMappingURL=Judgeval.d.ts.map

package/dist/Judgeval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Judgeval.d.ts","sourceRoot":"","sources":["../src/Judgeval.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoB;IAEnD,SAAS,aAAa,MAAM,GAAE,cAAmB;IAkBjD;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,MAAM,CAAC,MAAM,GAAE,cAAmB,GAAG,QAAQ;CAGrD"}
+{"version":3,"file":"Judgeval.d.ts","sourceRoot":"","sources":["../src/Judgeval.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,MAAM,4BAA4B,GAAG,IAAI,CAC7C,mBAAmB,EACnB,aAAa,GAAG,QAAQ,GAAG,gBAAgB,GAAG,QAAQ,CACvD,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAE3C,OAAO;IAUP;;;;;;;;;;;;;;;;OAgBG;WACU,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgC9D;;;;;;;;;;;;;;;;OAgBG;IACG,aAAa,CACjB,OAAO,EAAE,4BAA4B,GACpC,OAAO,CAAC,aAAa,CAAC;IAWzB,qDAAqD;IACrD,IAAI,QAAQ,IAAI,cAAc,CAE7B;IAED,kDAAkD;IAClD,IAAI,UAAU,IAAI,iBAAiB,CAMlC;CACF"}

package/dist/data/Example.d.ts

@@ -0,0 +1,55 @@
+import type { Example as APIExample } from "../internal/api/models/Example";
+/**
+ * The wire format for examples: the fixed API fields plus arbitrary
+ * user-defined properties (input, actual_output, etc.).
+ */
+export type ExampleDict = APIExample & Record<string, unknown>;
+/**
+ * A single evaluation example with flexible key-value properties.
+ *
+ * Use `Example.create()` to construct an example with arbitrary fields
+ * such as `input`, `actualOutput`, `expectedOutput`, etc.
+ *
+ * @example
+ * ```typescript
+ * const example = Example.create({
+ *   input: "What is the capital of France?",
+ *   actual_output: "Paris is the capital of France.",
+ *   expected_output: "Paris",
+ * });
+ *
+ * example.get("input"); // "What is the capital of France?"
+ * ```
+ */
+export declare class Example {
+    readonly exampleId: string;
+    readonly createdAt: string;
+    readonly name: string | null;
+    private readonly _properties;
+    private constructor();
+    /**
+     * Create an example with the given properties.
+     *
+     * Any key-value pairs passed in `props` become accessible via `.get()`.
+     * Common keys: `input`, `actual_output`, `expected_output`, `retrieval_context`.
+     */
+    static create(props?: Record<string, unknown>): Example;
+    /** Known keys on the API Example interface that are not user properties. */
+    private static readonly META_KEYS;
+    /**
+     * Reconstruct an Example from an API response dict.
+     *
+     * Separates the fixed metadata fields (`example_id`, `created_at`, `name`)
+     * from user-defined properties.
+     */
+    static from(data: ExampleDict): Example;
+    /** Get a property by key. */
+    get(key: string): unknown;
+    /** Check if a property key exists. */
+    has(key: string): boolean;
+    /** Return a shallow copy of all custom properties. */
+    get properties(): Record<string, unknown>;
+    /** Serialize to the API wire format. */
+    toJSON(): ExampleDict;
+}
+//# sourceMappingURL=Example.d.ts.map

package/dist/data/Example.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Example.d.ts","sourceRoot":"","sources":["../../src/data/Example.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,UAAU,EAAE,MAAM,gCAAgC,CAAC;AAE5E;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAE/D;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,OAAO;IAClB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA0B;IAEtD,OAAO;IAYP;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,KAAK,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,OAAO;IAM3D,4EAA4E;IAC5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAO9B;IAEH;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO;IAevC,6BAA6B;IAC7B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB,sCAAsC;IACtC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB,sDAAsD;IACtD,IAAI,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAExC;IAED,wCAAwC;IACxC,MAAM,IAAI,WAAW;CAatB"}

package/dist/data/ScoringResult.d.ts

@@ -0,0 +1,12 @@
+import type { Example } from "./Example";
+import type { ExperimentScorer } from "../internal/api/models/ExperimentScorer";
+/** The combined result of running scorers against a single example. */
+export interface ScoringResult {
+    /** True only if every scorer passed its threshold. */
+    success: boolean;
+    /** Per-scorer results, directly from the API. */
+    scorers: ExperimentScorer[];
+    /** The original example that was evaluated. */
+    example: Example;
+}
+//# sourceMappingURL=ScoringResult.d.ts.map

package/dist/data/ScoringResult.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ScoringResult.d.ts","sourceRoot":"","sources":["../../src/data/ScoringResult.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAEhF,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,EAAE,OAAO,CAAC;CAClB"}

package/dist/data/index.d.ts

@@ -0,0 +1,3 @@
+export { Example, type ExampleDict } from "./Example";
+export type { ScoringResult } from "./ScoringResult";
+//# sourceMappingURL=index.d.ts.map

package/dist/data/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/data/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AACtD,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/datasets/Dataset.d.ts

@@ -0,0 +1,56 @@
+import type { JudgmentApiClient } from "../internal/api/client";
+import { Example } from "../data/Example";
+/**
+ * A collection of {@link Example} objects stored on the Judgment platform.
+ *
+ * Datasets are retrieved via {@link DatasetFactory.get} or created via
+ * {@link DatasetFactory.create}. Once obtained, you can iterate over
+ * the examples directly, or add new ones.
+ *
+ * @example
+ * ```typescript
+ * const dataset = await client.datasets.get("golden-set");
+ * for (const example of dataset) {
+ *   console.log(example.get("input"));
+ * }
+ * ```
+ */
+export declare class Dataset {
+    readonly name: string;
+    readonly projectId: string;
+    readonly projectName: string;
+    readonly datasetKind: string;
+    readonly examples: Example[];
+    private readonly _client;
+    constructor(opts: {
+        name: string;
+        projectId: string;
+        projectName: string;
+        datasetKind?: string;
+        examples?: Example[];
+        client?: JudgmentApiClient | null;
+    });
+    /**
+     * Upload examples to this dataset in batches.
+     *
+     * @param examples - The examples to upload.
+     * @param batchSize - Number of examples per batch request. Defaults to 100.
+     */
+    addExamples(examples: Example[], batchSize?: number): Promise<void>;
+    /**
+     * Load examples from a JSON file and add them to the dataset.
+     *
+     * Expects the file to contain a JSON array of objects, each with
+     * properties like `input`, `actual_output`, etc.
+     *
+     * @param filePath - Path to the JSON file.
+     * @param batchSize - Number of examples per batch request. Defaults to 100.
+     */
+    addFromJson(filePath: string, batchSize?: number): Promise<void>;
+    /** Number of examples in this dataset. */
+    get length(): number;
+    /** Iterate over examples. */
+    [Symbol.iterator](): Iterator<Example>;
+    toString(): string;
+}
+//# sourceMappingURL=Dataset.d.ts.map

package/dist/datasets/Dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/Dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE1C;;;;;;;;;;;;;;GAcG;AACH,qBAAa,OAAO;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;gBAEvC,IAAI,EAAE;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC;KACnC;IASD;;;;;OAKG;IACG,WAAW,CACf,QAAQ,EAAE,OAAO,EAAE,EACnB,SAAS,GAAE,MAAY,GACtB,OAAO,CAAC,IAAI,CAAC;IAahB;;;;;;;;OAQG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAe3E,0CAA0C;IAC1C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,6BAA6B;IAC7B,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC;IAItC,QAAQ,IAAI,MAAM;CAGnB"}

package/dist/datasets/DatasetFactory.d.ts

@@ -0,0 +1,50 @@
+import type { JudgmentApiClient } from "../internal/api/client";
+import type { DatasetInfo } from "../internal/api/models/DatasetInfo";
+import { Example } from "../data/Example";
+import { Dataset } from "./Dataset";
+/**
+ * Creates, retrieves, and lists datasets in your project.
+ *
+ * Access via `client.datasets`.
+ *
+ * @example
+ * ```typescript
+ * const datasets = await client.datasets.list();
+ * const dataset = await client.datasets.get("golden-set");
+ * ```
+ */
+export declare class DatasetFactory {
+    private readonly _client;
+    private readonly _projectId;
+    private readonly _projectName;
+    constructor(client: JudgmentApiClient, projectId: string | null, projectName: string);
+    /**
+     * Retrieve a dataset by name, including all its examples.
+     *
+     * @param name - The dataset name.
+     * @returns The dataset with all examples hydrated, or `null` if the project is unresolved.
+     */
+    get(name: string): Promise<Dataset | null>;
+    /**
+     * Create a new dataset, optionally pre-populated with examples.
+     *
+     * @param name - The dataset name.
+     * @param options.examples - Examples to upload after creation.
+     * @param options.overwrite - If `true`, overwrite an existing dataset with the same name.
+     * @param options.batchSize - Number of examples per batch upload request. Defaults to 100.
+     * @returns The newly created dataset, or `null` if the project is unresolved.
+     */
+    create(name: string, options?: {
+        examples?: Example[];
+        overwrite?: boolean;
+        batchSize?: number;
+    }): Promise<Dataset | null>;
+    /**
+     * List all datasets in the project.
+     *
+     * @returns An array of dataset metadata, or `null` if the project is unresolved.
+     */
+    list(): Promise<DatasetInfo[] | null>;
+    private _expectProjectId;
+}
+//# sourceMappingURL=DatasetFactory.d.ts.map

package/dist/datasets/DatasetFactory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DatasetFactory.d.ts","sourceRoot":"","sources":["../../src/datasets/DatasetFactory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAoB,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;GAUG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAGpC,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM;IAOrB;;;;;OAKG;IACG,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAyBhD;;;;;;;;OAQG;IACG,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;QACP,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACf,GACL,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IA4B1B;;;;OAIG;IACH,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC;IAOrC,OAAO,CAAC,gBAAgB;CASzB"}

package/dist/datasets/index.d.ts

@@ -0,0 +1,4 @@
+export { Dataset } from "./Dataset";
+export { DatasetFactory } from "./DatasetFactory";
+export type { DatasetInfo } from "../internal/api/models/DatasetInfo";
+//# sourceMappingURL=index.d.ts.map

package/dist/datasets/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/datasets/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,YAAY,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC"}

package/dist/evaluation/Evaluation.d.ts

@@ -0,0 +1,65 @@
+import type { JudgmentApiClient } from "../internal/api/client";
+import type { Example } from "../data/Example";
+import type { ScoringResult } from "../data/ScoringResult";
+import { Judge } from "../judges/Judge";
+export interface EvaluationRunOptions {
+    /** The examples to evaluate. */
+    examples: Example[];
+    /**
+     * Hosted scorer names (strings like `"faithfulness"`) **or**
+     * custom `Judge` instances. Cannot mix both.
+     */
+    scorers: string[] | Judge[];
+    /** A name for this run, visible in the dashboard. */
+    evalRunName: string;
+    /**
+     * If true, throws an error when any scorer fails its threshold.
+     * Useful in CI/CD pipelines.
+     */
+    assertTest?: boolean;
+    /**
+     * Maximum seconds to wait for hosted scorer results before timing out.
+     * @default 300
+     */
+    timeoutSeconds?: number;
+}
+/**
+ * Score a batch of examples using hosted scorers or custom judges.
+ *
+ * Two modes are supported:
+ *
+ * - **Hosted scorers** — pass scorer names as strings (e.g.
+ *   `"faithfulness"`, `"answer_relevancy"`). Evaluation runs server-side
+ *   on the Judgment platform.
+ * - **Custom judges** — pass {@link Judge} subclass instances for
+ *   in-process evaluation with your own scoring logic.
+ *
+ * Create an `Evaluation` via `client.evaluation.create()`, then call
+ * `.run()` to execute scorers against your examples.
+ *
+ * @example
+ * ```typescript
+ * const evaluation = client.evaluation.create();
+ * const results = await evaluation.run({
+ *   examples,
+ *   scorers: ["faithfulness", "answer_relevancy"],
+ *   evalRunName: "nightly-eval",
+ * });
+ * ```
+ */
+export declare class Evaluation {
+    private readonly _local;
+    private readonly _hosted;
+    constructor(client: JudgmentApiClient, projectId: string | null, projectName: string);
+    /**
+     * Run scorers against your examples and return results.
+     *
+     * Pass **either** hosted scorer names (strings) **or** custom {@link Judge}
+     * instances. Mixing both in one call is not supported.
+     *
+     * @param options - Evaluation configuration including examples, scorers, and run name.
+     * @returns A list of {@link ScoringResult} objects, one per example.
+     */
+    run(options: EvaluationRunOptions): Promise<ScoringResult[]>;
+}
+//# sourceMappingURL=Evaluation.d.ts.map

package/dist/evaluation/Evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Evaluation.d.ts","sourceRoot":"","sources":["../../src/evaluation/Evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAIxC,MAAM,WAAW,oBAAoB;IACnC,gCAAgC;IAChC,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB;;;OAGG;IACH,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IAC5B,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;gBAG9C,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM;IAMrB;;;;;;;;OAQG;IACH,GAAG,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;CA+C7D"}

package/dist/evaluation/EvaluationFactory.d.ts

@@ -0,0 +1,22 @@
+import type { JudgmentApiClient } from "../internal/api/client";
+import { Evaluation } from "./Evaluation";
+/**
+ * Creates {@link Evaluation} instances for running batch scoring.
+ *
+ * Access via `client.evaluation`.
+ *
+ * @example
+ * ```typescript
+ * const evaluation = client.evaluation.create();
+ * const results = await evaluation.run({ examples, scorers, evalRunName: "my-eval" });
+ * ```
+ */
+export declare class EvaluationFactory {
+    private readonly _client;
+    private readonly _projectId;
+    private readonly _projectName;
+    constructor(client: JudgmentApiClient, projectId: string | null, projectName: string);
+    /** Create a new `Evaluation` bound to the current project. */
+    create(): Evaluation;
+}
+//# sourceMappingURL=EvaluationFactory.d.ts.map

package/dist/evaluation/EvaluationFactory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvaluationFactory.d.ts","sourceRoot":"","sources":["../../src/evaluation/EvaluationFactory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C;;;;;;;;;;GAUG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;IAC3C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAGpC,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM;IAOrB,8DAA8D;IAC9D,MAAM,IAAI,UAAU;CAGrB"}

package/dist/evaluation/EvaluatorRunner.d.ts

@@ -0,0 +1,27 @@
+import type { JudgmentApiClient } from "../internal/api/client";
+import type { ExampleEvaluationRun } from "../internal/api/models/ExampleEvaluationRun";
+import type { ExperimentRunItem } from "../internal/api/models/ExperimentRunItem";
+import type { Example } from "../data/Example";
+import type { ScoringResult } from "../data/ScoringResult";
+import type { Judge } from "../judges/Judge";
+/**
+ * Abstract base for evaluation runners.
+ *
+ * Provides the shared run -> poll -> display flow.
+ * Subclasses implement `_buildPayload` and `_submit` for local vs hosted mode.
+ */
+export declare abstract class EvaluatorRunner<S extends string | Judge> {
+    protected readonly _client: JudgmentApiClient;
+    protected readonly _projectId: string | null;
+    protected readonly _projectName: string;
+    constructor(client: JudgmentApiClient, projectId: string | null, projectName: string);
+    protected abstract _buildPayload(evalId: string, projectId: string, evalRunName: string, createdAt: string, examples: Example[], scorers: S[]): ExampleEvaluationRun;
+    protected abstract _submit(projectId: string, evalId: string, examples: Example[], scorers: S[], payload: ExampleEvaluationRun): Promise<number>;
+    protected _poll(projectId: string, evalId: string, expectedCount: number, timeoutSeconds: number): Promise<{
+        results: ExperimentRunItem[];
+        url: string;
+    }>;
+    protected _displayResults(examples: Example[], resultsData: ExperimentRunItem[], url: string, assertTest: boolean): ScoringResult[];
+    run(examples: Example[], scorers: S[], evalRunName: string, assertTest?: boolean, timeoutSeconds?: number): Promise<ScoringResult[]>;
+}
+//# sourceMappingURL=EvaluatorRunner.d.ts.map

package/dist/evaluation/EvaluatorRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvaluatorRunner.d.ts","sourceRoot":"","sources":["../../src/evaluation/EvaluatorRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6CAA6C,CAAC;AACxF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0CAA0C,CAAC;AAClF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAI7C;;;;;GAKG;AACH,8BAAsB,eAAe,CAAC,CAAC,SAAS,MAAM,GAAG,KAAK;IAC5D,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IAC9C,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAGtC,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,WAAW,EAAE,MAAM;IAOrB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,CAAC,EAAE,GACX,oBAAoB;IAEvB,SAAS,CAAC,QAAQ,CAAC,OAAO,CACxB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,CAAC,EAAE,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC;cAEF,KAAK,CACnB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IA4BzD,SAAS,CAAC,eAAe,CACvB,QAAQ,EAAE,OAAO,EAAE,EACnB,WAAW,EAAE,iBAAiB,EAAE,EAChC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,OAAO,GAClB,aAAa,EAAE;IAsEZ,GAAG,CACP,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,CAAC,EAAE,EACZ,WAAW,EAAE,MAAM,EACnB,UAAU,GAAE,OAAe,EAC3B,cAAc,GAAE,MAAY,GAC3B,OAAO,CAAC,aAAa,EAAE,CAAC;CA8C5B"}

package/dist/evaluation/HostedEvaluatorRunner.d.ts

@@ -0,0 +1,14 @@
+import type { ExampleEvaluationRun } from "../internal/api/models/ExampleEvaluationRun";
+import type { Example } from "../data/Example";
+import { EvaluatorRunner } from "./EvaluatorRunner";
+/**
+ * Evaluation runner for hosted (server-side) scorers.
+ *
+ * Submits scorer names to the Judgment platform's evaluation queue
+ * and polls for results. Used internally by {@link Evaluation}.
+ */
+export declare class HostedEvaluatorRunner extends EvaluatorRunner<string> {
+    protected _buildPayload(evalId: string, projectId: string, evalRunName: string, createdAt: string, examples: Example[], scorers: string[]): ExampleEvaluationRun;
+    protected _submit(projectId: string, _evalId: string, examples: Example[], _scorers: string[], payload: ExampleEvaluationRun): Promise<number>;
+}
+//# sourceMappingURL=HostedEvaluatorRunner.d.ts.map

package/dist/evaluation/HostedEvaluatorRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HostedEvaluatorRunner.d.ts","sourceRoot":"","sources":["../../src/evaluation/HostedEvaluatorRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6CAA6C,CAAC;AACxF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;;;GAKG;AACH,qBAAa,qBAAsB,SAAQ,eAAe,CAAC,MAAM,CAAC;IAChE,SAAS,CAAC,aAAa,CACrB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,MAAM,EAAE,GAChB,oBAAoB;cAYP,OAAO,CACrB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,OAAO,EAAE,EACnB,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC;CAKnB"}

package/dist/evaluation/LocalEvaluatorRunner.d.ts

@@ -0,0 +1,16 @@
+import type { ExampleEvaluationRun } from "../internal/api/models/ExampleEvaluationRun";
+import type { Example } from "../data/Example";
+import type { Judge } from "../judges/Judge";
+import { EvaluatorRunner } from "./EvaluatorRunner";
+/**
+ * Evaluation runner for custom (in-process) scorers.
+ *
+ * Runs all {@link Judge} instances locally against the provided examples,
+ * posts results to the Judgment platform, then polls for finalized scores.
+ * Used internally by {@link Evaluation}.
+ */
+export declare class LocalEvaluatorRunner extends EvaluatorRunner<Judge> {
+    protected _buildPayload(evalId: string, projectId: string, evalRunName: string, createdAt: string, examples: Example[], _scorers: Judge[]): ExampleEvaluationRun;
+    protected _submit(projectId: string, _evalId: string, examples: Example[], scorers: Judge[], payload: ExampleEvaluationRun): Promise<number>;
+}
+//# sourceMappingURL=LocalEvaluatorRunner.d.ts.map

package/dist/evaluation/LocalEvaluatorRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LocalEvaluatorRunner.d.ts","sourceRoot":"","sources":["../../src/evaluation/LocalEvaluatorRunner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6CAA6C,CAAC;AAExF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AASpD;;;;;;GAMG;AACH,qBAAa,oBAAqB,SAAQ,eAAe,CAAC,KAAK,CAAC;IAC9D,SAAS,CAAC,aAAa,CACrB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EAAE,EACnB,QAAQ,EAAE,KAAK,EAAE,GAChB,oBAAoB;cAYP,OAAO,CACrB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,KAAK,EAAE,EAChB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,MAAM,CAAC;CAgFnB"}

package/dist/evaluation/index.d.ts

@@ -0,0 +1,6 @@
+export { Evaluation, type EvaluationRunOptions } from "./Evaluation";
+export { EvaluationFactory } from "./EvaluationFactory";
+export { EvaluatorRunner } from "./EvaluatorRunner";
+export { LocalEvaluatorRunner } from "./LocalEvaluatorRunner";
+export { HostedEvaluatorRunner } from "./HostedEvaluatorRunner";
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evaluation/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC"}