langsmith 0.7.4 → 0.7.6

package/README.md

@@ -25,6 +25,47 @@ A typical workflow looks like:
 
 We'll walk through these steps in more detail below.
 
+## Sandbox AWS auth proxy
+
+When you create a LangSmith sandbox that needs to call AWS services, use the
+sandbox AWS auth proxy helpers. The proxy keeps the real AWS credentials outside
+the sandbox and signs supported AWS HTTPS requests with SigV4, so code in the
+sandbox can use AWS SDKs normally without storing long-lived AWS keys in files,
+environment variables, shell history, or logs.
+
+First, store `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as LangSmith
+workspace secrets. Then create the sandbox with an AWS auth proxy config:
+
+```ts
+import {
+  SandboxClient,
+  awsAuthProxyConfig,
+  workspaceSecret,
+} from "langsmith/sandbox";
+
+const client = new SandboxClient();
+
+const sandbox = await client.createSandbox({
+  name: "aws-sandbox",
+  proxyConfig: awsAuthProxyConfig({
+    accessKeyId: workspaceSecret("AWS_ACCESS_KEY_ID"),
+    secretAccessKey: workspaceSecret("AWS_SECRET_ACCESS_KEY"),
+  }),
+});
+
+try {
+  // Your sandbox code can use the AWS SDK, the AWS CLI, or other AWS tooling normally.
+  const result = await sandbox.run("node your-aws-script.js");
+  console.log(result.stdout);
+} finally {
+  await sandbox.delete();
+}
+```
+
+Use `opaqueSecret("...")` instead of `workspaceSecret(...)` when your application
+needs to pass short-lived write-only AWS credentials at sandbox creation time.
+Plaintext AWS credential values are not supported.
+
 ## 1. Connect to LangSmith
 
 Sign up for [LangSmith](https://smith.langchain.com/) using your GitHub, Discord accounts, or an email address and password. If you sign up with an email, make sure to verify your email address before logging in.

package/dist/client.cjs

@@ -2557,7 +2557,7 @@ class Client {
             _hostUrl: this.getHostUrl(),
         }));
     }
-    async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
+    async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
         const upsert_ = upsert ? `?upsert=true` : "";
         const endpoint = `${this.apiUrl}/sessions${upsert_}`;
         const extra = projectExtra || {};
@@ -2572,6 +2572,15 @@ class Client {
         if (referenceDatasetId !== null) {
             body["reference_dataset_id"] = referenceDatasetId;
         }
+        if (numExamples != null) {
+            body["num_examples"] = numExamples;
+        }
+        if (numRepetitions != null) {
+            body["num_repetitions"] = numRepetitions;
+        }
+        if (evaluatorKeys != null && evaluatorKeys.length > 0) {
+            body["evaluator_keys"] = evaluatorKeys;
+        }
         const serializedBody = JSON.stringify(body);
         const response = await this.caller.call(async () => {
             const res = await this._fetch(endpoint, {
@@ -4103,6 +4112,23 @@ class Client {
         }
         return json.commits[0].commit_hash;
     }
+    async _createCommitTags(promptOwnerAndName, commitId, tags) {
+        const tagList = typeof tags === "string" ? [tags] : tags;
+        await Promise.all(tagList.map(async (tag) => this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/repos/${promptOwnerAndName}/tags`, {
+                method: "POST",
+                headers: {
+                    ...this._mergedHeaders,
+                    "Content-Type": "application/json",
+                },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify({ tag_name: tag, commit_id: commitId }),
+            });
+            await (0, error_js_1.raiseForStatus)(res, "create commit tag");
+            return res;
+        })));
+    }
     async _likeOrUnlikePrompt(promptIdentifier, like) {
         const [owner, promptName, _] = (0, prompts_js_1.parseHubIdentifier)(promptIdentifier);
         const body = JSON.stringify({ like: like });
@@ -4369,6 +4395,8 @@ class Client {
      * @param object - The prompt object/manifest to commit (e.g., ChatPromptTemplate, messages array, etc.)
      * @param options - Optional configuration for the commit
      * @param options.parentCommitHash - The parent commit hash. Defaults to "latest" (the most recent commit).
+     * @param options.tags - A tag or list of tags to apply to the commit.
+     * @param options.description - A description for the commit.
      * @returns A Promise that resolves to the URL of the newly created commit
      * @throws {Error} If the prompt does not exist
      * @example
@@ -4384,9 +4412,9 @@ class Client {
      * const commitUrl = await client.createCommit("my-prompt", template);
      * console.log(`Commit created: ${commitUrl}`);
      *
-     * // Create a commit based on a specific parent commit
+     * // Create a commit with tags
      * const commitUrl2 = await client.createCommit("my-prompt", template, {
-     *   parentCommitHash: "abc123def456"
+     *   tags: ["production", "v1"]
      * });
      * ```
      */
@@ -4421,7 +4449,11 @@ class Client {
             return res;
         });
         const result = await response.json();
-        return this._getPromptUrl(`${owner}/${promptName}${result.commit_hash ? `:${result.commit_hash}` : ""}`);
+        const commit = result.commit ?? result;
+        if (options?.tags) {
+            await this._createCommitTags(`${owner}/${promptName}`, commit.id, options.tags);
+        }
+        return this._getPromptUrl(`${owner}/${promptName}${commit.commit_hash ? `:${commit.commit_hash}` : ""}`);
     }
     /**
      * Update examples with attachments using multipart form data.
@@ -4756,7 +4788,8 @@ class Client {
     async pushPrompt(promptIdentifier, options) {
         // Create or update prompt metadata
         if (await this.promptExists(promptIdentifier)) {
-            if (options && Object.keys(options).some((key) => key !== "object")) {
+            if (options &&
+                ["description", "readme", "tags", "isPublic"].some((key) => options[key] !== undefined)) {
                 await this.updatePrompt(promptIdentifier, {
                     description: options?.description,
                     readme: options?.readme,
@@ -4779,6 +4812,7 @@ class Client {
         // Create a commit with the new manifest
         const url = await this.createCommit(promptIdentifier, options?.object, {
             parentCommitHash: options?.parentCommitHash,
+            tags: options?.commitTags,
             description: options?.commitDescription,
         });
         return url;
@@ -4866,7 +4900,7 @@ class Client {
     async _pullDirectory(identifier, repoType, version) {
         const [owner, name, parsedVersion] = (0, prompts_js_1.parseHubIdentifier)(identifier);
         const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
-        const url = new URL(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`);
+        const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
         url.searchParams.set("repo_type", repoType);
         if (resolvedVersion) {
             url.searchParams.set("commit", resolvedVersion);
@@ -4912,7 +4946,7 @@ class Client {
             body.parent_commit = options.parentCommit;
         }
         const response = await this.caller.call(async () => {
-            const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories/commits`, {
+            const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
                 method: "POST",
                 headers: {
                     ...this._mergedHeaders,
@@ -4937,7 +4971,7 @@ class Client {
             throw await this._ownerConflictError("delete", owner);
         }
         await this.caller.call(async () => {
-            const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`, {
+            const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
                 method: "DELETE",
                 headers: this._mergedHeaders,
                 signal: AbortSignal.timeout(this.timeout_ms),

package/dist/client.d.ts

@@ -349,6 +349,9 @@ export type CreateProjectParams = {
     upsert?: boolean;
     projectExtra?: RecordStringAny | null;
     referenceDatasetId?: string | null;
+    numExamples?: number | null;
+    numRepetitions?: number | null;
+    evaluatorKeys?: string[] | null;
 };
 type AutoBatchQueueItem = {
     action: "create" | "update";
@@ -714,7 +717,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     listSharedExamples(shareToken: string, options?: {
         exampleIds?: string[];
     }): Promise<Example[]>;
-    createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, }: CreateProjectParams): Promise<TracerSession>;
+    createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, numExamples, numRepetitions, evaluatorKeys, }: CreateProjectParams): Promise<TracerSession>;
     updateProject(projectId: string, { name, description, metadata, projectExtra, endTime, }: {
         name?: string | null;
         description?: string | null;
@@ -1129,6 +1132,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     protected _currentTenantIsOwner(owner: string): Promise<boolean>;
     protected _ownerConflictError(action: string, owner: string): Promise<Error>;
     protected _getLatestCommitHash(promptOwnerAndName: string): Promise<string | undefined>;
+    protected _createCommitTags(promptOwnerAndName: string, commitId: string, tags: string | string[]): Promise<void>;
     protected _likeOrUnlikePrompt(promptIdentifier: string, like: boolean): Promise<LikePromptResponse>;
     protected _getPromptUrl(promptIdentifier: string): Promise<string>;
     /**
@@ -1287,6 +1291,8 @@ export declare class Client implements LangSmithTracingClientInterface {
      * @param object - The prompt object/manifest to commit (e.g., ChatPromptTemplate, messages array, etc.)
      * @param options - Optional configuration for the commit
      * @param options.parentCommitHash - The parent commit hash. Defaults to "latest" (the most recent commit).
+     * @param options.tags - A tag or list of tags to apply to the commit.
+     * @param options.description - A description for the commit.
      * @returns A Promise that resolves to the URL of the newly created commit
      * @throws {Error} If the prompt does not exist
      * @example
@@ -1302,14 +1308,15 @@ export declare class Client implements LangSmithTracingClientInterface {
      * const commitUrl = await client.createCommit("my-prompt", template);
      * console.log(`Commit created: ${commitUrl}`);
      *
-     * // Create a commit based on a specific parent commit
+     * // Create a commit with tags
      * const commitUrl2 = await client.createCommit("my-prompt", template, {
-     *   parentCommitHash: "abc123def456"
+     *   tags: ["production", "v1"]
      * });
      * ```
      */
     createCommit(promptIdentifier: string, object: any, options?: {
         parentCommitHash?: string;
+        tags?: string | string[];
         description?: string;
     }): Promise<string>;
     /**
@@ -1413,6 +1420,7 @@ export declare class Client implements LangSmithTracingClientInterface {
         description?: string;
         readme?: string;
         tags?: string[];
+        commitTags?: string | string[];
         commitDescription?: string;
     }): Promise<string>;
     /**

package/dist/client.js

@@ -2519,7 +2519,7 @@ export class Client {
             _hostUrl: this.getHostUrl(),
         }));
     }
-    async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
+    async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
         const upsert_ = upsert ? `?upsert=true` : "";
         const endpoint = `${this.apiUrl}/sessions${upsert_}`;
         const extra = projectExtra || {};
@@ -2534,6 +2534,15 @@ export class Client {
         if (referenceDatasetId !== null) {
             body["reference_dataset_id"] = referenceDatasetId;
         }
+        if (numExamples != null) {
+            body["num_examples"] = numExamples;
+        }
+        if (numRepetitions != null) {
+            body["num_repetitions"] = numRepetitions;
+        }
+        if (evaluatorKeys != null && evaluatorKeys.length > 0) {
+            body["evaluator_keys"] = evaluatorKeys;
+        }
         const serializedBody = JSON.stringify(body);
         const response = await this.caller.call(async () => {
             const res = await this._fetch(endpoint, {
@@ -4065,6 +4074,23 @@ export class Client {
         }
         return json.commits[0].commit_hash;
     }
+    async _createCommitTags(promptOwnerAndName, commitId, tags) {
+        const tagList = typeof tags === "string" ? [tags] : tags;
+        await Promise.all(tagList.map(async (tag) => this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/repos/${promptOwnerAndName}/tags`, {
+                method: "POST",
+                headers: {
+                    ...this._mergedHeaders,
+                    "Content-Type": "application/json",
+                },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify({ tag_name: tag, commit_id: commitId }),
+            });
+            await raiseForStatus(res, "create commit tag");
+            return res;
+        })));
+    }
     async _likeOrUnlikePrompt(promptIdentifier, like) {
         const [owner, promptName, _] = parseHubIdentifier(promptIdentifier);
         const body = JSON.stringify({ like: like });
@@ -4331,6 +4357,8 @@ export class Client {
      * @param object - The prompt object/manifest to commit (e.g., ChatPromptTemplate, messages array, etc.)
      * @param options - Optional configuration for the commit
      * @param options.parentCommitHash - The parent commit hash. Defaults to "latest" (the most recent commit).
+     * @param options.tags - A tag or list of tags to apply to the commit.
+     * @param options.description - A description for the commit.
      * @returns A Promise that resolves to the URL of the newly created commit
      * @throws {Error} If the prompt does not exist
      * @example
@@ -4346,9 +4374,9 @@ export class Client {
      * const commitUrl = await client.createCommit("my-prompt", template);
      * console.log(`Commit created: ${commitUrl}`);
      *
-     * // Create a commit based on a specific parent commit
+     * // Create a commit with tags
      * const commitUrl2 = await client.createCommit("my-prompt", template, {
-     *   parentCommitHash: "abc123def456"
+     *   tags: ["production", "v1"]
      * });
      * ```
      */
@@ -4383,7 +4411,11 @@ export class Client {
             return res;
         });
         const result = await response.json();
-        return this._getPromptUrl(`${owner}/${promptName}${result.commit_hash ? `:${result.commit_hash}` : ""}`);
+        const commit = result.commit ?? result;
+        if (options?.tags) {
+            await this._createCommitTags(`${owner}/${promptName}`, commit.id, options.tags);
+        }
+        return this._getPromptUrl(`${owner}/${promptName}${commit.commit_hash ? `:${commit.commit_hash}` : ""}`);
     }
     /**
      * Update examples with attachments using multipart form data.
@@ -4718,7 +4750,8 @@ export class Client {
     async pushPrompt(promptIdentifier, options) {
         // Create or update prompt metadata
         if (await this.promptExists(promptIdentifier)) {
-            if (options && Object.keys(options).some((key) => key !== "object")) {
+            if (options &&
+                ["description", "readme", "tags", "isPublic"].some((key) => options[key] !== undefined)) {
                 await this.updatePrompt(promptIdentifier, {
                     description: options?.description,
                     readme: options?.readme,
@@ -4741,6 +4774,7 @@ export class Client {
         // Create a commit with the new manifest
         const url = await this.createCommit(promptIdentifier, options?.object, {
             parentCommitHash: options?.parentCommitHash,
+            tags: options?.commitTags,
             description: options?.commitDescription,
         });
         return url;
@@ -4828,7 +4862,7 @@ export class Client {
     async _pullDirectory(identifier, repoType, version) {
         const [owner, name, parsedVersion] = parseHubIdentifier(identifier);
         const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
-        const url = new URL(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`);
+        const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
         url.searchParams.set("repo_type", repoType);
         if (resolvedVersion) {
             url.searchParams.set("commit", resolvedVersion);
@@ -4874,7 +4908,7 @@ export class Client {
             body.parent_commit = options.parentCommit;
         }
         const response = await this.caller.call(async () => {
-            const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories/commits`, {
+            const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
                 method: "POST",
                 headers: {
                     ...this._mergedHeaders,
@@ -4899,7 +4933,7 @@ export class Client {
             throw await this._ownerConflictError("delete", owner);
         }
         await this.caller.call(async () => {
-            const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`, {
+            const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
                 method: "DELETE",
                 headers: this._mergedHeaders,
                 signal: AbortSignal.timeout(this.timeout_ms),

package/dist/evaluation/_runner.cjs

@@ -149,12 +149,24 @@ class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_numExamples", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_numRepetitions", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_evaluatorKeys", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_runsArray", {
             enumerable: true,
             configurable: true,
@@ -231,6 +243,8 @@ class _ExperimentManager {
         this._summaryResults = args.summaryResults;
         this._resultRows = args.resultRows;
         this._numRepetitions = args.numRepetitions;
+        this._numExamples = args.numExamples;
+        this._evaluatorKeys = args.evaluatorKeys;
         this._includeAttachments = args.includeAttachments;
     }
     _getExperiment() {
@@ -263,6 +277,12 @@ class _ExperimentManager {
         // Create the project, updating the experimentName until we find a unique one.
         let project;
         const originalExperimentName = this._experimentName;
+        if (this._numExamples === undefined) {
+            this._numExamples = await _resolveNumExamples(this._data, this.client);
+        }
+        const numExamples = this._numExamples ?? null;
+        const numRepetitions = this._numRepetitions ?? null;
+        const evaluatorKeys = this._evaluatorKeys ?? null;
         for (let i = 0; i < 10; i++) {
             try {
                 project = await this.client.createProject({
@@ -270,6 +290,9 @@ class _ExperimentManager {
                     referenceDatasetId: firstExample.dataset_id,
                     metadata: projectMetadata,
                     description: this._description,
+                    numExamples,
+                    numRepetitions,
+                    evaluatorKeys,
                 });
                 return project;
             }
@@ -319,6 +342,9 @@ class _ExperimentManager {
             client: this.client,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -345,6 +371,9 @@ class _ExperimentManager {
                     yield pred.run;
                 }
             })(),
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -369,6 +398,9 @@ class _ExperimentManager {
                 }
             })(),
             summaryResults: this._summaryResults,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -384,6 +416,9 @@ class _ExperimentManager {
             evaluationResults: this._evaluationResults,
             resultRows: this._resultRows,
             summaryResults: aggregateFeedbackGen,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -732,6 +767,7 @@ async function _evaluate(target, fields) {
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
         numRepetitions: fields.numRepetitions ?? 1,
+        evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
         includeAttachments: standardFields.includeAttachments,
     }).start();
     const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -852,6 +888,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
         example,
     };
 }
+function _collectEvaluatorKeys(evaluators) {
+    if (!evaluators)
+        return [];
+    const keys = [];
+    for (const ev of evaluators) {
+        const name = ev.name;
+        if (name && name.length > 0)
+            keys.push(name);
+    }
+    return keys;
+}
+async function _resolveNumExamples(data, client) {
+    // Best-effort dataset size for the experiment progress hint. Returns
+    // undefined for lazy async iterators where size cannot be inferred without
+    // consuming.
+    if (data === undefined)
+        return undefined;
+    try {
+        if (Array.isArray(data)) {
+            return data.length;
+        }
+        if (typeof data === "string") {
+            let isUUID = false;
+            try {
+                (0, _uuid_js_1.assertUuid)(data);
+                isUUID = true;
+            }
+            catch (_) {
+                isUUID = false;
+            }
+            const dataset = isUUID
+                ? await client.readDataset({ datasetId: data })
+                : await client.readDataset({ datasetName: data });
+            return dataset.example_count ?? undefined;
+        }
+        return undefined;
+    }
+    catch (_) {
+        return undefined;
+    }
+}
 function _resolveData(data, options) {
     let isUUID = false;
     try {

package/dist/evaluation/_runner.d.ts

@@ -68,6 +68,8 @@ interface _ExperimentManagerArgs {
     summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
     examples?: Example[];
     numRepetitions?: number;
+    numExamples?: number;
+    evaluatorKeys?: string[];
     _runsArray?: Run[];
     resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
     includeAttachments?: boolean;
@@ -182,7 +184,9 @@ export declare class _ExperimentManager {
     _summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
     _resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
     _examples?: Example[];
+    _numExamples?: number;
     _numRepetitions?: number;
+    _evaluatorKeys?: string[];
     _runsArray?: Run[];
     client: Client;
     _experiment?: TracerSession;

package/dist/evaluation/_runner.js

@@ -143,12 +143,24 @@ export class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_numExamples", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_numRepetitions", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_evaluatorKeys", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_runsArray", {
             enumerable: true,
             configurable: true,
@@ -225,6 +237,8 @@ export class _ExperimentManager {
         this._summaryResults = args.summaryResults;
         this._resultRows = args.resultRows;
         this._numRepetitions = args.numRepetitions;
+        this._numExamples = args.numExamples;
+        this._evaluatorKeys = args.evaluatorKeys;
         this._includeAttachments = args.includeAttachments;
     }
     _getExperiment() {
@@ -257,6 +271,12 @@ export class _ExperimentManager {
         // Create the project, updating the experimentName until we find a unique one.
         let project;
         const originalExperimentName = this._experimentName;
+        if (this._numExamples === undefined) {
+            this._numExamples = await _resolveNumExamples(this._data, this.client);
+        }
+        const numExamples = this._numExamples ?? null;
+        const numRepetitions = this._numRepetitions ?? null;
+        const evaluatorKeys = this._evaluatorKeys ?? null;
         for (let i = 0; i < 10; i++) {
             try {
                 project = await this.client.createProject({
@@ -264,6 +284,9 @@ export class _ExperimentManager {
                     referenceDatasetId: firstExample.dataset_id,
                     metadata: projectMetadata,
                     description: this._description,
+                    numExamples,
+                    numRepetitions,
+                    evaluatorKeys,
                 });
                 return project;
             }
@@ -313,6 +336,9 @@ export class _ExperimentManager {
             client: this.client,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -339,6 +365,9 @@ export class _ExperimentManager {
                     yield pred.run;
                 }
             })(),
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -363,6 +392,9 @@ export class _ExperimentManager {
                 }
             })(),
             summaryResults: this._summaryResults,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -378,6 +410,9 @@ export class _ExperimentManager {
             evaluationResults: this._evaluationResults,
             resultRows: this._resultRows,
             summaryResults: aggregateFeedbackGen,
+            numRepetitions: this._numRepetitions,
+            numExamples: this._numExamples,
+            evaluatorKeys: this._evaluatorKeys,
             includeAttachments: this._includeAttachments,
         });
     }
@@ -725,6 +760,7 @@ async function _evaluate(target, fields) {
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
         numRepetitions: fields.numRepetitions ?? 1,
+        evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
         includeAttachments: standardFields.includeAttachments,
     }).start();
     const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -845,6 +881,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
         example,
     };
 }
+function _collectEvaluatorKeys(evaluators) {
+    if (!evaluators)
+        return [];
+    const keys = [];
+    for (const ev of evaluators) {
+        const name = ev.name;
+        if (name && name.length > 0)
+            keys.push(name);
+    }
+    return keys;
+}
+async function _resolveNumExamples(data, client) {
+    // Best-effort dataset size for the experiment progress hint. Returns
+    // undefined for lazy async iterators where size cannot be inferred without
+    // consuming.
+    if (data === undefined)
+        return undefined;
+    try {
+        if (Array.isArray(data)) {
+            return data.length;
+        }
+        if (typeof data === "string") {
+            let isUUID = false;
+            try {
+                assertUuid(data);
+                isUUID = true;
+            }
+            catch (_) {
+                isUUID = false;
+            }
+            const dataset = isUUID
+                ? await client.readDataset({ datasetId: data })
+                : await client.readDataset({ datasetName: data });
+            return dataset.example_count ?? undefined;
+        }
+        return undefined;
+    }
+    catch (_) {
+        return undefined;
+    }
+}
 function _resolveData(data, options) {
     let isUUID = false;
     try {