langsmith 0.8.4 → 0.8.6

package/dist/client.cjs

@@ -648,6 +648,12 @@ class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_stainlessVersionChecked", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         Object.defineProperty(this, "manualFlushMode", {
             enumerable: true,
             configurable: true,
@@ -975,27 +981,37 @@ class Client {
         return needsV1Prefix ? `/v1/platform/${path}` : `/platform/${path}`;
     }
     get evaluators() {
+        this._checkStainlessVersion();
         return this.openAPIClient.onlineEvaluators;
     }
     get runs() {
+        this._checkStainlessVersion();
         return this.openAPIClient.runs;
     }
     /** Access the v2 sandboxes resource (registries, snapshots, boxes). */
     get sandboxes() {
+        this._checkStainlessVersion();
         return this.openAPIClient.sandboxes;
     }
     /** Access the v2 datasets resource (experimentRuns, etc.). */
     get datasets() {
+        this._checkStainlessVersion();
         return this.openAPIClient.datasets;
     }
     /** Access the threads resource (query, stats, listTraces). */
     get threads() {
+        this._checkStainlessVersion();
         return this.openAPIClient.threads;
     }
     /** Access the traces resource (query, listRuns). */
     get traces() {
+        this._checkStainlessVersion();
         return this.openAPIClient.traces;
     }
+    /** Access the public shared-run resource. */
+    get public() {
+        return this.openAPIClient.public;
+    }
     async processInputs(inputs) {
         if (this.hideInputs === false) {
             return inputs;
@@ -1482,15 +1498,26 @@ class Client {
         }
         return json;
     }
+    _checkStainlessVersion() {
+        if (this._stainlessVersionChecked)
+            return;
+        this._stainlessVersionChecked = true;
+        this._ensureServerInfo()
+            .then((serverInfo) => {
+            if (serverInfo?.version) {
+                _checkBackendVersion(serverInfo.version);
+            }
+        })
+            .catch(() => {
+            // _ensureServerInfo handles and logs its own errors
+        });
+    }
     async _ensureServerInfo() {
         if (this._getServerInfoPromise === undefined) {
             this._getServerInfoPromise = (async () => {
                 if (this._serverInfo === undefined) {
                     try {
                         this._serverInfo = await this._getServerInfo();
-                        if (this._serverInfo?.version) {
-                            _checkBackendVersion(this._serverInfo.version);
-                        }
                     }
                     catch (e) {
                         console.warn(`[LANGSMITH]: Failed to fetch info on supported operations. Falling back to batch operations and default limits. Info: ${e.status ?? "Unspecified status code"} ${e.message}`);
@@ -1506,6 +1533,10 @@ class Client {
             return serverInfo;
         });
     }
+    async _supportsSDBQuery() {
+        const serverInfo = await this._ensureServerInfo();
+        return serverInfo.instance_flags?.sdb_query_enabled === true;
+    }
     async _getSettings() {
         if (!this.settings) {
             this.settings = this._get("/settings");
@@ -2444,6 +2475,9 @@ class Client {
                 ...(await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)))),
             ];
         }
+        if (projectIds_.length === 0) {
+            throw new Error("At least one of projectNames or projectIds must be provided.");
+        }
         const payload = {
             id,
             trace,
@@ -3827,7 +3861,7 @@ class Client {
         }
         return results_;
     }
-    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo, sessionId) {
         const evalResults = this._selectEvalResults(evaluatorResponse);
         const feedbacks = [];
         for (const res of evalResults) {
@@ -3851,14 +3885,20 @@ class Client {
                 sourceRunId: res.sourceRunId,
                 feedbackConfig: res.feedbackConfig,
                 feedbackSourceType: "model",
-                sessionId: run?.session_id,
+                sessionId: run?.session_id ?? sessionId,
                 startTime: run?.start_time,
             }));
         }
         return [evalResults, feedbacks];
     }
-    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
-        const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
+    async logEvaluationFeedback(evaluatorResponseOrParams, run, sourceInfo, sessionId) {
+        if (evaluatorResponseOrParams != null &&
+            typeof evaluatorResponseOrParams === "object" &&
+            "evaluatorResponse" in evaluatorResponseOrParams) {
+            const [results] = await this._logEvaluationFeedback(evaluatorResponseOrParams.evaluatorResponse, evaluatorResponseOrParams.run, evaluatorResponseOrParams.sourceInfo, evaluatorResponseOrParams.projectId);
+            return results;
+        }
+        const [results] = await this._logEvaluationFeedback(evaluatorResponseOrParams, run, sourceInfo, sessionId);
         return results;
     }
     /**

package/dist/client.d.ts

@@ -9,6 +9,7 @@ import { Sandboxes } from "./_openapi_client/resources/sandboxes/sandboxes.js";
 import { Datasets } from "./_openapi_client/resources/datasets/datasets.js";
 import { Threads } from "./_openapi_client/resources/threads.js";
 import { Traces } from "./_openapi_client/resources/traces.js";
+import { Public } from "./_openapi_client/resources/public/public.js";
 import { PromptCache } from "./utils/prompt_cache/index.js";
 import { ProfileAuth } from "./utils/profiles.js";
 export interface ClientConfig {
@@ -461,6 +462,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     private traceBatchConcurrency;
     private _serverInfo;
     private _getServerInfoPromise?;
+    private _stainlessVersionChecked;
     private manualFlushMode;
     private _serializeWorker?;
     /**
@@ -528,6 +530,8 @@ export declare class Client implements LangSmithTracingClientInterface {
     get threads(): Threads;
     /** Access the traces resource (query, listRuns). */
     get traces(): Traces;
+    /** Access the public shared-run resource. */
+    get public(): Public;
     private processInputs;
     private processOutputs;
     private processMetadata;
@@ -580,7 +584,9 @@ export declare class Client implements LangSmithTracingClientInterface {
     private _sendBatchToOTELTranslator;
     private processRunOperation;
     protected _getServerInfo(): Promise<any>;
+    private _checkStainlessVersion;
     protected _ensureServerInfo(): Promise<Record<string, any>>;
+    _supportsSDBQuery(): Promise<boolean>;
     protected _getSettings(): Promise<LangSmithSettings>;
     /**
      * Flushes current queued traces.
@@ -1027,10 +1033,19 @@ export declare class Client implements LangSmithTracingClientInterface {
     _selectEvalResults(results: EvaluationResult | EvaluationResult[] | EvaluationResults): Array<EvaluationResult>;
     _logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResult[] | EvaluationResults, run?: Run, sourceInfo?: {
         [key: string]: any;
-    }): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]>;
+    }, sessionId?: string): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]>;
+    logEvaluationFeedback(params: {
+        evaluatorResponse: EvaluationResult | EvaluationResult[] | EvaluationResults;
+        run: Run;
+        projectId: string;
+        sourceInfo?: {
+            [key: string]: any;
+        };
+    }): Promise<EvaluationResult[]>;
+    /** @deprecated Pass all params within an object and populate projectId. */
     logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResult[] | EvaluationResults, run?: Run, sourceInfo?: {
         [key: string]: any;
-    }): Promise<EvaluationResult[]>;
+    }, sessionId?: string): Promise<EvaluationResult[]>;
     /**
      * API for managing feedback configs
      */

package/dist/client.js

@@ -609,6 +609,12 @@ export class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_stainlessVersionChecked", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         Object.defineProperty(this, "manualFlushMode", {
             enumerable: true,
             configurable: true,
@@ -936,27 +942,37 @@ export class Client {
         return needsV1Prefix ? `/v1/platform/${path}` : `/platform/${path}`;
     }
     get evaluators() {
+        this._checkStainlessVersion();
         return this.openAPIClient.onlineEvaluators;
     }
     get runs() {
+        this._checkStainlessVersion();
         return this.openAPIClient.runs;
     }
     /** Access the v2 sandboxes resource (registries, snapshots, boxes). */
     get sandboxes() {
+        this._checkStainlessVersion();
         return this.openAPIClient.sandboxes;
     }
     /** Access the v2 datasets resource (experimentRuns, etc.). */
     get datasets() {
+        this._checkStainlessVersion();
         return this.openAPIClient.datasets;
     }
     /** Access the threads resource (query, stats, listTraces). */
     get threads() {
+        this._checkStainlessVersion();
         return this.openAPIClient.threads;
     }
     /** Access the traces resource (query, listRuns). */
     get traces() {
+        this._checkStainlessVersion();
         return this.openAPIClient.traces;
     }
+    /** Access the public shared-run resource. */
+    get public() {
+        return this.openAPIClient.public;
+    }
     async processInputs(inputs) {
         if (this.hideInputs === false) {
             return inputs;
@@ -1443,15 +1459,26 @@ export class Client {
         }
         return json;
     }
+    _checkStainlessVersion() {
+        if (this._stainlessVersionChecked)
+            return;
+        this._stainlessVersionChecked = true;
+        this._ensureServerInfo()
+            .then((serverInfo) => {
+            if (serverInfo?.version) {
+                _checkBackendVersion(serverInfo.version);
+            }
+        })
+            .catch(() => {
+            // _ensureServerInfo handles and logs its own errors
+        });
+    }
     async _ensureServerInfo() {
         if (this._getServerInfoPromise === undefined) {
             this._getServerInfoPromise = (async () => {
                 if (this._serverInfo === undefined) {
                     try {
                         this._serverInfo = await this._getServerInfo();
-                        if (this._serverInfo?.version) {
-                            _checkBackendVersion(this._serverInfo.version);
-                        }
                     }
                     catch (e) {
                         console.warn(`[LANGSMITH]: Failed to fetch info on supported operations. Falling back to batch operations and default limits. Info: ${e.status ?? "Unspecified status code"} ${e.message}`);
@@ -1467,6 +1494,10 @@ export class Client {
             return serverInfo;
         });
     }
+    async _supportsSDBQuery() {
+        const serverInfo = await this._ensureServerInfo();
+        return serverInfo.instance_flags?.sdb_query_enabled === true;
+    }
     async _getSettings() {
         if (!this.settings) {
             this.settings = this._get("/settings");
@@ -2405,6 +2436,9 @@ export class Client {
                 ...(await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)))),
             ];
         }
+        if (projectIds_.length === 0) {
+            throw new Error("At least one of projectNames or projectIds must be provided.");
+        }
         const payload = {
             id,
             trace,
@@ -3788,7 +3822,7 @@ export class Client {
         }
         return results_;
     }
-    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo, sessionId) {
         const evalResults = this._selectEvalResults(evaluatorResponse);
         const feedbacks = [];
         for (const res of evalResults) {
@@ -3812,14 +3846,20 @@ export class Client {
                 sourceRunId: res.sourceRunId,
                 feedbackConfig: res.feedbackConfig,
                 feedbackSourceType: "model",
-                sessionId: run?.session_id,
+                sessionId: run?.session_id ?? sessionId,
                 startTime: run?.start_time,
             }));
         }
         return [evalResults, feedbacks];
     }
-    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
-        const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
+    async logEvaluationFeedback(evaluatorResponseOrParams, run, sourceInfo, sessionId) {
+        if (evaluatorResponseOrParams != null &&
+            typeof evaluatorResponseOrParams === "object" &&
+            "evaluatorResponse" in evaluatorResponseOrParams) {
+            const [results] = await this._logEvaluationFeedback(evaluatorResponseOrParams.evaluatorResponse, evaluatorResponseOrParams.run, evaluatorResponseOrParams.sourceInfo, evaluatorResponseOrParams.projectId);
+            return results;
+        }
+        const [results] = await this._logEvaluationFeedback(evaluatorResponseOrParams, run, sourceInfo, sessionId);
         return results;
     }
     /**

package/dist/evaluation/_runner.cjs

@@ -342,6 +342,8 @@ class _ExperimentManager {
             experiment: project,
             metadata: this._metadata,
             client: this.client,
+            runs: this._runs,
+            _runsArray: this._runsArray,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
             numRepetitions: this._numRepetitions,
@@ -542,7 +544,11 @@ class _ExperimentManager {
                     tracingEnabled: true,
                 };
                 const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
-                evaluationResults.results.push(...(await fields.client.logEvaluationFeedback(evaluatorResponse, run)));
+                evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
+                    evaluatorResponse,
+                    run,
+                    projectId: this._getExperiment().id,
+                })));
             }
             catch (e) {
                 console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
@@ -1049,7 +1055,12 @@ async function _resolveExperiment(experiment, runs, client) {
         const firstRun = await runsCloneIterator
             .next()
             .then((result) => result.value);
-        const retrievedExperiment = await client.readProject(firstRun.sessionId);
+        if (firstRun?.session_id == null) {
+            throw new Error("Experiment ID not found for provided runs.");
+        }
+        const retrievedExperiment = await client.readProject({
+            projectId: firstRun.session_id,
+        });
         if (!retrievedExperiment.name) {
             throw new Error("Experiment name not found for provided runs.");
         }

package/dist/evaluation/_runner.js

@@ -334,6 +334,8 @@ export class _ExperimentManager {
             experiment: project,
             metadata: this._metadata,
             client: this.client,
+            runs: this._runs,
+            _runsArray: this._runsArray,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
             numRepetitions: this._numRepetitions,
@@ -534,7 +536,11 @@ export class _ExperimentManager {
                     tracingEnabled: true,
                 };
                 const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
-                evaluationResults.results.push(...(await fields.client.logEvaluationFeedback(evaluatorResponse, run)));
+                evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
+                    evaluatorResponse,
+                    run,
+                    projectId: this._getExperiment().id,
+                })));
             }
             catch (e) {
                 console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
@@ -1040,7 +1046,12 @@ async function _resolveExperiment(experiment, runs, client) {
         const firstRun = await runsCloneIterator
             .next()
             .then((result) => result.value);
-        const retrievedExperiment = await client.readProject(firstRun.sessionId);
+        if (firstRun?.session_id == null) {
+            throw new Error("Experiment ID not found for provided runs.");
+        }
+        const retrievedExperiment = await client.readProject({
+            projectId: firstRun.session_id,
+        });
         if (!retrievedExperiment.name) {
             throw new Error("Experiment name not found for provided runs.");
         }

package/dist/evaluation/evaluate_comparative.cjs

@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTracesForExperiment = loadTracesForExperiment;
 exports.evaluateComparative = evaluateComparative;
 const index_js_1 = require("../utils/uuid/src/index.cjs");
 const index_js_2 = require("../index.cjs");
@@ -10,6 +11,7 @@ const shuffle_js_1 = require("../utils/shuffle.cjs");
 const async_caller_js_1 = require("../utils/async_caller.cjs");
 const index_js_3 = __importDefault(require("../utils/p-retry/index.cjs"));
 const traceable_js_1 = require("../traceable.cjs");
+const v2_migration_js_1 = require("../utils/v2_migration.cjs");
 function isExperimentResultsList(value) {
     return value.some((x) => typeof x !== "string");
 }
@@ -17,11 +19,15 @@ async function loadExperiment(client, experiment) {
     const value = typeof experiment === "string" ? experiment : experiment.experimentName;
     return client.readProject((0, index_js_1.validate)(value) ? { projectId: value } : { projectName: value });
 }
-async function loadTraces(client, experiment, options) {
-    const executionOrder = options.loadNested ? undefined : 1;
-    const runs = await client.listRuns((0, index_js_1.validate)(experiment)
-        ? { projectId: experiment, executionOrder }
-        : { projectName: experiment, executionOrder });
+async function loadTracesForExperiment(client, project, options) {
+    // v1 `/runs/query` returns 501 on SmithDB-only backends; use v2 when it's available.
+    const isRoot = options.loadNested ? undefined : true;
+    const runs = (await client._supportsSDBQuery())
+        ? await (0, v2_migration_js_1.loadTracesV2)(client, project, { isRoot })
+        : await client.listRuns({
+            projectId: project.id,
+            executionOrder: options.loadNested ? undefined : 1,
+        });
     const treeMap = {};
     const runIdMap = {};
     const results = [];
@@ -121,7 +127,9 @@ async function evaluateComparative(experiments, options) {
     if (viewUrl != null) {
         console.log(`View results at: ${viewUrl}`);
     }
-    const experimentRuns = await Promise.all(projects.map((p) => loadTraces(client, p.id, { loadNested: !!options.loadNested })));
+    const experimentRuns = await Promise.all(projects.map((project) => loadTracesForExperiment(client, project, {
+        loadNested: !!options.loadNested,
+    })));
     let exampleIdsIntersect;
     for (const runs of experimentRuns) {
         const exampleIdsSet = new Set(runs

package/dist/evaluation/evaluate_comparative.d.ts

@@ -1,7 +1,10 @@
 import { Client } from "../index.js";
-import { ComparativeExperiment, ComparisonEvaluationResult as ComparisonEvaluationResultRow, Example, Run } from "../schemas.js";
+import { ComparativeExperiment, ComparisonEvaluationResult as ComparisonEvaluationResultRow, Example, Run, TracerSession } from "../schemas.js";
 import { evaluate } from "./index.js";
 type ExperimentResults = Awaited<ReturnType<typeof evaluate>>;
+export declare function loadTracesForExperiment(client: Client, project: TracerSession, options: {
+    loadNested: boolean;
+}): Promise<Run[]>;
 /** @deprecated Use ComparativeEvaluatorNew instead: (args: { runs, example, inputs, outputs, referenceOutputs }) => ... */
 export type _ComparativeEvaluatorLegacy = (runs: Run[], example: Example) => ComparisonEvaluationResultRow | Promise<ComparisonEvaluationResultRow>;
 export type _ComparativeEvaluator = (args: {

package/dist/evaluation/evaluate_comparative.js

@@ -4,6 +4,7 @@ import { shuffle } from "../utils/shuffle.js";
 import { AsyncCaller } from "../utils/async_caller.js";
 import pRetry from "../utils/p-retry/index.js";
 import { getCurrentRunTree, traceable } from "../traceable.js";
+import { loadTracesV2 } from "../utils/v2_migration.js";
 function isExperimentResultsList(value) {
     return value.some((x) => typeof x !== "string");
 }
@@ -11,11 +12,15 @@ async function loadExperiment(client, experiment) {
     const value = typeof experiment === "string" ? experiment : experiment.experimentName;
     return client.readProject(validate(value) ? { projectId: value } : { projectName: value });
 }
-async function loadTraces(client, experiment, options) {
-    const executionOrder = options.loadNested ? undefined : 1;
-    const runs = await client.listRuns(validate(experiment)
-        ? { projectId: experiment, executionOrder }
-        : { projectName: experiment, executionOrder });
+export async function loadTracesForExperiment(client, project, options) {
+    // v1 `/runs/query` returns 501 on SmithDB-only backends; use v2 when it's available.
+    const isRoot = options.loadNested ? undefined : true;
+    const runs = (await client._supportsSDBQuery())
+        ? await loadTracesV2(client, project, { isRoot })
+        : await client.listRuns({
+            projectId: project.id,
+            executionOrder: options.loadNested ? undefined : 1,
+        });
     const treeMap = {};
     const runIdMap = {};
     const results = [];
@@ -115,7 +120,9 @@ export async function evaluateComparative(experiments, options) {
     if (viewUrl != null) {
         console.log(`View results at: ${viewUrl}`);
     }
-    const experimentRuns = await Promise.all(projects.map((p) => loadTraces(client, p.id, { loadNested: !!options.loadNested })));
+    const experimentRuns = await Promise.all(projects.map((project) => loadTracesForExperiment(client, project, {
+        loadNested: !!options.loadNested,
+    })));
     let exampleIdsIntersect;
     for (const runs of experimentRuns) {
         const exampleIdsSet = new Set(runs