langsmith 0.3.65 → 0.3.66

package/dist/index.cjs

@@ -10,4 +10,4 @@ Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true
 var project_js_1 = require("./utils/project.cjs");
 Object.defineProperty(exports, "getDefaultProjectName", { enumerable: true, get: function () { return project_js_1.getDefaultProjectName; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.65";
+exports.__version__ = "0.3.66";

package/dist/index.d.ts

@@ -3,4 +3,4 @@ export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, }
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
-export declare const __version__ = "0.3.65";
+export declare const __version__ = "0.3.66";

package/dist/index.js

@@ -3,4 +3,4 @@ export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.65";
+export const __version__ = "0.3.66";

package/dist/traceable.cjs

@@ -105,25 +105,6 @@ const _extractUsage = (runData) => {
         .usage_metadata;
     return runData.outputs?.usage_metadata ?? usageMetadataFromMetadata;
 };
-function validateExtractedUsageMetadata(data) {
-    const allowedKeys = new Set([
-        "input_tokens",
-        "output_tokens",
-        "total_tokens",
-        "input_token_details",
-        "output_token_details",
-        "input_cost",
-        "output_cost",
-        "total_cost",
-        "input_cost_details",
-        "output_cost_details",
-    ]);
-    const extraKeys = Object.keys(data).filter((key) => !allowedKeys.has(key));
-    if (extraKeys.length > 0) {
-        throw new Error(`Unexpected keys in usage metadata: ${extraKeys.join(", ")}`);
-    }
-    return data;
-}
 async function handleEnd(params) {
     const { runTree, on_end, postRunPromise, excludeInputs } = params;
     const onEnd = on_end;
@@ -152,7 +133,7 @@ const _populateUsageMetadata = (processedOutputs, runTree) => {
         if (usageMetadata !== undefined) {
             runTree.extra.metadata = {
                 ...runTree.extra.metadata,
-                usage_metadata: validateExtractedUsageMetadata(usageMetadata),
+                usage_metadata: usageMetadata,
             };
             processedOutputs.usage_metadata = usageMetadata;
         }

package/dist/traceable.js

@@ -101,25 +101,6 @@ const _extractUsage = (runData) => {
         .usage_metadata;
     return runData.outputs?.usage_metadata ?? usageMetadataFromMetadata;
 };
-function validateExtractedUsageMetadata(data) {
-    const allowedKeys = new Set([
-        "input_tokens",
-        "output_tokens",
-        "total_tokens",
-        "input_token_details",
-        "output_token_details",
-        "input_cost",
-        "output_cost",
-        "total_cost",
-        "input_cost_details",
-        "output_cost_details",
-    ]);
-    const extraKeys = Object.keys(data).filter((key) => !allowedKeys.has(key));
-    if (extraKeys.length > 0) {
-        throw new Error(`Unexpected keys in usage metadata: ${extraKeys.join(", ")}`);
-    }
-    return data;
-}
 async function handleEnd(params) {
     const { runTree, on_end, postRunPromise, excludeInputs } = params;
     const onEnd = on_end;
@@ -148,7 +129,7 @@ const _populateUsageMetadata = (processedOutputs, runTree) => {
         if (usageMetadata !== undefined) {
             runTree.extra.metadata = {
                 ...runTree.extra.metadata,
-                usage_metadata: validateExtractedUsageMetadata(usageMetadata),
+                usage_metadata: usageMetadata,
             };
             processedOutputs.usage_metadata = usageMetadata;
         }

package/dist/utils/jestlike/index.cjs

@@ -161,17 +161,29 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return (0, uuid_1.v5)(identifier, constants_js_1.UUID5_NAMESPACE);
     }
     async function syncExample(params) {
-        const { client, exampleId, inputs, outputs, metadata, createdAt, datasetId, } = params;
+        const { client, exampleId, inputs, outputs, metadata, split, createdAt, datasetId, } = params;
         let example;
         try {
             example = await client.readExample(exampleId);
+            const normalizedSplit = split
+                ? typeof split === "string"
+                    ? [split]
+                    : split
+                : undefined;
+            const { dataset_split, ...restMetadata } = example.metadata ?? {};
             if (_objectHash(example.inputs) !== _objectHash(inputs) ||
                 _objectHash(example.outputs ?? {}) !== _objectHash(outputs ?? {}) ||
-                example.dataset_id !== datasetId) {
+                example.dataset_id !== datasetId ||
+                (normalizedSplit !== undefined &&
+                    _objectHash(dataset_split ?? []) !==
+                        _objectHash(normalizedSplit ?? [])) ||
+                (metadata !== undefined &&
+                    _objectHash(restMetadata ?? {}) !== _objectHash(metadata ?? {}))) {
                 await client.updateExample(exampleId, {
                     inputs,
                     outputs,
                     metadata,
+                    split,
                     dataset_id: datasetId,
                 });
             }
@@ -182,6 +194,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     exampleId,
                     datasetId,
                     createdAt: new Date(createdAt ?? new Date()),
+                    split,
                     metadata,
                 });
             }
@@ -398,7 +411,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 lsParams.config?.enableTestTracking !== undefined) {
                 context.enableTestTracking = lsParams.config.enableTestTracking;
             }
-            const { id, config, inputs, ...rest } = lsParams;
+            const { id, config, inputs, split, metadata, ...rest } = lsParams;
             let referenceOutputs = rest.referenceOutputs;
             if (!referenceOutputs && "outputs" in rest) {
                 referenceOutputs = rest.outputs;
@@ -471,6 +484,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                         datasetId: dataset?.id,
                                         testTrackingEnabled: (0, globals_js_1.trackingEnabled)(testContext),
                                         repetition: i,
+                                        split,
                                     },
                                 }));
                                 (0, globals_js_1._logTestFeedback)({
@@ -536,7 +550,8 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     datasetId: dataset.id,
                                     inputs,
                                     outputs: referenceOutputs ?? {},
-                                    metadata: {},
+                                    metadata,
+                                    split,
                                     createdAt,
                                 }));
                             }

package/dist/utils/jestlike/index.d.ts

@@ -6,7 +6,7 @@ export declare function logFeedback(feedback: SimpleEvaluationResult, config?: {
     sourceRunId?: string;
 }): void;
 export declare function logOutputs(output: Record<string, unknown>): void;
-export declare function _objectHash(obj: KVMap, depth?: number): string;
+export declare function _objectHash(obj: KVMap | unknown[], depth?: number): string;
 export declare function generateWrapperFromJestlikeMethods(methods: Record<string, any>, testRunnerName: string): {
     test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: LangSmithJestlikeTestFunction<I, O>, timeout?: number) => void) & {
         only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: LangSmithJestlikeTestFunction<I, O>, timeout?: number) => void) & {

package/dist/utils/jestlike/index.js

@@ -114,17 +114,29 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return v5(identifier, UUID5_NAMESPACE);
     }
     async function syncExample(params) {
-        const { client, exampleId, inputs, outputs, metadata, createdAt, datasetId, } = params;
+        const { client, exampleId, inputs, outputs, metadata, split, createdAt, datasetId, } = params;
         let example;
         try {
             example = await client.readExample(exampleId);
+            const normalizedSplit = split
+                ? typeof split === "string"
+                    ? [split]
+                    : split
+                : undefined;
+            const { dataset_split, ...restMetadata } = example.metadata ?? {};
             if (_objectHash(example.inputs) !== _objectHash(inputs) ||
                 _objectHash(example.outputs ?? {}) !== _objectHash(outputs ?? {}) ||
-                example.dataset_id !== datasetId) {
+                example.dataset_id !== datasetId ||
+                (normalizedSplit !== undefined &&
+                    _objectHash(dataset_split ?? []) !==
+                        _objectHash(normalizedSplit ?? [])) ||
+                (metadata !== undefined &&
+                    _objectHash(restMetadata ?? {}) !== _objectHash(metadata ?? {}))) {
                 await client.updateExample(exampleId, {
                     inputs,
                     outputs,
                     metadata,
+                    split,
                     dataset_id: datasetId,
                 });
             }
@@ -135,6 +147,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     exampleId,
                     datasetId,
                     createdAt: new Date(createdAt ?? new Date()),
+                    split,
                     metadata,
                 });
             }
@@ -351,7 +364,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 lsParams.config?.enableTestTracking !== undefined) {
                 context.enableTestTracking = lsParams.config.enableTestTracking;
             }
-            const { id, config, inputs, ...rest } = lsParams;
+            const { id, config, inputs, split, metadata, ...rest } = lsParams;
             let referenceOutputs = rest.referenceOutputs;
             if (!referenceOutputs && "outputs" in rest) {
                 referenceOutputs = rest.outputs;
@@ -424,6 +437,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                         datasetId: dataset?.id,
                                         testTrackingEnabled: trackingEnabled(testContext),
                                         repetition: i,
+                                        split,
                                     },
                                 }));
                                 _logTestFeedback({
@@ -489,7 +503,8 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     datasetId: dataset.id,
                                     inputs,
                                     outputs: referenceOutputs ?? {},
-                                    metadata: {},
+                                    metadata,
+                                    split,
                                     createdAt,
                                 }));
                             }

package/dist/utils/jestlike/types.d.ts

@@ -1,6 +1,5 @@
-import { CreateProjectParams } from "../../client.js";
-import { EvaluationResult } from "../../evaluation/evaluator.js";
-import { Client } from "../../index.js";
+import type { CreateProjectParams, CreateExampleOptions, Client } from "../../client.js";
+import type { EvaluationResult } from "../../evaluation/evaluator.js";
 import type { RunTreeConfig } from "../../run_trees.js";
 import type { SimpleEvaluator } from "./vendor/evaluatedBy.js";
 export { type SimpleEvaluator };
@@ -15,7 +14,7 @@ export type LangSmithJestlikeWrapperParams<I, O> = {
     inputs: I;
     referenceOutputs?: O;
     config?: LangSmithJestlikeWrapperConfig;
-};
+} & Pick<CreateExampleOptions, "split" | "metadata">;
 export type LangSmithJestlikeDescribeWrapperConfig = {
     client?: Client;
     enableTestTracking?: boolean;
@@ -35,6 +34,7 @@ export type LangSmithJestlikeTestMetadata = {
     datasetId?: string;
     testTrackingEnabled: boolean;
     repetition: number;
+    split?: string | string[];
 };
 export type LangSmithJestlikeTestFunction<I, O> = (data: {
     inputs: I;

package/dist/wrappers/openai.cjs

@@ -72,7 +72,8 @@ function _combineChatCompletionChoices(choices
     }
     return {
         index: choices[0].index,
-        finish_reason: reversedChoices.find((c) => c.finish_reason) || null,
+        finish_reason: (reversedChoices.find((c) => c.finish_reason) || null)
+            ?.finish_reason,
         message: message,
     };
 }

package/dist/wrappers/openai.js

@@ -69,7 +69,8 @@ function _combineChatCompletionChoices(choices
     }
     return {
         index: choices[0].index,
-        finish_reason: reversedChoices.find((c) => c.finish_reason) || null,
+        finish_reason: (reversedChoices.find((c) => c.finish_reason) || null)
+            ?.finish_reason,
         message: message,
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.65",
+  "version": "0.3.66",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [