langsmith 0.2.15 → 0.2.16-rc.1

package/dist/client.d.ts

@@ -164,6 +164,14 @@ export type CreateExampleOptions = {
     /** The ID of the source run associated with this example. */
     sourceRunId?: string;
 };
+export type CreateProjectParams = {
+    projectName: string;
+    description?: string | null;
+    metadata?: RecordStringAny | null;
+    upsert?: boolean;
+    projectExtra?: RecordStringAny | null;
+    referenceDatasetId?: string | null;
+};
 type AutoBatchQueueItem = {
     action: "create" | "update";
     item: RunCreate | RunUpdate;
@@ -395,14 +403,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     listSharedExamples(shareToken: string, options?: {
         exampleIds?: string[];
     }): Promise<Example[]>;
-    createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, }: {
-        projectName: string;
-        description?: string | null;
-        metadata?: RecordStringAny | null;
-        upsert?: boolean;
-        projectExtra?: RecordStringAny | null;
-        referenceDatasetId?: string | null;
-    }): Promise<TracerSession>;
+    createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, }: CreateProjectParams): Promise<TracerSession>;
     updateProject(projectId: string, { name, description, metadata, projectExtra, endTime, }: {
         name?: string | null;
         description?: string | null;

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.15";
+exports.__version__ = "0.2.16-rc.1";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.15";
+export declare const __version__ = "0.2.16-rc.1";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.15";
+export const __version__ = "0.2.16-rc.1";

package/dist/jest/index.cjs

@@ -0,0 +1,45 @@
+"use strict";
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logOutput = exports.logFeedback = exports.toBeSemanticCloseTo = exports.toBeAbsoluteCloseTo = exports.toBeRelativeCloseTo = exports.expect = exports.describe = exports.it = exports.test = void 0;
+const globals_1 = require("@jest/globals");
+const matchers_js_1 = require("../utils/jestlike/matchers.cjs");
+Object.defineProperty(exports, "toBeRelativeCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeRelativeCloseTo; } });
+Object.defineProperty(exports, "toBeAbsoluteCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeAbsoluteCloseTo; } });
+Object.defineProperty(exports, "toBeSemanticCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeSemanticCloseTo; } });
+const index_js_1 = require("../utils/jestlike/index.cjs");
+Object.defineProperty(exports, "logFeedback", { enumerable: true, get: function () { return index_js_1.logFeedback; } });
+Object.defineProperty(exports, "logOutput", { enumerable: true, get: function () { return index_js_1.logOutput; } });
+const index_js_2 = require("../utils/jestlike/index.cjs");
+globals_1.expect.extend({
+    toBeRelativeCloseTo: matchers_js_1.toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
+});
+const { test, it, describe, expect } = (0, index_js_2.generateWrapperFromJestlikeMethods)({
+    expect: globals_1.expect,
+    test: globals_1.test,
+    describe: globals_1.describe,
+    beforeAll: globals_1.beforeAll,
+    afterAll: globals_1.afterAll,
+});
+exports.test = test;
+exports.it = it;
+exports.describe = describe;
+exports.expect = expect;
+__exportStar(require("../utils/jestlike/types.cjs"), exports);

package/dist/jest/index.d.ts

@@ -0,0 +1,99 @@
+/// <reference types="jest" />
+import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, type AbsoluteCloseToMatcherOptions, type SemanticCloseToMatcherOptions, type RelativeCloseToMatcherOptions } from "../utils/jestlike/matchers.js";
+import type { SimpleEvaluator } from "../utils/jestlike/vendor/evaluatedBy.js";
+import { logFeedback, logOutput } from "../utils/jestlike/index.js";
+declare global {
+    namespace jest {
+        interface AsymmetricMatchers {
+            toBeRelativeCloseTo(expected: string, options?: RelativeCloseToMatcherOptions): Promise<void>;
+            toBeAbsoluteCloseTo(expected: string, options?: AbsoluteCloseToMatcherOptions): Promise<void>;
+            toBeSemanticCloseTo(expected: string, options?: SemanticCloseToMatcherOptions): Promise<void>;
+        }
+        interface Matchers<R> {
+            toBeRelativeCloseTo(expected: string, options?: RelativeCloseToMatcherOptions): Promise<R>;
+            toBeAbsoluteCloseTo(expected: string, options?: AbsoluteCloseToMatcherOptions): Promise<R>;
+            toBeSemanticCloseTo(expected: string, options?: SemanticCloseToMatcherOptions): Promise<R>;
+            evaluatedBy(evaluator: SimpleEvaluator): jest.Matchers<Promise<R>> & {
+                not: jest.Matchers<Promise<R>>;
+                resolves: jest.Matchers<Promise<R>>;
+                rejects: jest.Matchers<Promise<R>>;
+            };
+        }
+    }
+}
+declare const test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+    inputs: I;
+    expected: O;
+}) => unknown, timeout?: number | undefined) => void) & {
+    only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        expected: O;
+    }) => unknown, timeout?: number | undefined) => void) & {
+        each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+            inputs: I_1;
+            expected: O_1;
+        }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+            inputs: I_1;
+            expected: O_1;
+        }) => unknown, timeout?: number | undefined) => void;
+    };
+    skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        expected: O;
+    }) => unknown, timeout?: number | undefined) => void) & {
+        each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+            inputs: I_1;
+            expected: O_1;
+        }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+            inputs: I_1;
+            expected: O_1;
+        }) => unknown, timeout?: number | undefined) => void;
+    };
+    each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+        inputs: I_1;
+        expected: O_1;
+    }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+        inputs: I_1;
+        expected: O_1;
+    }) => unknown, timeout?: number | undefined) => void;
+}, it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+    inputs: I;
+    expected: O;
+}) => unknown, timeout?: number | undefined) => void) & {
+    only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        expected: O;
+    }) => unknown, timeout?: number | undefined) => void) & {
+        each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+            inputs: I_1;
+            expected: O_1;
+        }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+            inputs: I_1;
+            expected: O_1;
+        }) => unknown, timeout?: number | undefined) => void;
+    };
+    skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        expected: O;
+    }) => unknown, timeout?: number | undefined) => void) & {
+        each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+            inputs: I_1;
+            expected: O_1;
+        }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+            inputs: I_1;
+            expected: O_1;
+        }) => unknown, timeout?: number | undefined) => void;
+    };
+    each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: {
+        inputs: I_1;
+        expected: O_1;
+    }[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
+        inputs: I_1;
+        expected: O_1;
+    }) => unknown, timeout?: number | undefined) => void;
+}, describe: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper & {
+    only: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
+    skip: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
+}, expect: jest.Expect;
+export { test, it, describe, expect, toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, logFeedback, logOutput, };
+export * from "../utils/jestlike/types.js";

package/dist/jest/index.js

@@ -0,0 +1,20 @@
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
+import { expect as jestExpect, test as jestTest, describe as jestDescribe, beforeAll as jestBeforeAll, afterAll as jestAfterAll, } from "@jest/globals";
+import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "../utils/jestlike/matchers.js";
+import { logFeedback, logOutput } from "../utils/jestlike/index.js";
+import { generateWrapperFromJestlikeMethods } from "../utils/jestlike/index.js";
+jestExpect.extend({
+    toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo,
+});
+const { test, it, describe, expect } = generateWrapperFromJestlikeMethods({
+    expect: jestExpect,
+    test: jestTest,
+    describe: jestDescribe,
+    beforeAll: jestBeforeAll,
+    afterAll: jestAfterAll,
+});
+export { test, it, describe, expect, toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, logFeedback, logOutput, };
+export * from "../utils/jestlike/types.js";

package/dist/jest/reporter.cjs

@@ -0,0 +1,7 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const reporter_js_1 = __importDefault(require("../utils/jestlike/reporter.cjs"));
+exports.default = reporter_js_1.default;

package/dist/jest/reporter.d.ts

@@ -0,0 +1,2 @@
+import CustomReporter from "../utils/jestlike/reporter.js";
+export default CustomReporter;

package/dist/jest/reporter.js

@@ -0,0 +1,2 @@
+import CustomReporter from "../utils/jestlike/reporter.js";
+export default CustomReporter;

package/dist/run_trees.cjs

@@ -270,7 +270,7 @@ class RunTree {
         return {
             id: uuid.v4(),
             run_type: "chain",
-            project_name: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_PROJECT") ??
+            project_name: (0, env_js_1.getLangSmithEnvironmentVariable)("PROJECT") ??
                 (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_SESSION") ?? // TODO: Deprecate
                 "default",
             child_runs: [],

package/dist/run_trees.d.ts

@@ -80,7 +80,7 @@ export declare class RunTree implements BaseRun {
     attachments?: Attachments;
     constructor(originalConfig: RunTreeConfig | RunTree);
     private static getDefaultConfig;
-    private static getSharedClient;
+    static getSharedClient(): Client;
     createChild(config: RunTreeConfig): RunTree;
     end(outputs?: KVMap, error?: string, endTime?: number, metadata?: KVMap): Promise<void>;
     private _convertToCreate;

package/dist/run_trees.js

@@ -1,5 +1,5 @@
 import * as uuid from "uuid";
-import { getEnvironmentVariable, getRuntimeEnvironment, } from "./utils/env.js";
+import { getEnvironmentVariable, getLangSmithEnvironmentVariable, getRuntimeEnvironment, } from "./utils/env.js";
 import { Client } from "./client.js";
 import { isTracingEnabled } from "./env.js";
 import { warnOnce } from "./utils/warn.js";
@@ -243,7 +243,7 @@ export class RunTree {
         return {
             id: uuid.v4(),
             run_type: "chain",
-            project_name: getEnvironmentVariable("LANGCHAIN_PROJECT") ??
+            project_name: getLangSmithEnvironmentVariable("PROJECT") ??
                 getEnvironmentVariable("LANGCHAIN_SESSION") ?? // TODO: Deprecate
                 "default",
             child_runs: [],

package/dist/singletons/traceable.cjs

@@ -38,7 +38,7 @@ const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/dist/singletons/traceable.js

@@ -35,7 +35,7 @@ export const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/dist/utils/jestlike/globals.cjs

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._logTestFeedback = exports.syncExamplePromises = exports.evaluatorLogFeedbackPromises = exports.trackingEnabled = exports.testWrapperAsyncLocalStorageInstance = void 0;
+const node_async_hooks_1 = require("node:async_hooks");
+const env_js_1 = require("../env.cjs");
+const env_js_2 = require("../../env.cjs");
+exports.testWrapperAsyncLocalStorageInstance = new node_async_hooks_1.AsyncLocalStorage();
+function trackingEnabled(context) {
+    if (typeof context.enableTestTracking === "boolean") {
+        return context.enableTestTracking;
+    }
+    if ((0, env_js_1.getEnvironmentVariable)("LANGSMITH_TEST_TRACKING") === "false") {
+        return false;
+    }
+    return (0, env_js_2.isTracingEnabled)();
+}
+exports.trackingEnabled = trackingEnabled;
+exports.evaluatorLogFeedbackPromises = new Set();
+exports.syncExamplePromises = new Map();
+function _logTestFeedback(params) {
+    const { exampleId, feedback, context, runTree, client, sourceRunId } = params;
+    if (trackingEnabled(context)) {
+        if (exampleId === undefined) {
+            throw new Error("Could not log feedback to LangSmith: missing example id. Please contact us for help.");
+        }
+        if (runTree === undefined) {
+            throw new Error("Could not log feedback to LangSmith: missing run information. Please contact us for help.");
+        }
+        exports.evaluatorLogFeedbackPromises.add((async () => {
+            await exports.syncExamplePromises.get(exampleId);
+            await client?.logEvaluationFeedback(feedback, runTree, sourceRunId !== undefined
+                ? { __run: { run_id: sourceRunId } }
+                : undefined);
+        })());
+    }
+    context.onFeedbackLogged?.(feedback);
+}
+exports._logTestFeedback = _logTestFeedback;

package/dist/utils/jestlike/globals.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" resolution-mode="require"/>
+import { AsyncLocalStorage } from "node:async_hooks";
+import { Dataset, TracerSession, Example } from "../../schemas.js";
+import { Client, CreateProjectParams } from "../../client.js";
+import { EvaluationResult } from "../../evaluation/evaluator.js";
+import { RunTree } from "../../run_trees.js";
+export type TestWrapperAsyncLocalStorageData = {
+    enableTestTracking?: boolean;
+    dataset?: Dataset;
+    createdAt: string;
+    projectConfig?: Partial<CreateProjectParams>;
+    project?: TracerSession;
+    setLoggedOutput?: (value: Record<string, unknown>) => void;
+    onFeedbackLogged?: (feedback: EvaluationResult) => void;
+    currentExample?: Partial<Example> & {
+        syncPromise?: Promise<Example>;
+    };
+    client: Client;
+    suiteUuid: string;
+    suiteName: string;
+};
+export declare const testWrapperAsyncLocalStorageInstance: AsyncLocalStorage<TestWrapperAsyncLocalStorageData>;
+export declare function trackingEnabled(context: TestWrapperAsyncLocalStorageData): boolean;
+export declare const evaluatorLogFeedbackPromises: Set<unknown>;
+export declare const syncExamplePromises: Map<any, any>;
+export declare function _logTestFeedback(params: {
+    exampleId?: string;
+    feedback: EvaluationResult;
+    context: TestWrapperAsyncLocalStorageData;
+    runTree?: RunTree;
+    client: Client;
+    sourceRunId?: string;
+}): void;

package/dist/utils/jestlike/globals.js

@@ -0,0 +1,33 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { getEnvironmentVariable } from "../env.js";
+import { isTracingEnabled } from "../../env.js";
+export const testWrapperAsyncLocalStorageInstance = new AsyncLocalStorage();
+export function trackingEnabled(context) {
+    if (typeof context.enableTestTracking === "boolean") {
+        return context.enableTestTracking;
+    }
+    if (getEnvironmentVariable("LANGSMITH_TEST_TRACKING") === "false") {
+        return false;
+    }
+    return isTracingEnabled();
+}
+export const evaluatorLogFeedbackPromises = new Set();
+export const syncExamplePromises = new Map();
+export function _logTestFeedback(params) {
+    const { exampleId, feedback, context, runTree, client, sourceRunId } = params;
+    if (trackingEnabled(context)) {
+        if (exampleId === undefined) {
+            throw new Error("Could not log feedback to LangSmith: missing example id. Please contact us for help.");
+        }
+        if (runTree === undefined) {
+            throw new Error("Could not log feedback to LangSmith: missing run information. Please contact us for help.");
+        }
+        evaluatorLogFeedbackPromises.add((async () => {
+            await syncExamplePromises.get(exampleId);
+            await client?.logEvaluationFeedback(feedback, runTree, sourceRunId !== undefined
+                ? { __run: { run_id: sourceRunId } }
+                : undefined);
+        })());
+    }
+    context.onFeedbackLogged?.(feedback);
+}