langsmith 0.2.0 → 0.2.1

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.0";
+exports.__version__ = "0.2.1";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.0";
+export declare const __version__ = "0.2.1";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.0";
+export const __version__ = "0.2.1";

package/dist/singletons/fetch.d.ts

@@ -5,7 +5,3 @@
  * @param fetch The new fetch functino to use.
  */
 export declare const overrideFetchImplementation: (fetch: (...args: any[]) => any) => void;
-/**
- * @internal
- */
-export declare const _getFetchImplementation: () => (...args: any[]) => any;

package/dist/vercel.cjs

@@ -0,0 +1,645 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AISDKExporter = void 0;
+const index_js_1 = require("./index.cjs");
+const uuid_1 = require("uuid");
+const traceable_js_1 = require("./singletons/traceable.cjs");
+const env_js_1 = require("./utils/env.cjs");
+// Attempt to convert CoreMessage to a LangChain-compatible format
+// which allows us to render messages more nicely in LangSmith
+function convertCoreToSmith(message) {
+    if (message.role === "assistant") {
+        const data = { content: message.content };
+        if (Array.isArray(message.content)) {
+            data.content = message.content.map((part) => {
+                if (part.type === "text") {
+                    return {
+                        type: "text",
+                        text: part.text,
+                        ...part.experimental_providerMetadata,
+                    };
+                }
+                if (part.type === "tool-call") {
+                    return {
+                        type: "tool_use",
+                        name: part.toolName,
+                        id: part.toolCallId,
+                        input: part.args,
+                        ...part.experimental_providerMetadata,
+                    };
+                }
+                return part;
+            });
+            const toolCalls = message.content.filter((part) => part.type === "tool-call");
+            if (toolCalls.length > 0) {
+                data.additional_kwargs ??= {};
+                data.additional_kwargs.tool_calls = toolCalls.map((part) => {
+                    return {
+                        id: part.toolCallId,
+                        type: "function",
+                        function: {
+                            name: part.toolName,
+                            id: part.toolCallId,
+                            arguments: JSON.stringify(part.args),
+                        },
+                    };
+                });
+            }
+        }
+        return { type: "ai", data };
+    }
+    if (message.role === "user") {
+        const data = { content: message.content };
+        if (Array.isArray(message.content)) {
+            data.content = message.content.map((part) => {
+                if (part.type === "text") {
+                    return {
+                        type: "text",
+                        text: part.text,
+                        ...part.experimental_providerMetadata,
+                    };
+                }
+                if (part.type === "image") {
+                    return {
+                        type: "image_url",
+                        image_url: part.image,
+                        ...part.experimental_providerMetadata,
+                    };
+                }
+                return part;
+            });
+        }
+        return { type: "human", data };
+    }
+    if (message.role === "system") {
+        return { type: "system", data: { content: message.content } };
+    }
+    if (message.role === "tool") {
+        const res = message.content.map((toolCall) => {
+            return {
+                type: "tool",
+                data: {
+                    content: JSON.stringify(toolCall.result),
+                    name: toolCall.toolName,
+                    tool_call_id: toolCall.toolCallId,
+                },
+            };
+        });
+        if (res.length === 1)
+            return res[0];
+        return res;
+    }
+    return message;
+}
+const tryJson = (str) => {
+    try {
+        if (!str)
+            return str;
+        if (typeof str !== "string")
+            return str;
+        return JSON.parse(str);
+    }
+    catch {
+        return str;
+    }
+};
+function stripNonAlphanumeric(input) {
+    return input.replace(/[-:.]/g, "");
+}
+function convertToDottedOrderFormat([seconds, nanoseconds], runId, executionOrder) {
+    // Date only has millisecond precision, so we use the microseconds to break
+    // possible ties, avoiding incorrect run order
+    const ms = Number(String(nanoseconds).slice(0, 3));
+    const ns = String(Number(String(nanoseconds).slice(3, 6)) + executionOrder)
+        .padStart(3, "0")
+        .slice(0, 3);
+    return (stripNonAlphanumeric(`${new Date(seconds * 1000 + ms).toISOString().slice(0, -1)}${ns}Z`) + runId);
+}
+function convertToTimestamp([seconds, nanoseconds]) {
+    const ms = String(nanoseconds).slice(0, 3);
+    return Number(String(seconds) + ms);
+}
+function sortByHr(a, b) {
+    if (a[0] !== b[0])
+        return Math.sign(a[0] - b[0]);
+    return Math.sign(a[1] - b[1]);
+}
+const ROOT = "$";
+const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47";
+const RUN_ID_METADATA_KEY = {
+    input: "langsmith:runId",
+    output: "ai.telemetry.metadata.langsmith:runId",
+};
+const RUN_NAME_METADATA_KEY = {
+    input: "langsmith:runName",
+    output: "ai.telemetry.metadata.langsmith:runName",
+};
+const TRACE_METADATA_KEY = {
+    input: "langsmith:trace",
+    output: "ai.telemetry.metadata.langsmith:trace",
+};
+const BAGGAGE_METADATA_KEY = {
+    input: "langsmith:baggage",
+    output: "ai.telemetry.metadata.langsmith:baggage",
+};
+const RESERVED_METADATA_KEYS = [
+    RUN_ID_METADATA_KEY.output,
+    RUN_NAME_METADATA_KEY.output,
+    TRACE_METADATA_KEY.output,
+    BAGGAGE_METADATA_KEY.output,
+];
+/**
+ * OpenTelemetry trace exporter for Vercel AI SDK.
+ *
+ * @example
+ * ```ts
+ * import { AISDKExporter } from "langsmith/vercel";
+ * import { Client } from "langsmith";
+ *
+ * import { generateText } from "ai";
+ * import { openai } from "@ai-sdk/openai";
+ *
+ * import { NodeSDK } from "@opentelemetry/sdk-node";
+ * import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+ *
+ * const client = new Client();
+ *
+ * const sdk = new NodeSDK({
+ *   traceExporter: new AISDKExporter({ client }),
+ *   instrumentations: [getNodeAutoInstrumentations()],
+ * });
+ *
+ * sdk.start();
+ *
+ * const res = await generateText({
+ *   model: openai("gpt-4o-mini"),
+ *   messages: [
+ *     {
+ *       role: "user",
+ *       content: "What color is the sky?",
+ *     },
+ *   ],
+ *   experimental_telemetry: AISDKExporter.getSettings({
+ *     runName: "langsmith_traced_call",
+ *     metadata: { userId: "123", language: "english" },
+ *   }),
+ * });
+ *
+ * await sdk.shutdown();
+ * ```
+ */
+class AISDKExporter {
+    constructor(args) {
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "traceByMap", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        /** @internal */
+        Object.defineProperty(this, "getSpanAttributeKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: (span, key) => {
+                const attributes = span.attributes;
+                return key in attributes && typeof attributes[key] === "string"
+                    ? attributes[key]
+                    : undefined;
+            }
+        });
+        this.client = args?.client ?? new index_js_1.Client();
+    }
+    static getSettings(settings) {
+        const { runId, runName, ...rest } = settings ?? {};
+        const metadata = { ...rest?.metadata };
+        if (runId != null)
+            metadata[RUN_ID_METADATA_KEY.input] = runId;
+        if (runName != null)
+            metadata[RUN_NAME_METADATA_KEY.input] = runName;
+        // attempt to obtain the run tree if used within a traceable function
+        let defaultEnabled = true;
+        try {
+            const runTree = (0, traceable_js_1.getCurrentRunTree)();
+            const headers = runTree.toHeaders();
+            metadata[TRACE_METADATA_KEY.input] = headers["langsmith-trace"];
+            metadata[BAGGAGE_METADATA_KEY.input] = headers["baggage"];
+            // honor the tracingEnabled flag if coming from traceable
+            if (runTree.tracingEnabled != null) {
+                defaultEnabled = runTree.tracingEnabled;
+            }
+        }
+        catch {
+            // pass
+        }
+        if (metadata[RUN_ID_METADATA_KEY.input] &&
+            metadata[TRACE_METADATA_KEY.input]) {
+            throw new Error("Cannot provide `runId` when used within traceable function.");
+        }
+        return { ...rest, isEnabled: rest.isEnabled ?? defaultEnabled, metadata };
+    }
+    /** @internal */
+    parseInteropFromMetadata(span) {
+        const userTraceId = this.getSpanAttributeKey(span, RUN_ID_METADATA_KEY.output);
+        const parentTrace = this.getSpanAttributeKey(span, TRACE_METADATA_KEY.output);
+        if (parentTrace && userTraceId) {
+            throw new Error(`Cannot provide both "${RUN_ID_METADATA_KEY.input}" and "${TRACE_METADATA_KEY.input}" metadata keys.`);
+        }
+        if (parentTrace) {
+            const parentRunTree = index_js_1.RunTree.fromHeaders({
+                "langsmith-trace": parentTrace,
+                baggage: this.getSpanAttributeKey(span, BAGGAGE_METADATA_KEY.output) || "",
+            });
+            if (!parentRunTree)
+                throw new Error("Unreachable code: empty parent run tree");
+            return { type: "traceable", parentRunTree };
+        }
+        if (userTraceId)
+            return { type: "user", userTraceId };
+        return undefined;
+    }
+    /** @internal */
+    getRunCreate(span) {
+        const runId = (0, uuid_1.v5)(span.spanContext().spanId, RUN_ID_NAMESPACE);
+        const parentRunId = span.parentSpanId
+            ? (0, uuid_1.v5)(span.parentSpanId, RUN_ID_NAMESPACE)
+            : undefined;
+        const asRunCreate = (rawConfig) => {
+            const aiMetadata = Object.keys(span.attributes)
+                .filter((key) => key.startsWith("ai.telemetry.metadata.") &&
+                !RESERVED_METADATA_KEYS.includes(key))
+                .reduce((acc, key) => {
+                acc[key.slice("ai.telemetry.metadata.".length)] =
+                    span.attributes[key];
+                return acc;
+            }, {});
+            if (("ai.telemetry.functionId" in span.attributes &&
+                span.attributes["ai.telemetry.functionId"]) ||
+                ("resource.name" in span.attributes && span.attributes["resource.name"])) {
+                aiMetadata["functionId"] =
+                    span.attributes["ai.telemetry.functionId"] ||
+                        span.attributes["resource.name"];
+            }
+            const parsedStart = convertToTimestamp(span.startTime);
+            const parsedEnd = convertToTimestamp(span.endTime);
+            let name = rawConfig.name;
+            // if user provided a custom name, only use it if it's the root
+            if (span.parentSpanId == null) {
+                name =
+                    this.getSpanAttributeKey(span, RUN_NAME_METADATA_KEY.output) || name;
+            }
+            const config = {
+                ...rawConfig,
+                name,
+                id: runId,
+                parent_run_id: parentRunId,
+                extra: {
+                    ...rawConfig.extra,
+                    metadata: {
+                        ...rawConfig.extra?.metadata,
+                        ...aiMetadata,
+                        "ai.operationId": span.attributes["ai.operationId"],
+                    },
+                },
+                session_name: (0, env_js_1.getLangSmithEnvironmentVariable)("PROJECT") ??
+                    (0, env_js_1.getLangSmithEnvironmentVariable)("SESSION"),
+                start_time: Math.min(parsedStart, parsedEnd),
+                end_time: Math.max(parsedStart, parsedEnd),
+            };
+            return config;
+        };
+        switch (span.name) {
+            case "ai.generateText.doGenerate":
+            case "ai.generateText":
+            case "ai.streamText.doStream":
+            case "ai.streamText": {
+                const inputs = (() => {
+                    if ("ai.prompt.messages" in span.attributes) {
+                        return {
+                            messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
+                        };
+                    }
+                    if ("ai.prompt" in span.attributes) {
+                        const input = tryJson(span.attributes["ai.prompt"]);
+                        if (typeof input === "object" &&
+                            input != null &&
+                            "messages" in input &&
+                            Array.isArray(input.messages)) {
+                            return {
+                                messages: input.messages.flatMap((i) => convertCoreToSmith(i)),
+                            };
+                        }
+                        return { input };
+                    }
+                    return {};
+                })();
+                const outputs = (() => {
+                    let result = undefined;
+                    if (span.attributes["ai.response.toolCalls"]) {
+                        let content = tryJson(span.attributes["ai.response.toolCalls"]);
+                        if (Array.isArray(content)) {
+                            content = content.map((i) => ({
+                                type: "tool-call",
+                                ...i,
+                                args: tryJson(i.args),
+                            }));
+                        }
+                        result = {
+                            llm_output: convertCoreToSmith({
+                                role: "assistant",
+                                content,
+                            }),
+                        };
+                    }
+                    else if (span.attributes["ai.response.text"]) {
+                        result = {
+                            llm_output: convertCoreToSmith({
+                                role: "assistant",
+                                content: span.attributes["ai.response.text"],
+                            }),
+                        };
+                    }
+                    if (span.attributes["ai.usage.completionTokens"]) {
+                        result ??= {};
+                        result.llm_output ??= {};
+                        result.llm_output.token_usage ??= {};
+                        result.llm_output.token_usage["completion_tokens"] =
+                            span.attributes["ai.usage.completionTokens"];
+                    }
+                    if (span.attributes["ai.usage.promptTokens"]) {
+                        result ??= {};
+                        result.llm_output ??= {};
+                        result.llm_output.token_usage ??= {};
+                        result.llm_output.token_usage["prompt_tokens"] =
+                            span.attributes["ai.usage.promptTokens"];
+                    }
+                    return result;
+                })();
+                const events = [];
+                const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
+                if (firstChunkEvent) {
+                    events.push({
+                        name: "new_token",
+                        time: convertToTimestamp(firstChunkEvent.time),
+                    });
+                }
+                // TODO: add first_token_time
+                return asRunCreate({
+                    run_type: "llm",
+                    name: span.attributes["ai.model.provider"],
+                    inputs,
+                    outputs,
+                    events,
+                    extra: {
+                        batch_size: 1,
+                        metadata: {
+                            ls_provider: span.attributes["ai.model.provider"]
+                                .split(".")
+                                .at(0),
+                            ls_model_type: span.attributes["ai.model.provider"]
+                                .split(".")
+                                .at(1),
+                            ls_model_name: span.attributes["ai.model.id"],
+                        },
+                    },
+                });
+            }
+            case "ai.toolCall": {
+                const args = tryJson(span.attributes["ai.toolCall.args"]);
+                let inputs = { args };
+                if (typeof args === "object" && args != null) {
+                    inputs = args;
+                }
+                const output = tryJson(span.attributes["ai.toolCall.result"]);
+                let outputs = { output };
+                if (typeof output === "object" && output != null) {
+                    outputs = output;
+                }
+                return asRunCreate({
+                    run_type: "tool",
+                    name: span.attributes["ai.toolCall.name"],
+                    inputs,
+                    outputs,
+                });
+            }
+            case "ai.streamObject":
+            case "ai.streamObject.doStream":
+            case "ai.generateObject":
+            case "ai.generateObject.doGenerate": {
+                const inputs = (() => {
+                    if ("ai.prompt.messages" in span.attributes) {
+                        return {
+                            messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
+                        };
+                    }
+                    if ("ai.prompt" in span.attributes) {
+                        return { input: tryJson(span.attributes["ai.prompt"]) };
+                    }
+                    return {};
+                })();
+                const outputs = (() => {
+                    let result = undefined;
+                    if (span.attributes["ai.response.object"]) {
+                        result = {
+                            output: tryJson(span.attributes["ai.response.object"]),
+                        };
+                    }
+                    if (span.attributes["ai.usage.completionTokens"]) {
+                        result ??= {};
+                        result.llm_output ??= {};
+                        result.llm_output.token_usage ??= {};
+                        result.llm_output.token_usage["completion_tokens"] =
+                            span.attributes["ai.usage.completionTokens"];
+                    }
+                    if (span.attributes["ai.usage.promptTokens"]) {
+                        result ??= {};
+                        result.llm_output ??= {};
+                        result.llm_output.token_usage ??= {};
+                        result.llm_output.token_usage["prompt_tokens"] =
+                            +span.attributes["ai.usage.promptTokens"];
+                    }
+                    return result;
+                })();
+                const events = [];
+                const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
+                if (firstChunkEvent) {
+                    events.push({
+                        name: "new_token",
+                        time: convertToTimestamp(firstChunkEvent.time),
+                    });
+                }
+                return asRunCreate({
+                    run_type: "llm",
+                    name: span.attributes["ai.model.provider"],
+                    inputs,
+                    outputs,
+                    events,
+                    extra: {
+                        batch_size: 1,
+                        metadata: {
+                            ls_provider: span.attributes["ai.model.provider"]
+                                .split(".")
+                                .at(0),
+                            ls_model_type: span.attributes["ai.model.provider"]
+                                .split(".")
+                                .at(1),
+                            ls_model_name: span.attributes["ai.model.id"],
+                        },
+                    },
+                });
+            }
+            case "ai.embed":
+            case "ai.embed.doEmbed":
+            case "ai.embedMany":
+            case "ai.embedMany.doEmbed":
+            default:
+                return undefined;
+        }
+    }
+    /** @internal */
+    isRootRun(span) {
+        switch (span.name) {
+            case "ai.generateText":
+            case "ai.streamText":
+            case "ai.generateObject":
+            case "ai.streamObject":
+            case "ai.embed":
+            case "ai.embedMany":
+                return true;
+            default:
+                return false;
+        }
+    }
+    export(spans, resultCallback) {
+        const typedSpans = spans
+            .slice()
+            .sort((a, b) => sortByHr(a.startTime, b.startTime));
+        for (const span of typedSpans) {
+            const { traceId, spanId } = span.spanContext();
+            const parentId = span.parentSpanId ?? undefined;
+            this.traceByMap[traceId] ??= {
+                childMap: {},
+                nodeMap: {},
+                relativeExecutionOrder: {},
+            };
+            const runId = (0, uuid_1.v5)(spanId, RUN_ID_NAMESPACE);
+            let parentRunId = parentId
+                ? (0, uuid_1.v5)(parentId, RUN_ID_NAMESPACE)
+                : undefined;
+            // in LangSmith we currently only support certain spans
+            // which may be deeply nested within other traces
+            if (this.isRootRun(span))
+                parentRunId = undefined;
+            const traceMap = this.traceByMap[traceId];
+            const run = this.getRunCreate(span);
+            if (!run)
+                continue;
+            traceMap.relativeExecutionOrder[parentRunId ?? ROOT] ??= -1;
+            traceMap.relativeExecutionOrder[parentRunId ?? ROOT] += 1;
+            traceMap.nodeMap[runId] ??= {
+                id: runId,
+                parentId: parentRunId,
+                startTime: span.startTime,
+                run,
+                sent: false,
+                executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? ROOT],
+            };
+            traceMap.childMap[parentRunId ?? ROOT] ??= [];
+            traceMap.childMap[parentRunId ?? ROOT].push(traceMap.nodeMap[runId]);
+            traceMap.interop = this.parseInteropFromMetadata(span);
+        }
+        // We separate `id`,
+        const sampled = [];
+        for (const traceId of Object.keys(this.traceByMap)) {
+            const traceMap = this.traceByMap[traceId];
+            const queue = traceMap.childMap[ROOT]?.map((item) => ({
+                item,
+                dottedOrder: convertToDottedOrderFormat(item.startTime, item.id, item.executionOrder),
+                traceId: item.id,
+            })) ?? [];
+            const seen = new Set();
+            while (queue.length) {
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                const task = queue.shift();
+                if (seen.has(task.item.id))
+                    continue;
+                if (!task.item.sent) {
+                    let override = {
+                        id: task.item.id,
+                        parent_run_id: task.item.parentId,
+                        dotted_order: task.dottedOrder,
+                        trace_id: task.traceId,
+                    };
+                    if (traceMap.interop) {
+                        // attach the run to a parent run tree
+                        // - id: preserve
+                        // - parent_run_id: use existing parent run id or hook to the provided run tree
+                        // - dotted_order: append to the dotted_order of the parent run tree
+                        // - trace_id: use from the existing run tree
+                        if (traceMap.interop.type === "traceable") {
+                            override = {
+                                id: override.id,
+                                parent_run_id: override.parent_run_id ?? traceMap.interop.parentRunTree.id,
+                                dotted_order: [
+                                    traceMap.interop.parentRunTree.dotted_order,
+                                    override.dotted_order,
+                                ]
+                                    .filter(Boolean)
+                                    .join("."),
+                                trace_id: traceMap.interop.parentRunTree.trace_id,
+                            };
+                        }
+                        else if (traceMap.interop.type === "user") {
+                            // Allow user to specify custom trace ID = run ID of the root run
+                            // - id: use user provided run ID if root run, otherwise preserve
+                            // - parent_run_id: use user provided run ID if root run, otherwise preserve
+                            // - dotted_order: replace the trace_id with the user provided run ID
+                            // - trace_id: use user provided run ID
+                            const userTraceId = traceMap.interop.userTraceId ?? (0, uuid_1.v4)();
+                            override = {
+                                id: override.id === override.trace_id ? userTraceId : override.id,
+                                parent_run_id: override.parent_run_id === override.trace_id
+                                    ? userTraceId
+                                    : override.parent_run_id,
+                                dotted_order: override.dotted_order.replace(override.trace_id, userTraceId),
+                                trace_id: userTraceId,
+                            };
+                        }
+                    }
+                    sampled.push([override, task.item.run]);
+                    task.item.sent = true;
+                }
+                const children = traceMap.childMap[task.item.id] ?? [];
+                queue.push(...children.map((child) => {
+                    return {
+                        item: child,
+                        dottedOrder: [
+                            task.dottedOrder,
+                            convertToDottedOrderFormat(child.startTime, child.id, child.executionOrder),
+                        ].join("."),
+                        traceId: task.traceId,
+                    };
+                }));
+            }
+        }
+        Promise.all(sampled.map(([override, value]) => this.client.createRun({ ...value, ...override }))).then(() => resultCallback({ code: 0 }), (error) => resultCallback({ code: 1, error }));
+    }
+    async shutdown() {
+        // find nodes which are incomplete
+        const incompleteNodes = Object.values(this.traceByMap).flatMap((trace) => Object.values(trace.nodeMap).filter((i) => !i.sent));
+        if (incompleteNodes.length > 0) {
+            console.warn("Some incomplete nodes were found before shutdown and not sent to LangSmith.");
+        }
+        await this.client?.awaitPendingTraceBatches();
+    }
+    async forceFlush() {
+        await this.client?.awaitPendingTraceBatches();
+    }
+}
+exports.AISDKExporter = AISDKExporter;