@langchain/core 0.1.61 → 0.1.63

package/dist/language_models/base.cjs

@@ -21,6 +21,9 @@ const getModelNameForTiktoken = (modelName) => {
     if (modelName.startsWith("gpt-4-")) {
         return "gpt-4";
     }
+    if (modelName.startsWith("gpt-4o")) {
+        return "gpt-4o";
+    }
     return modelName;
 };
 exports.getModelNameForTiktoken = getModelNameForTiktoken;

package/dist/language_models/base.js

@@ -18,6 +18,9 @@ export const getModelNameForTiktoken = (modelName) => {
     if (modelName.startsWith("gpt-4-")) {
         return "gpt-4";
     }
+    if (modelName.startsWith("gpt-4o")) {
+        return "gpt-4o";
+    }
     return modelName;
 };
 export const getEmbeddingContextSize = (modelName) => {

package/dist/runnables/base.cjs

@@ -16,6 +16,8 @@ const root_listener_js_1 = require("../tracers/root_listener.cjs");
 const utils_js_1 = require("./utils.cjs");
 const index_js_1 = require("../singletons/index.cjs");
 const graph_js_1 = require("./graph.cjs");
+const wrappers_js_1 = require("./wrappers.cjs");
+const iter_js_1 = require("./iter.cjs");
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function _coerceToDict(value, defaultKey) {
     return value &&
@@ -448,46 +450,16 @@ class Runnable extends serializable_js_1.Serializable {
             await runnableStreamConsumePromise;
         }
     }
-    /**
-     * Generate a stream of events emitted by the internal steps of the runnable.
-     *
-     * Use to create an iterator over StreamEvents that provide real-time information
-     * about the progress of the runnable, including StreamEvents from intermediate
-     * results.
-     *
-     * A StreamEvent is a dictionary with the following schema:
-     *
-     * - `event`: string - Event names are of the format: on_[runnable_type]_(start|stream|end).
-     * - `name`: string - The name of the runnable that generated the event.
-     * - `run_id`: string - Randomly generated ID associated with the given execution of
-     *   the runnable that emitted the event. A child runnable that gets invoked as part of the execution of a
-     *   parent runnable is assigned its own unique ID.
-     * - `tags`: string[] - The tags of the runnable that generated the event.
-     * - `metadata`: Record<string, any> - The metadata of the runnable that generated the event.
-     * - `data`: Record<string, any>
-     *
-     * Below is a table that illustrates some events that might be emitted by various
-     * chains. Metadata fields have been omitted from the table for brevity.
-     * Chain definitions have been included after the table.
-     *
-     * | event                | name             | chunk                              | input                                         | output                                          |
-     * |----------------------|------------------|------------------------------------|-----------------------------------------------|-------------------------------------------------|
-     * | on_llm_start         | [model name]     |                                    | {'input': 'hello'}                            |                                                 |
-     * | on_llm_stream        | [model name]     | 'Hello' OR AIMessageChunk("hello") |                                               |                                                 |
-     * | on_llm_end           | [model name]     |                                    | 'Hello human!'                                |
-     * | on_chain_start       | format_docs      |                                    |                                               |                                                 |
-     * | on_chain_stream      | format_docs      | "hello world!, goodbye world!"     |                                               |                                                 |
-     * | on_chain_end         | format_docs      |                                    | [Document(...)]                               | "hello world!, goodbye world!"                  |
-     * | on_tool_start        | some_tool        |                                    | {"x": 1, "y": "2"}                            |                                                 |
-     * | on_tool_stream       | some_tool        |   {"x": 1, "y": "2"}               |                                               |                                                 |
-     * | on_tool_end          | some_tool        |                                    |                                               | {"x": 1, "y": "2"}                              |
-     * | on_retriever_start   | [retriever name] |                                    | {"query": "hello"}                            |                                                 |
-     * | on_retriever_chunk   | [retriever name] |  {documents: [...]}                |                                               |                                                 |
-     * | on_retriever_end     | [retriever name] |                                    | {"query": "hello"}                            | {documents: [...]}                              |
-     * | on_prompt_start      | [template_name]  |                                    | {"question": "hello"}                         |                                                 |
-     * | on_prompt_end        | [template_name]  |                                    | {"question": "hello"}                         | ChatPromptValue(messages: [SystemMessage, ...]) |
-     */
     async *streamEvents(input, options, streamOptions) {
+        if (options.encoding === "text/event-stream") {
+            const stream = await this._streamEvents(input, options, streamOptions);
+            yield* (0, wrappers_js_1.convertToHttpEventStream)(stream);
+        }
+        else {
+            yield* this._streamEvents(input, options, streamOptions);
+        }
+    }
+    async *_streamEvents(input, options, streamOptions) {
         if (options.version !== "v1") {
             throw new Error(`Only version "v1" of the events schema is currently supported.`);
         }
@@ -1375,6 +1347,44 @@ class RunnableLambda extends Runnable {
                             recursionLimit: (childConfig.recursionLimit ?? config_js_1.DEFAULT_RECURSION_LIMIT) - 1,
                         });
                     }
+                    else if ((0, iter_js_1.isAsyncIterable)(output)) {
+                        let finalOutput;
+                        for await (const chunk of (0, iter_js_1.consumeAsyncIterableInContext)(childConfig, output)) {
+                            if (finalOutput === undefined) {
+                                finalOutput = chunk;
+                            }
+                            else {
+                                // Make a best effort to gather, for any type that supports concat.
+                                try {
+                                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                                    finalOutput = (0, stream_js_1.concat)(finalOutput, chunk);
+                                }
+                                catch (e) {
+                                    finalOutput = chunk;
+                                }
+                            }
+                        }
+                        output = finalOutput;
+                    }
+                    else if ((0, iter_js_1.isIterator)(output)) {
+                        let finalOutput;
+                        for (const chunk of (0, iter_js_1.consumeIteratorInContext)(childConfig, output)) {
+                            if (finalOutput === undefined) {
+                                finalOutput = chunk;
+                            }
+                            else {
+                                // Make a best effort to gather, for any type that supports concat.
+                                try {
+                                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                                    finalOutput = (0, stream_js_1.concat)(finalOutput, chunk);
+                                }
+                                catch (e) {
+                                    finalOutput = chunk;
+                                }
+                            }
+                        }
+                        output = finalOutput;
+                    }
                     resolve(output);
                 }
                 catch (e) {
@@ -1429,6 +1439,16 @@ class RunnableLambda extends Runnable {
                 yield chunk;
             }
         }
+        else if ((0, iter_js_1.isAsyncIterable)(output)) {
+            for await (const chunk of (0, iter_js_1.consumeAsyncIterableInContext)(config, output)) {
+                yield chunk;
+            }
+        }
+        else if ((0, iter_js_1.isIterator)(output)) {
+            for (const chunk of (0, iter_js_1.consumeIteratorInContext)(config, output)) {
+                yield chunk;
+            }
+        }
         else {
             yield output;
         }

package/dist/runnables/base.d.ts

@@ -199,6 +199,13 @@ export declare abstract class Runnable<RunInput = any, RunOutput = any, CallOpti
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
+    _streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
     static isRunnable(thing: any): thing is Runnable;
     /**
      * Bind lifecycle listeners to a Runnable, returning a new Runnable.
@@ -257,6 +264,10 @@ export declare class RunnableBinding<RunInput, RunOutput, CallOptions extends Ru
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
     static isRunnableBinding(thing: any): thing is RunnableBinding<any, any, any>;
     /**
      * Bind lifecycle listeners to a Runnable, returning a new Runnable.

package/dist/runnables/base.js

@@ -10,6 +10,8 @@ import { RootListenersTracer } from "../tracers/root_listener.js";
 import { _RootEventFilter, isRunnableInterface } from "./utils.js";
 import { AsyncLocalStorageProviderSingleton } from "../singletons/index.js";
 import { Graph } from "./graph.js";
+import { convertToHttpEventStream } from "./wrappers.js";
+import { consumeAsyncIterableInContext, consumeIteratorInContext, isAsyncIterable, isIterator, } from "./iter.js";
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function _coerceToDict(value, defaultKey) {
     return value &&
@@ -441,46 +443,16 @@ export class Runnable extends Serializable {
             await runnableStreamConsumePromise;
         }
     }
-    /**
-     * Generate a stream of events emitted by the internal steps of the runnable.
-     *
-     * Use to create an iterator over StreamEvents that provide real-time information
-     * about the progress of the runnable, including StreamEvents from intermediate
-     * results.
-     *
-     * A StreamEvent is a dictionary with the following schema:
-     *
-     * - `event`: string - Event names are of the format: on_[runnable_type]_(start|stream|end).
-     * - `name`: string - The name of the runnable that generated the event.
-     * - `run_id`: string - Randomly generated ID associated with the given execution of
-     *   the runnable that emitted the event. A child runnable that gets invoked as part of the execution of a
-     *   parent runnable is assigned its own unique ID.
-     * - `tags`: string[] - The tags of the runnable that generated the event.
-     * - `metadata`: Record<string, any> - The metadata of the runnable that generated the event.
-     * - `data`: Record<string, any>
-     *
-     * Below is a table that illustrates some events that might be emitted by various
-     * chains. Metadata fields have been omitted from the table for brevity.
-     * Chain definitions have been included after the table.
-     *
-     * | event                | name             | chunk                              | input                                         | output                                          |
-     * |----------------------|------------------|------------------------------------|-----------------------------------------------|-------------------------------------------------|
-     * | on_llm_start         | [model name]     |                                    | {'input': 'hello'}                            |                                                 |
-     * | on_llm_stream        | [model name]     | 'Hello' OR AIMessageChunk("hello") |                                               |                                                 |
-     * | on_llm_end           | [model name]     |                                    | 'Hello human!'                                |
-     * | on_chain_start       | format_docs      |                                    |                                               |                                                 |
-     * | on_chain_stream      | format_docs      | "hello world!, goodbye world!"     |                                               |                                                 |
-     * | on_chain_end         | format_docs      |                                    | [Document(...)]                               | "hello world!, goodbye world!"                  |
-     * | on_tool_start        | some_tool        |                                    | {"x": 1, "y": "2"}                            |                                                 |
-     * | on_tool_stream       | some_tool        |   {"x": 1, "y": "2"}               |                                               |                                                 |
-     * | on_tool_end          | some_tool        |                                    |                                               | {"x": 1, "y": "2"}                              |
-     * | on_retriever_start   | [retriever name] |                                    | {"query": "hello"}                            |                                                 |
-     * | on_retriever_chunk   | [retriever name] |  {documents: [...]}                |                                               |                                                 |
-     * | on_retriever_end     | [retriever name] |                                    | {"query": "hello"}                            | {documents: [...]}                              |
-     * | on_prompt_start      | [template_name]  |                                    | {"question": "hello"}                         |                                                 |
-     * | on_prompt_end        | [template_name]  |                                    | {"question": "hello"}                         | ChatPromptValue(messages: [SystemMessage, ...]) |
-     */
     async *streamEvents(input, options, streamOptions) {
+        if (options.encoding === "text/event-stream") {
+            const stream = await this._streamEvents(input, options, streamOptions);
+            yield* convertToHttpEventStream(stream);
+        }
+        else {
+            yield* this._streamEvents(input, options, streamOptions);
+        }
+    }
+    async *_streamEvents(input, options, streamOptions) {
         if (options.version !== "v1") {
             throw new Error(`Only version "v1" of the events schema is currently supported.`);
         }
@@ -1362,6 +1334,44 @@ export class RunnableLambda extends Runnable {
                             recursionLimit: (childConfig.recursionLimit ?? DEFAULT_RECURSION_LIMIT) - 1,
                         });
                     }
+                    else if (isAsyncIterable(output)) {
+                        let finalOutput;
+                        for await (const chunk of consumeAsyncIterableInContext(childConfig, output)) {
+                            if (finalOutput === undefined) {
+                                finalOutput = chunk;
+                            }
+                            else {
+                                // Make a best effort to gather, for any type that supports concat.
+                                try {
+                                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                                    finalOutput = concat(finalOutput, chunk);
+                                }
+                                catch (e) {
+                                    finalOutput = chunk;
+                                }
+                            }
+                        }
+                        output = finalOutput;
+                    }
+                    else if (isIterator(output)) {
+                        let finalOutput;
+                        for (const chunk of consumeIteratorInContext(childConfig, output)) {
+                            if (finalOutput === undefined) {
+                                finalOutput = chunk;
+                            }
+                            else {
+                                // Make a best effort to gather, for any type that supports concat.
+                                try {
+                                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                                    finalOutput = concat(finalOutput, chunk);
+                                }
+                                catch (e) {
+                                    finalOutput = chunk;
+                                }
+                            }
+                        }
+                        output = finalOutput;
+                    }
                     resolve(output);
                 }
                 catch (e) {
@@ -1416,6 +1426,16 @@ export class RunnableLambda extends Runnable {
                 yield chunk;
             }
         }
+        else if (isAsyncIterable(output)) {
+            for await (const chunk of consumeAsyncIterableInContext(config, output)) {
+                yield chunk;
+            }
+        }
+        else if (isIterator(output)) {
+            for (const chunk of consumeIteratorInContext(config, output)) {
+                yield chunk;
+            }
+        }
         else {
             yield output;
         }

package/dist/runnables/iter.cjs

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.consumeAsyncIterableInContext = exports.consumeIteratorInContext = exports.isAsyncIterable = exports.isIterator = void 0;
+const index_js_1 = require("../singletons/index.cjs");
+function isIterator(thing) {
+    return (typeof thing === "object" &&
+        thing !== null &&
+        typeof thing[Symbol.iterator] === "function" &&
+        // avoid detecting array/set as iterator
+        typeof thing.next === "function");
+}
+exports.isIterator = isIterator;
+function isAsyncIterable(thing) {
+    return (typeof thing === "object" &&
+        thing !== null &&
+        typeof thing[Symbol.asyncIterator] ===
+            "function");
+}
+exports.isAsyncIterable = isAsyncIterable;
+function* consumeIteratorInContext(context, iter) {
+    const storage = index_js_1.AsyncLocalStorageProviderSingleton.getInstance();
+    while (true) {
+        const { value, done } = storage.run(context, iter.next.bind(iter));
+        if (done) {
+            break;
+        }
+        else {
+            yield value;
+        }
+    }
+}
+exports.consumeIteratorInContext = consumeIteratorInContext;
+async function* consumeAsyncIterableInContext(context, iter) {
+    const storage = index_js_1.AsyncLocalStorageProviderSingleton.getInstance();
+    const iterator = iter[Symbol.asyncIterator]();
+    while (true) {
+        const { value, done } = await storage.run(context, iterator.next.bind(iter));
+        if (done) {
+            break;
+        }
+        else {
+            yield value;
+        }
+    }
+}
+exports.consumeAsyncIterableInContext = consumeAsyncIterableInContext;

package/dist/runnables/iter.d.ts

@@ -0,0 +1,5 @@
+import { RunnableConfig } from "./config.js";
+export declare function isIterator(thing: unknown): thing is IterableIterator<unknown>;
+export declare function isAsyncIterable(thing: unknown): thing is AsyncIterable<unknown>;
+export declare function consumeIteratorInContext<T>(context: Partial<RunnableConfig> | undefined, iter: IterableIterator<T>): IterableIterator<T>;
+export declare function consumeAsyncIterableInContext<T>(context: Partial<RunnableConfig> | undefined, iter: AsyncIterable<T>): AsyncIterableIterator<T>;

package/dist/runnables/iter.js

@@ -0,0 +1,39 @@
+import { AsyncLocalStorageProviderSingleton } from "../singletons/index.js";
+export function isIterator(thing) {
+    return (typeof thing === "object" &&
+        thing !== null &&
+        typeof thing[Symbol.iterator] === "function" &&
+        // avoid detecting array/set as iterator
+        typeof thing.next === "function");
+}
+export function isAsyncIterable(thing) {
+    return (typeof thing === "object" &&
+        thing !== null &&
+        typeof thing[Symbol.asyncIterator] ===
+            "function");
+}
+export function* consumeIteratorInContext(context, iter) {
+    const storage = AsyncLocalStorageProviderSingleton.getInstance();
+    while (true) {
+        const { value, done } = storage.run(context, iter.next.bind(iter));
+        if (done) {
+            break;
+        }
+        else {
+            yield value;
+        }
+    }
+}
+export async function* consumeAsyncIterableInContext(context, iter) {
+    const storage = AsyncLocalStorageProviderSingleton.getInstance();
+    const iterator = iter[Symbol.asyncIterator]();
+    while (true) {
+        const { value, done } = await storage.run(context, iterator.next.bind(iter));
+        if (done) {
+            break;
+        }
+        else {
+            yield value;
+        }
+    }
+}

package/dist/runnables/remote.d.ts

@@ -30,5 +30,9 @@ export declare class RemoteRunnable<RunInput, RunOutput, CallOptions extends Run
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
 }
 export {};

package/dist/runnables/wrappers.cjs

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToHttpEventStream = void 0;
+const stream_js_1 = require("../utils/stream.cjs");
+function convertToHttpEventStream(stream) {
+    const encoder = new TextEncoder();
+    const finalStream = new ReadableStream({
+        async start(controller) {
+            for await (const chunk of stream) {
+                controller.enqueue(encoder.encode(`event: data\ndata: ${JSON.stringify(chunk)}\n\n`));
+            }
+            controller.enqueue(encoder.encode("event: end\n\n"));
+            controller.close();
+        },
+    });
+    return stream_js_1.IterableReadableStream.fromReadableStream(finalStream);
+}
+exports.convertToHttpEventStream = convertToHttpEventStream;

package/dist/runnables/wrappers.d.ts

@@ -0,0 +1,2 @@
+import { IterableReadableStream } from "../utils/stream.js";
+export declare function convertToHttpEventStream(stream: AsyncGenerator): IterableReadableStream<Uint8Array>;

package/dist/runnables/wrappers.js

@@ -0,0 +1,14 @@
+import { IterableReadableStream } from "../utils/stream.js";
+export function convertToHttpEventStream(stream) {
+    const encoder = new TextEncoder();
+    const finalStream = new ReadableStream({
+        async start(controller) {
+            for await (const chunk of stream) {
+                controller.enqueue(encoder.encode(`event: data\ndata: ${JSON.stringify(chunk)}\n\n`));
+            }
+            controller.enqueue(encoder.encode("event: end\n\n"));
+            controller.close();
+        },
+    });
+    return IterableReadableStream.fromReadableStream(finalStream);
+}

package/dist/singletons/index.cjs

@@ -7,7 +7,7 @@ class MockAsyncLocalStorage {
         return undefined;
     }
     run(_store, callback) {
-        callback();
+        return callback();
     }
 }
 exports.MockAsyncLocalStorage = MockAsyncLocalStorage;

package/dist/singletons/index.d.ts

@@ -1,10 +1,10 @@
 export interface AsyncLocalStorageInterface {
     getStore: () => any | undefined;
-    run: (store: any, callback: () => any) => any;
+    run: <T>(store: any, callback: () => T) => T;
 }
 export declare class MockAsyncLocalStorage implements AsyncLocalStorageInterface {
     getStore(): any;
-    run(_store: any, callback: () => any): any;
+    run<T>(_store: any, callback: () => T): T;
 }
 declare class AsyncLocalStorageProvider {
     private asyncLocalStorage;

package/dist/singletons/index.js

@@ -4,7 +4,7 @@ export class MockAsyncLocalStorage {
         return undefined;
     }
     run(_store, callback) {
-        callback();
+        return callback();
     }
 }
 class AsyncLocalStorageProvider {

package/dist/tracers/base.cjs

@@ -11,9 +11,9 @@ function _coerceToDict(value, defaultKey) {
 function stripNonAlphanumeric(input) {
     return input.replace(/[-:.]/g, "");
 }
-function convertToDottedOrderFormat(epoch, runId) {
-    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}000Z`) +
-        runId);
+function convertToDottedOrderFormat(epoch, runId, executionOrder) {
+    const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
+    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
 class BaseTracer extends base_js_1.BaseCallbackHandler {
     constructor(_fields) {
@@ -42,7 +42,7 @@ class BaseTracer extends base_js_1.BaseCallbackHandler {
         parentRun.child_runs.push(childRun);
     }
     async _startTrace(run) {
-        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id);
+        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id, run.execution_order);
         const storedRun = { ...run };
         if (storedRun.parent_run_id !== undefined) {
             const parentRun = this.runMap.get(storedRun.parent_run_id);

package/dist/tracers/base.js

@@ -8,9 +8,9 @@ function _coerceToDict(value, defaultKey) {
 function stripNonAlphanumeric(input) {
     return input.replace(/[-:.]/g, "");
 }
-function convertToDottedOrderFormat(epoch, runId) {
-    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}000Z`) +
-        runId);
+function convertToDottedOrderFormat(epoch, runId, executionOrder) {
+    const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
+    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
 export class BaseTracer extends BaseCallbackHandler {
     constructor(_fields) {
@@ -39,7 +39,7 @@ export class BaseTracer extends BaseCallbackHandler {
         parentRun.child_runs.push(childRun);
     }
     async _startTrace(run) {
-        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id);
+        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id, run.execution_order);
         const storedRun = { ...run };
         if (storedRun.parent_run_id !== undefined) {
             const parentRun = this.runMap.get(storedRun.parent_run_id);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/core",
-  "version": "0.1.61",
+  "version": "0.1.63",
   "description": "Core LangChain.js abstractions and schemas",
   "type": "module",
   "engines": {
@@ -43,7 +43,7 @@
     "ansi-styles": "^5.0.0",
     "camelcase": "6",
     "decamelize": "1.2.0",
-    "js-tiktoken": "^1.0.8",
+    "js-tiktoken": "^1.0.12",
     "langsmith": "~0.1.7",
     "ml-distance": "^4.0.0",
     "mustache": "^4.2.0",