langchain 0.2.2 → 0.2.3

package/dist/util/tests/openai-stream.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/util/tests/openai-stream.test.js

@@ -0,0 +1,135 @@
+import { describe, it, expect, jest } from "@jest/globals";
+import fetchAdapter from "../axios-fetch-adapter.js";
+const mockFetchForOpenAIStream = async ({ chunks, status, contentType, }) => {
+    // Mock stream response chunks.
+    const stream = new ReadableStream({
+        async start(controller) {
+            chunks.forEach((chunk) => {
+                controller.enqueue(new TextEncoder().encode(chunk));
+            });
+            controller.close();
+        },
+    });
+    // Mock Fetch API call.
+    jest.spyOn(global, "fetch").mockImplementation(async () => new Response(stream, {
+        status,
+        headers: {
+            "Content-Type": contentType,
+        },
+    }));
+    let error = null;
+    let done = false;
+    const receivedChunks = [];
+    const resp = await fetchAdapter({
+        url: "https://example.com",
+        method: "POST",
+        responseType: "stream",
+        onmessage: (event) => {
+            if (event.data?.trim?.() === "[DONE]") {
+                done = true;
+            }
+            else {
+                receivedChunks.push(JSON.parse(event.data).choices[0].delta.content ?? "");
+            }
+        },
+    }).catch((err) => {
+        error = err;
+        return null;
+    });
+    return { resp, receivedChunks, error, done };
+};
+describe("OpenAI Stream Tests", () => {
+    it("should return a 200 response chunk by chunk", async () => {
+        // When stream mode enabled, OpenAI responds with a stream of `data: {...}\n\n` chunks
+        // followed by `data: [DONE]\n\n`.
+        const { resp, receivedChunks, error, done } = await mockFetchForOpenAIStream({
+            status: 200,
+            contentType: "text/event-stream",
+            chunks: [
+                'data: {"choices":[{"delta":{"role":"assistant"},"index":0,"finish_reason":null}]}\n\n',
+                'data: {"choices":[{"delta":{"content":"Hello"},"index":0,"finish_reason":null}]}\n\n',
+                'data: {"choices":[{"delta":{"content":" World"},"index":0,"finish_reason":null}]}\n\n',
+                'data: {"choices":[{"delta":{"content":"!"},"index":0,"finish_reason":null}]}\n\n',
+                'data: {"choices":[{"delta":{},"index":0,"finish_reason":"stop"}]}\n\n',
+                "data: [DONE]\n\n",
+            ],
+        });
+        expect(error).toEqual(null);
+        expect(resp?.status).toEqual(200);
+        expect(receivedChunks).toEqual(["", "Hello", " World", "!", ""]);
+        expect(done).toBe(true);
+    });
+    it("should handle OpenAI 400 json error", async () => {
+        // OpenAI returns errors with application/json content type.
+        // Even if stream mode is enabled, the error is returned as a normal JSON body.
+        // Error information is in the `error` field.
+        const { resp, receivedChunks, error } = await mockFetchForOpenAIStream({
+            status: 400,
+            contentType: "application/json",
+            chunks: [
+                JSON.stringify({
+                    error: {},
+                }),
+            ],
+        });
+        expect(error).toEqual(null);
+        expect(resp?.status).toEqual(400);
+        expect(resp?.data).toEqual({ error: {} });
+        expect(receivedChunks).toEqual([]);
+    });
+    it("should handle 500 non-json error", async () => {
+        const { resp, receivedChunks, error } = await mockFetchForOpenAIStream({
+            status: 500,
+            contentType: "text/plain",
+            chunks: ["Some error message..."],
+        });
+        expect(error).toEqual(null);
+        expect(resp?.status).toEqual(500);
+        expect(resp?.data).toEqual("Some error message...");
+        expect(receivedChunks).toEqual([]);
+    });
+    it("should throw on 500 non-json body with json content type", async () => {
+        const { resp, receivedChunks, error } = await mockFetchForOpenAIStream({
+            status: 500,
+            contentType: "application/json",
+            chunks: ["a non-json error body"],
+        });
+        expect(resp).toEqual(null);
+        expect(error?.message).toContain("Unexpected token");
+        expect(receivedChunks).toEqual([]);
+    });
+    it("should throw the generic error if non-stream content is detected", async () => {
+        const { resp, receivedChunks, error } = await mockFetchForOpenAIStream({
+            status: 200,
+            contentType: "text/plain",
+            chunks: ["a non-stream body"],
+        });
+        expect(resp).toEqual(null);
+        expect(error?.message).toBe("Expected content-type to be text/event-stream, Actual: text/plain");
+        expect(receivedChunks).toEqual([]);
+    });
+});
+describe("Azure OpenAI Stream Tests", () => {
+    it("should return a 200 response chunk by chunk", async () => {
+        // When stream mode enabled, Azure OpenAI responds with chunks without tailing blank line.
+        // In addition, Azure sends chunks in batch.
+        const { resp, receivedChunks, error, done } = await mockFetchForOpenAIStream({
+            status: 200,
+            contentType: "text/event-stream",
+            chunks: [
+                // First batch
+                'data: {"choices":[{"delta":{"role":"assistant"},"index":0,"finish_reason":null}]}\n\n' +
+                    'data: {"choices":[{"delta":{"content":"Hello"},"index":0,"finish_reason":null}]}\n\n' +
+                    'data: {"choices":[{"delta":{"content":" World"},"index":0,"finish_reason":null}]}\n\n',
+                // Second batch
+                'data: {"choices":[{"delta":{"content":"!"},"index":0,"finish_reason":null}]}\n\n' +
+                    'data: {"choices":[{"delta":{},"index":0,"finish_reason":"stop"}]}\n\n' +
+                    "data: [DONE]\n", // no blank line
+            ],
+        });
+        expect(error).toEqual(null);
+        expect(resp?.status).toEqual(200);
+        expect(receivedChunks).toEqual(["", "Hello", " World", "!", ""]);
+        expect(done).toBe(true);
+    });
+});

package/dist/util/tests/set.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/util/tests/set.test.js

@@ -0,0 +1,36 @@
+import { test, expect } from "@jest/globals";
+import { difference, intersection, union } from "../set.js";
+test("difference", () => {
+    const set1 = new Set(["a", "b"]);
+    const set2 = new Set(["b", "c"]);
+    const resultSet = difference(set1, set2);
+    expect(resultSet).toMatchInlineSnapshot(`
+      Set {
+        "a",
+      }
+    `);
+});
+test("intersection", () => {
+    const set1 = new Set(["a", "b", "c", "d"]);
+    const set2 = new Set(["b", "c", "e"]);
+    const resultSet = intersection(set1, set2);
+    expect(resultSet).toMatchInlineSnapshot(`
+      Set {
+        "b",
+        "c",
+      }
+    `);
+});
+test("union", () => {
+    const set1 = new Set(["a", "b"]);
+    const set2 = new Set(["c", "d"]);
+    const resultSet = union(set1, set2);
+    expect(resultSet).toMatchInlineSnapshot(`
+      Set {
+        "a",
+        "b",
+        "c",
+        "d",
+      }
+    `);
+});

package/dist/util/tests/sql_utils.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/util/tests/sql_utils.test.js

@@ -0,0 +1,50 @@
+import { test, expect } from "@jest/globals";
+import { DataSource } from "typeorm";
+import { getPromptTemplateFromDataSource, verifyIgnoreTablesExistInDatabase, verifyIncludeTablesExistInDatabase, } from "../sql_utils.js";
+import { SQL_SQLITE_PROMPT } from "../../chains/sql_db/sql_db_prompt.js";
+test("Find include tables when there are there", () => {
+    const includeTables = ["user", "score"];
+    const allTables = [
+        { tableName: "plop", columns: [{ columnName: "id" }] },
+        { tableName: "score", columns: [{ columnName: "id" }] },
+        { tableName: "user", columns: [{ columnName: "id" }] },
+        { tableName: "log", columns: [{ columnName: "id" }] },
+    ];
+    expect(() => verifyIncludeTablesExistInDatabase(allTables, includeTables)).not.toThrow();
+});
+test("Throw Error when include tables are not there", () => {
+    const includeTables = ["user", "score"];
+    const allTables = [
+        { tableName: "plop", columns: [{ columnName: "id" }] },
+        { tableName: "score", columns: [{ columnName: "id" }] },
+        { tableName: "log", columns: [{ columnName: "id" }] },
+    ];
+    expect(() => verifyIncludeTablesExistInDatabase(allTables, includeTables)).toThrow();
+});
+test("Find include tables when there are there", () => {
+    const includeTables = ["user", "score"];
+    const allTables = [
+        { tableName: "user", columns: [{ columnName: "id" }] },
+        { tableName: "plop", columns: [{ columnName: "id" }] },
+        { tableName: "score", columns: [{ columnName: "id" }] },
+        { tableName: "log", columns: [{ columnName: "id" }] },
+    ];
+    expect(() => verifyIgnoreTablesExistInDatabase(allTables, includeTables)).not.toThrow();
+});
+test("Throw Error when include tables are not there", () => {
+    const includeTables = ["user", "score"];
+    const allTables = [
+        { tableName: "plop", columns: [{ columnName: "id" }] },
+        { tableName: "score", columns: [{ columnName: "id" }] },
+        { tableName: "log", columns: [{ columnName: "id" }] },
+    ];
+    expect(() => verifyIgnoreTablesExistInDatabase(allTables, includeTables)).toThrow();
+});
+test.skip("return sqlite template when the DataSource is sqlite", () => {
+    const datasource = new DataSource({
+        type: "sqlite",
+        database: "Chinook.db",
+    });
+    const promptTemplate = getPromptTemplateFromDataSource(datasource);
+    expect(promptTemplate).toEqual(SQL_SQLITE_PROMPT);
+});

package/dist/vectorstores/tests/memory.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/vectorstores/tests/memory.test.js

@@ -0,0 +1,78 @@
+import { test, expect } from "@jest/globals";
+import { Document } from "@langchain/core/documents";
+import { SyntheticEmbeddings } from "@langchain/core/utils/testing";
+import { similarity } from "ml-distance";
+import { MemoryVectorStore } from "../memory.js";
+test("MemoryVectorStore with external ids", async () => {
+    const embeddings = new SyntheticEmbeddings({
+        vectorSize: 1536,
+    });
+    const store = new MemoryVectorStore(embeddings);
+    expect(store).toBeDefined();
+    await store.addDocuments([
+        { pageContent: "hello", metadata: { a: 1 } },
+        { pageContent: "hi", metadata: { a: 1 } },
+        { pageContent: "bye", metadata: { a: 1 } },
+        { pageContent: "what's this", metadata: { a: 1 } },
+    ]);
+    const results = await store.similaritySearch("hello", 1);
+    expect(results).toHaveLength(1);
+    expect(results).toEqual([
+        new Document({ metadata: { a: 1 }, pageContent: "hello" }),
+    ]);
+});
+test("MemoryVectorStore as retriever can filter metadata", async () => {
+    const embeddings = new SyntheticEmbeddings({
+        vectorSize: 1536,
+    });
+    const store = new MemoryVectorStore(embeddings);
+    const filterFunc = (doc) => {
+        const { metadata } = doc;
+        if (metadata.namespace <= 2) {
+            return true;
+        }
+        return false;
+    };
+    const retriever = store.asRetriever({
+        k: 2,
+        filter: filterFunc,
+    });
+    expect(retriever).toBeDefined();
+    await retriever.addDocuments([
+        { pageContent: "hello", metadata: { namespace: 1 } },
+        { pageContent: "hello", metadata: { namespace: 2 } },
+        { pageContent: "hello", metadata: { namespace: 3 } },
+        { pageContent: "hello", metadata: { namespace: 4 } },
+    ]);
+    const results = await retriever.getRelevantDocuments("hello");
+    expect(results).toHaveLength(2);
+    expect(results).toEqual([
+        new Document({ metadata: { namespace: 1 }, pageContent: "hello" }),
+        new Document({ metadata: { namespace: 2 }, pageContent: "hello" }),
+    ]);
+});
+test("MemoryVectorStore with custom similarity", async () => {
+    const embeddings = new SyntheticEmbeddings({
+        vectorSize: 1536,
+    });
+    let similarityCalled = false;
+    let similarityCalledCount = 0;
+    const store = new MemoryVectorStore(embeddings, {
+        similarity: (a, b) => {
+            similarityCalledCount += 1;
+            similarityCalled = true;
+            return similarity.cosine(a, b);
+        },
+    });
+    expect(store).toBeDefined();
+    await store.addDocuments([
+        { pageContent: "hello", metadata: { a: 1 } },
+        { pageContent: "hi", metadata: { a: 1 } },
+        { pageContent: "bye", metadata: { a: 1 } },
+        { pageContent: "what's this", metadata: { a: 1 } },
+    ]);
+    const results = await store.similaritySearch("hello", 3);
+    expect(similarityCalled).toBe(true);
+    expect(similarityCalledCount).toBe(4);
+    expect(results).toHaveLength(3);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -561,7 +561,8 @@
   },
   "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/langchain/",
   "scripts": {
-    "build": "yarn run build:deps && yarn clean && yarn build:esm && yarn build:cjs && yarn build:scripts",
+    "build": "yarn turbo:command build:internal --filter=langchain",
+    "build:internal": "yarn lc-build:v2 --create-entrypoints --pre --tree-shaking --gen-maps",
     "build:deps": "yarn run turbo:command build --filter=@langchain/openai --filter=@langchain/textsplitters --filter=@langchain/cohere --concurrency=1",
     "build:esm": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist/ && rimraf dist/tests dist/**/tests",
     "build:cjs": "NODE_OPTIONS=--max-old-space-size=4096 tsc --outDir dist-cjs/ -p tsconfig.cjs.json && yarn move-cjs-to-dist && rimraf dist-cjs",
@@ -572,7 +573,7 @@
     "lint": "yarn lint:eslint && yarn lint:dpdm",
     "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm",
     "precommit": "lint-staged",
-    "clean": "rimraf .turbo/ dist/ && NODE_OPTIONS=--max-old-space-size=4096 yarn lc-build --config ./langchain.config.js --create-entrypoints --pre --gen-maps",
+    "clean": "rm -rf .turbo dist/",
     "prepack": "yarn build",
     "release": "release-it --only-version --config .release-it.json",
     "test": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%",
@@ -599,7 +600,6 @@
     "@faker-js/faker": "^7.6.0",
     "@gomomento/sdk": "^1.51.1",
     "@gomomento/sdk-core": "^1.51.1",
-    "@google-ai/generativelanguage": "^2.5.0",
     "@jest/globals": "^29.5.0",
     "@langchain/cohere": "^0.0.8",
     "@langchain/scripts": "~0.0.14",
@@ -686,7 +686,6 @@
     "@gomomento/sdk": "^1.51.1",
     "@gomomento/sdk-core": "^1.51.1",
     "@gomomento/sdk-web": "^1.51.1",
-    "@google-ai/generativelanguage": "^2.5.0",
     "@mendable/firecrawl-js": "^0.0.13",
     "@notionhq/client": "^2.2.10",
     "@pinecone-database/pinecone": "*",
@@ -756,9 +755,6 @@
     "@gomomento/sdk-web": {
       "optional": true
     },
-    "@google-ai/generativelanguage": {
-      "optional": true
-    },
     "@mendable/firecrawl-js": {
       "optional": true
     },