langchain 0.2.10 → 0.2.11

package/dist/retrievers/tests/time_weighted.test.js

@@ -1,320 +0,0 @@
-import { describe, expect, jest, test } from "@jest/globals";
-import { FakeEmbeddings } from "@langchain/core/utils/testing";
-import { MemoryVectorStore } from "../../vectorstores/memory.js";
-import { BUFFER_IDX, LAST_ACCESSED_AT_KEY, TimeWeightedVectorStoreRetriever, } from "../time_weighted.js";
-jest.useFakeTimers();
-const mockNow = new Date("2023-04-18 15:30");
-jest.setSystemTime(mockNow);
-const getSec = (date) => Math.floor(date.getTime() / 1000);
-const getMemoryStream = () => [
-    {
-        pageContent: "foo",
-        metadata: {
-            [BUFFER_IDX]: 0,
-            [LAST_ACCESSED_AT_KEY]: getSec(new Date("2023-04-18 12:00")),
-            created_at: getSec(new Date("2023-04-18 12:00")),
-        },
-    },
-    {
-        pageContent: "bar",
-        metadata: {
-            [BUFFER_IDX]: 1,
-            [LAST_ACCESSED_AT_KEY]: getSec(new Date("2023-04-18 13:00")),
-            created_at: getSec(new Date("2023-04-18 13:00")),
-        },
-    },
-    {
-        pageContent: "baz",
-        metadata: {
-            [BUFFER_IDX]: 2,
-            [LAST_ACCESSED_AT_KEY]: getSec(new Date("2023-04-18 11:00")),
-            created_at: getSec(new Date("2023-04-18 11:00")),
-        },
-    },
-];
-describe("Test getRelevantDocuments", () => {
-    test("Should fail on a vector store with documents that have not been added through the addDocuments method on the retriever", async () => {
-        const vectorStore = new MemoryVectorStore(new FakeEmbeddings());
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore,
-            memoryStream: [],
-            searchKwargs: 2,
-        });
-        await vectorStore.addDocuments([
-            { pageContent: "aaa", metadata: {} },
-            { pageContent: "aaaa", metadata: {} },
-            { pageContent: "bbb", metadata: {} },
-        ]);
-        const query = "aaa";
-        await expect(() => retriever.getRelevantDocuments(query)).rejects.toThrow();
-    });
-    test("For different pageContent with the same lastAccessedAt, return in descending order of similar words.", async () => {
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: [],
-            searchKwargs: 2,
-        });
-        await retriever.addDocuments([
-            { pageContent: "aaa", metadata: {} },
-            { pageContent: "aaaa", metadata: {} },
-            { pageContent: "bbb", metadata: {} },
-        ]);
-        const query = "aaa";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: "aaa",
-                metadata: {
-                    [BUFFER_IDX]: 0,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "aaaa",
-                metadata: {
-                    [BUFFER_IDX]: 1,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "bbb",
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-        ];
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-    test("Return in descending order of lastAccessedAt when memoryStream of the same pageContent", async () => {
-        const samePageContent = "Test query";
-        const samePageContentMemoryStream = getMemoryStream().map((doc) => ({
-            ...doc,
-            pageContent: samePageContent,
-        }));
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: samePageContentMemoryStream,
-        });
-        await retriever.addDocuments([
-            { pageContent: samePageContent, metadata: {} },
-        ]);
-        const query = "Test query";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: samePageContent,
-                metadata: {
-                    [BUFFER_IDX]: 3,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: samePageContent,
-                metadata: {
-                    [BUFFER_IDX]: 1,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 13:00")),
-                },
-            },
-            {
-                pageContent: samePageContent,
-                metadata: {
-                    [BUFFER_IDX]: 0,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 12:00")),
-                },
-            },
-            {
-                pageContent: samePageContent,
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 11:00")),
-                },
-            },
-        ];
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-    test("Return in descending order of lastAccessedAt when memoryStream of different pageContent", async () => {
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: getMemoryStream(),
-        });
-        await retriever.addDocuments([{ pageContent: "qux", metadata: {} }]);
-        const query = "Test query";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: "qux",
-                metadata: {
-                    [BUFFER_IDX]: 3,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "bar",
-                metadata: {
-                    [BUFFER_IDX]: 1,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 13:00")),
-                },
-            },
-            {
-                pageContent: "foo",
-                metadata: {
-                    [BUFFER_IDX]: 0,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 12:00")),
-                },
-            },
-            {
-                pageContent: "baz",
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 11:00")),
-                },
-            },
-        ];
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-    test("Return in descending order of lastAccessedAt when memoryStream of different pageContent and decayRate", async () => {
-        const decayRate = 0.5;
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: getMemoryStream(),
-            decayRate,
-        });
-        await retriever.addDocuments([{ pageContent: "qux", metadata: {} }]);
-        const query = "Test query";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: "qux",
-                metadata: {
-                    [BUFFER_IDX]: 3,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "bar",
-                metadata: {
-                    [BUFFER_IDX]: 1,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 13:00")),
-                },
-            },
-            {
-                pageContent: "foo",
-                metadata: {
-                    [BUFFER_IDX]: 0,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 12:00")),
-                },
-            },
-            {
-                pageContent: "baz",
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 11:00")),
-                },
-            },
-        ];
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-    test("Return in descending order of lastAccessedAt when memoryStream of different pageContent and k = 3", async () => {
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: getMemoryStream(),
-            k: 3,
-        });
-        await retriever.addDocuments([{ pageContent: "qux", metadata: {} }]);
-        const query = "Test query";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: "qux",
-                metadata: {
-                    [BUFFER_IDX]: 3,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "bar",
-                metadata: {
-                    [BUFFER_IDX]: 1,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 13:00")),
-                },
-            },
-            {
-                pageContent: "baz",
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 11:00")),
-                },
-            },
-        ];
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-    test("Return in descending order of lastAccessedAt when memoryStream of different pageContent and searchKwargs = 2", async () => {
-        const retriever = new TimeWeightedVectorStoreRetriever({
-            vectorStore: new MemoryVectorStore(new FakeEmbeddings()),
-            memoryStream: getMemoryStream(),
-            searchKwargs: 2,
-        });
-        await retriever.addDocuments([
-            { pageContent: "qux", metadata: {} },
-            { pageContent: "quux", metadata: {} },
-            { pageContent: "corge", metadata: {} },
-        ]);
-        const query = "Test query";
-        const resultsDocs = await retriever.getRelevantDocuments(query);
-        const expected = [
-            {
-                pageContent: "qux",
-                metadata: {
-                    [BUFFER_IDX]: 3,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "quux",
-                metadata: {
-                    [BUFFER_IDX]: 4,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "corge",
-                metadata: {
-                    [BUFFER_IDX]: 5,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(mockNow),
-                },
-            },
-            {
-                pageContent: "baz",
-                metadata: {
-                    [BUFFER_IDX]: 2,
-                    [LAST_ACCESSED_AT_KEY]: getSec(mockNow),
-                    created_at: getSec(new Date("2023-04-18 11:00")),
-                },
-            },
-        ];
-        console.log(resultsDocs);
-        expect(resultsDocs).toStrictEqual(expected);
-    });
-});

package/dist/retrievers/tests/vectorstores.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/retrievers/tests/vectorstores.test.js

@@ -1,50 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { FakeEmbeddings } from "@langchain/core/utils/testing";
-import { MemoryVectorStore } from "../../vectorstores/memory.js";
-test("Test Memory Retriever with Callback", async () => {
-    const pageContent = "Hello world";
-    const embeddings = new FakeEmbeddings();
-    const vectorStore = new MemoryVectorStore(embeddings);
-    expect(vectorStore).toBeDefined();
-    await vectorStore.addDocuments([
-        { pageContent, metadata: { a: 1 } },
-        { pageContent, metadata: { a: 1 } },
-        { pageContent, metadata: { a: 1 } },
-        { pageContent, metadata: { a: 1 } },
-    ]);
-    const queryStr = "testing testing";
-    let startRun = 0;
-    let endRun = 0;
-    let startPromiseResolve;
-    const startPromise = new Promise((resolve) => {
-        startPromiseResolve = resolve;
-    });
-    let endPromiseResolve;
-    const endPromise = new Promise((resolve) => {
-        endPromiseResolve = resolve;
-    });
-    const retriever = vectorStore.asRetriever({
-        k: 1,
-        vectorStore,
-        callbacks: [
-            {
-                handleRetrieverStart: async (_, query) => {
-                    expect(query).toBe(queryStr);
-                    startRun += 1;
-                    startPromiseResolve();
-                },
-                handleRetrieverEnd: async (documents) => {
-                    expect(documents[0].pageContent).toBe(pageContent);
-                    endRun += 1;
-                    endPromiseResolve();
-                },
-            },
-        ],
-    });
-    const results = await retriever.getRelevantDocuments(queryStr);
-    expect(results).toEqual([new Document({ metadata: { a: 1 }, pageContent })]);
-    await startPromise;
-    await endPromise;
-    expect(startRun).toBe(1);
-    expect(endRun).toBe(1);
-});

package/dist/smith/tests/run_on_dataset.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/smith/tests/run_on_dataset.int.test.js

@@ -1,257 +0,0 @@
-/* eslint-disable @typescript-eslint/no-floating-promises */
-/* eslint-disable @typescript-eslint/no-explicit-any */
-/* eslint-disable arrow-body-style */
-import OpenAI from "openai";
-import wiki from "wikipedia";
-import { Client, RunTree } from "langsmith";
-import { runOnDataset } from "../runner_utils.js";
-const oaiClient = new OpenAI();
-test(`Chat model dataset`, async () => {
-    async function generateWikiSearch(question, parentRun) {
-        const messages = [
-            {
-                role: "system",
-                content: "Generate a search query to pass into wikipedia to answer the user's question. Return only the search query and nothing more. This will be passed in directly to the Wikipedia search engine.",
-            },
-            { role: "user", content: question },
-        ];
-        const childRun = await parentRun.createChild({
-            name: "Generate Wiki Search",
-            run_type: "llm",
-            inputs: {
-                messages,
-            },
-        });
-        try {
-            const chatCompletion = await oaiClient.chat.completions.create({
-                model: "gpt-3.5-turbo",
-                // @ts-expect-error Typing is wrong
-                messages,
-                temperature: 0,
-            });
-            childRun.end(chatCompletion);
-            await childRun.postRun();
-            return chatCompletion.choices[0].message.content ?? "";
-        }
-        catch (error) {
-            console.error("Error generating wiki search query:", error);
-            childRun.end({ error: error.toString() });
-            await childRun.postRun();
-            throw error;
-        }
-    }
-    function convertDocs(results) {
-        return results.map((r) => ({
-            page_content: r.summary,
-            type: "Document",
-            metadata: { url: r.url },
-        }));
-    }
-    async function retrieve(query, parentRun) {
-        const childRun = await parentRun.createChild({
-            name: "Wikipedia Retriever",
-            run_type: "retriever",
-            inputs: { query },
-        });
-        try {
-            const { results } = await wiki.default.search(query, { limit: 10 });
-            const finalResults = [];
-            for (const result of results) {
-                if (finalResults.length >= 2) {
-                    // Just return the top 2 pages for now
-                    break;
-                }
-                const page = await wiki.default.page(result.title, {
-                    autoSuggest: false,
-                });
-                const summary = await page.summary();
-                finalResults.push({
-                    summary: summary.extract,
-                    url: page.fullurl,
-                });
-            }
-            childRun.end({
-                documents: convertDocs(finalResults),
-            });
-            await childRun.postRun();
-            return finalResults;
-        }
-        catch (error) {
-            console.error("Error in retrieval:", error);
-            childRun.end({ error: error.toString() });
-            await childRun.postRun();
-            throw error;
-        }
-    }
-    async function generateAnswer(question, context, parentRun) {
-        const messages = [
-            {
-                role: "system",
-                content: `Answer the user's question based only on the content below:\n\n${context}`,
-            },
-            { role: "user", content: question },
-        ];
-        const childRun = await parentRun.createChild({
-            name: "Generate Answer",
-            run_type: "llm",
-            inputs: {
-                messages,
-            },
-        });
-        try {
-            const chatCompletion = await oaiClient.chat.completions.create({
-                model: "gpt-3.5-turbo",
-                // @ts-expect-error Typing is wrong
-                messages,
-                temperature: 0,
-            });
-            childRun.end(chatCompletion);
-            await childRun.postRun();
-            return chatCompletion.choices[0].message.content ?? "";
-        }
-        catch (error) {
-            console.error("Error generating answer:", error);
-            childRun.end({ error: error.toString() });
-            await childRun.postRun();
-            throw error;
-        }
-    }
-    async function ragPipeline(question) {
-        const parentRunConfig = {
-            name: "Wikipedia RAG Pipeline",
-            run_type: "chain",
-            inputs: { question },
-        };
-        const parentRun = new RunTree(parentRunConfig);
-        try {
-            const query = await generateWikiSearch(question, parentRun);
-            const retrieverResults = await retrieve(query, parentRun);
-            const context = retrieverResults
-                .map((result) => result.summary)
-                .join("\n\n");
-            const answer = await generateAnswer(question, context, parentRun);
-            await parentRun.end({
-                outputs: answer,
-            });
-            await parentRun.postRun();
-            return answer;
-        }
-        catch (error) {
-            console.error("Error running RAG Pipeline:", error);
-            parentRun.end({ error: error.toString() });
-            await parentRun.postRun();
-            throw error;
-        }
-    }
-    const examples = [
-        [
-            "When was the Apple Vision Pro released in the US?",
-            "The Apple Vision Pro was released in the United States on February 2, 2024.",
-        ],
-        [
-            "What is LangChain?",
-            "LangChain is an open-source framework for building applications using large language models.",
-        ],
-        [
-            "Who is the chairman of OpenAI?",
-            "Bret Taylor is the chairman of the OpenAI",
-        ],
-    ];
-    const lsClient = new Client();
-    const datasetName = "JS run on dataset integration test";
-    let dataset;
-    try {
-        dataset = await lsClient.readDataset({ datasetName });
-    }
-    catch (e) {
-        dataset = await lsClient.createDataset(datasetName);
-        await Promise.all(examples.map(async ([question, answer]) => {
-            await lsClient.createExample({ question }, { answer }, { datasetId: dataset.id });
-        }));
-    }
-    // An illustrative custom evaluator example
-    const unsure = async ({ prediction }) => {
-        if (typeof prediction?.output !== "string") {
-            throw new Error("Invalid prediction format for this evaluator. Please check your chain's outputs and try again.");
-        }
-        return {
-            key: "unsure",
-            score: prediction.output.includes("not sure"),
-        };
-    };
-    const evaluation = {
-        // The 'evaluators' are loaded from LangChain's evaluation
-        // library.
-        evaluators: [
-            {
-                evaluatorType: "labeled_criteria",
-                criteria: "correctness",
-                feedbackKey: "correctness",
-                formatEvaluatorInputs: ({ rawInput, rawPrediction, rawReferenceOutput, }) => {
-                    return {
-                        input: rawInput.question,
-                        prediction: rawPrediction.output,
-                        reference: rawReferenceOutput.answer,
-                    };
-                },
-            },
-        ],
-        // Custom evaluators can be user-defined RunEvaluator's
-        // or a compatible function
-        customEvaluators: [unsure],
-    };
-    const wrappedRagPipeline = async ({ question, }) => {
-        return ragPipeline(question);
-    };
-    console.log(await runOnDataset(wrappedRagPipeline, datasetName, {
-        evaluationConfig: evaluation,
-    }));
-});
-test("Thrown errors should not interrupt dataset run", async () => {
-    async function ragPipeline(_) {
-        throw new Error("I don't know, I am learning from aliens.");
-    }
-    const examples = [
-        [
-            "When was the Apple Vision Pro released in the US?",
-            "The Apple Vision Pro was released in the United States on February 2, 2024.",
-        ],
-        [
-            "What is LangChain?",
-            "LangChain is an open-source framework for building applications using large language models.",
-        ],
-        [
-            "Who is the chairman of OpenAI?",
-            "Bret Taylor is the chairman of the OpenAI",
-        ],
-    ];
-    const lsClient = new Client();
-    const datasetName = "JS run on dataset integration test";
-    let dataset;
-    try {
-        dataset = await lsClient.readDataset({ datasetName });
-    }
-    catch (e) {
-        dataset = await lsClient.createDataset(datasetName);
-        await Promise.all(examples.map(async ([question, answer]) => {
-            await lsClient.createExample({ question }, { answer }, { datasetId: dataset.id });
-        }));
-    }
-    // An illustrative custom evaluator example
-    const dummy = async (_) => {
-        console.log("RUNNING EVAL");
-        throw new Error("Expected error");
-    };
-    const evaluation = {
-        // Custom evaluators can be user-defined RunEvaluator's
-        // or a compatible function
-        customEvaluators: [dummy],
-    };
-    const wrappedRagPipeline = async ({ question, }) => {
-        return ragPipeline(question);
-    };
-    console.log(await runOnDataset(wrappedRagPipeline, datasetName, {
-        evaluationConfig: evaluation,
-        maxConcurrency: 1,
-    }));
-});

package/dist/smith/tests/runner_utils.int.test.d.ts

@@ -1,9 +0,0 @@
-import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
-import { BaseChatModel } from "@langchain/core/language_models/chat_models";
-import { BaseMessage } from "@langchain/core/messages";
-import { ChatResult } from "@langchain/core/outputs";
-export declare class FakeChatModel extends BaseChatModel {
-    _combineLLMOutput(): never[];
-    _llmType(): string;
-    _generate(messages: BaseMessage[], _?: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<ChatResult>;
-}