langchain 0.0.76 → 0.0.78

package/dist/util/event-source-parse.js

@@ -13,9 +13,18 @@ export const EventStreamContentType = "text/event-stream";
  */
 export async function getBytes(stream, onChunk) {
     const reader = stream.getReader();
-    let result;
-    // eslint-disable-next-line no-cond-assign
-    while (!(result = await reader.read()).done) {
+    // CHANGED: Introduced a "flush" mechanism to process potential pending messages when the stream ends.
+    //          This change is essential to ensure that we capture every last piece of information from streams,
+    //          such as those from Azure OpenAI, which may not terminate with a blank line. Without this
+    //          mechanism, we risk ignoring a possibly significant last message.
+    //          See https://github.com/hwchase17/langchainjs/issues/1299 for details.
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = await reader.read();
+        if (result.done) {
+            onChunk(new Uint8Array(), true);
+            break;
+        }
         onChunk(result.value);
     }
 }
@@ -31,7 +40,11 @@ export function getLines(onLine) {
     let fieldLength; // length of the `field` portion of the line
     let discardTrailingNewline = false;
     // return a function that can process each incoming byte chunk:
-    return function onChunk(arr) {
+    return function onChunk(arr, flush) {
+        if (flush) {
+            onLine(arr, 0, true);
+            return;
+        }
         if (buffer === undefined) {
             buffer = arr;
             position = 0;
@@ -102,7 +115,14 @@ export function getMessages(onMessage, onId, onRetry) {
     let message = newMessage();
     const decoder = new TextDecoder();
     // return a function that can process each incoming line buffer:
-    return function onLine(line, fieldLength) {
+    return function onLine(line, fieldLength, flush) {
+        if (flush) {
+            if (!isEmpty(message)) {
+                onMessage?.(message);
+                message = newMessage();
+            }
+            return;
+        }
         if (line.length === 0) {
             // empty line denotes end of message. Trigger the callback and start a new message:
             onMessage?.(message);
@@ -157,3 +177,9 @@ function newMessage() {
         retry: undefined,
     };
 }
+function isEmpty(message) {
+    return (message.data === "" &&
+        message.event === "" &&
+        message.id === "" &&
+        message.retry === undefined);
+}

package/dist/vectorstores/faiss.cjs

@@ -0,0 +1,245 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FaissStore = void 0;
+const uuid = __importStar(require("uuid"));
+const base_js_1 = require("./base.cjs");
+const document_js_1 = require("../document.cjs");
+const index_js_1 = require("../docstore/index.cjs");
+class FaissStore extends base_js_1.SaveableVectorStore {
+    constructor(embeddings, args) {
+        super(embeddings, args);
+        Object.defineProperty(this, "_index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_mapping", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "docstore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "args", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.args = args;
+        this._index = args.index;
+        this._mapping = args.mapping ?? {};
+        this.embeddings = embeddings;
+        this.docstore = args?.docstore ?? new index_js_1.InMemoryDocstore();
+    }
+    async addDocuments(documents) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+    }
+    get index() {
+        if (!this._index) {
+            throw new Error("Vector store not initialised yet. Try calling `fromTexts` or `fromDocuments` first.");
+        }
+        return this._index;
+    }
+    set index(index) {
+        this._index = index;
+    }
+    async addVectors(vectors, documents) {
+        if (vectors.length === 0) {
+            return;
+        }
+        if (vectors.length !== documents.length) {
+            throw new Error(`Vectors and documents must have the same length`);
+        }
+        const dv = vectors[0].length;
+        if (!this._index) {
+            const { IndexFlatL2 } = await FaissStore.importFaiss();
+            this._index = new IndexFlatL2(dv);
+        }
+        const d = this.index.getDimension();
+        if (dv !== d) {
+            throw new Error(`Vectors must have the same length as the number of dimensions (${d})`);
+        }
+        const docstoreSize = this.docstore.count;
+        for (let i = 0; i < vectors.length; i += 1) {
+            const documentId = uuid.v4();
+            const id = docstoreSize + i;
+            this.index.add(vectors[i]);
+            this._mapping[id] = documentId;
+            this.docstore.add({ [documentId]: documents[i] });
+        }
+    }
+    async similaritySearchVectorWithScore(query, k) {
+        const d = this.index.getDimension();
+        if (query.length !== d) {
+            throw new Error(`Query vector must have the same length as the number of dimensions (${d})`);
+        }
+        if (k > this.index.ntotal()) {
+            const total = this.index.ntotal();
+            console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`);
+            // eslint-disable-next-line no-param-reassign
+            k = total;
+        }
+        const result = this.index.search(query, k);
+        return result.labels.map((id, index) => {
+            const uuid = this._mapping[id];
+            return [this.docstore.search(uuid), result.distances[index]];
+        });
+    }
+    async save(directory) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        await fs.mkdir(directory, { recursive: true });
+        await Promise.all([
+            this.index.write(path.join(directory, "faiss.index")),
+            await fs.writeFile(path.join(directory, "docstore.json"), JSON.stringify([
+                Array.from(this.docstore._docs.entries()),
+                this._mapping,
+            ])),
+        ]);
+    }
+    static async load(directory, embeddings) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        const readStore = (directory) => fs
+            .readFile(path.join(directory, "docstore.json"), "utf8")
+            .then(JSON.parse);
+        const readIndex = async (directory) => {
+            const { IndexFlatL2 } = await this.importFaiss();
+            return IndexFlatL2.read(path.join(directory, "faiss.index"));
+        };
+        const [[docstoreFiles, mapping], index] = await Promise.all([
+            readStore(directory),
+            readIndex(directory),
+        ]);
+        const docstore = new index_js_1.InMemoryDocstore(new Map(docstoreFiles));
+        return new this(embeddings, { docstore, index, mapping });
+    }
+    static async loadFromPython(directory, embeddings) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        const { Parser, NameRegistry } = await this.importPickleparser();
+        class PyDocument extends Map {
+            toDocument() {
+                return new document_js_1.Document({
+                    pageContent: this.get("page_content"),
+                    metadata: this.get("metadata"),
+                });
+            }
+        }
+        class PyInMemoryDocstore {
+            constructor() {
+                Object.defineProperty(this, "_dict", {
+                    enumerable: true,
+                    configurable: true,
+                    writable: true,
+                    value: void 0
+                });
+            }
+            toInMemoryDocstore() {
+                const s = new index_js_1.InMemoryDocstore();
+                for (const [key, value] of Object.entries(this._dict)) {
+                    s._docs.set(key, value.toDocument());
+                }
+                return s;
+            }
+        }
+        const readStore = async (directory) => {
+            const pkl = await fs.readFile(path.join(directory, "index.pkl"), "binary");
+            const buffer = Buffer.from(pkl, "binary");
+            const registry = new NameRegistry()
+                .register("langchain.docstore.in_memory", "InMemoryDocstore", PyInMemoryDocstore)
+                .register("langchain.schema", "Document", PyDocument)
+                .register("pathlib", "WindowsPath", (...args) => args.join("\\"))
+                .register("pathlib", "PosixPath", (...args) => args.join("/"));
+            const pickleparser = new Parser({
+                nameResolver: registry,
+            });
+            const [rawStore, mapping] = pickleparser.parse(buffer);
+            const store = rawStore.toInMemoryDocstore();
+            return { store, mapping };
+        };
+        const readIndex = async (directory) => {
+            const { IndexFlatL2 } = await this.importFaiss();
+            return IndexFlatL2.read(path.join(directory, "index.faiss"));
+        };
+        const [store, index] = await Promise.all([
+            readStore(directory),
+            readIndex(directory),
+        ]);
+        return new this(embeddings, {
+            docstore: store.store,
+            index,
+            mapping: store.mapping,
+        });
+    }
+    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
+        const docs = [];
+        for (let i = 0; i < texts.length; i += 1) {
+            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
+            const newDoc = new document_js_1.Document({
+                pageContent: texts[i],
+                metadata,
+            });
+            docs.push(newDoc);
+        }
+        return this.fromDocuments(docs, embeddings, dbConfig);
+    }
+    static async fromDocuments(docs, embeddings, dbConfig) {
+        const args = {
+            docstore: dbConfig?.docstore,
+        };
+        const instance = new this(embeddings, args);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+    static async importFaiss() {
+        try {
+            const { default: { IndexFlatL2 }, } = await import("faiss-node");
+            return { IndexFlatL2 };
+        }
+        catch (err) {
+            throw new Error("Please install faiss-node as a dependency with, e.g. `npm install -S faiss-node`");
+        }
+    }
+    static async importPickleparser() {
+        try {
+            const { default: { Parser, NameRegistry }, } = await import("pickleparser");
+            return { Parser, NameRegistry };
+        }
+        catch (err) {
+            throw new Error("Please install pickleparser as a dependency with, e.g. `npm install -S pickleparser`");
+        }
+    }
+}
+exports.FaissStore = FaissStore;

package/dist/vectorstores/faiss.d.ts

@@ -0,0 +1,39 @@
+import type { IndexFlatL2 } from "faiss-node";
+import type { NameRegistry, Parser } from "pickleparser";
+import { Embeddings } from "../embeddings/base.js";
+import { SaveableVectorStore } from "./base.js";
+import { Document } from "../document.js";
+import { InMemoryDocstore } from "../docstore/index.js";
+export interface FaissLibArgs {
+    docstore?: InMemoryDocstore;
+    index?: IndexFlatL2;
+    mapping?: Record<number, string>;
+}
+export declare class FaissStore extends SaveableVectorStore {
+    _index?: IndexFlatL2;
+    _mapping: Record<number, string>;
+    docstore: InMemoryDocstore;
+    args: FaissLibArgs;
+    constructor(embeddings: Embeddings, args: FaissLibArgs);
+    addDocuments(documents: Document[]): Promise<void>;
+    get index(): IndexFlatL2;
+    private set index(value);
+    addVectors(vectors: number[][], documents: Document[]): Promise<void>;
+    similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document<Record<string, any>>, number][]>;
+    save(directory: string): Promise<void>;
+    static load(directory: string, embeddings: Embeddings): Promise<FaissStore>;
+    static loadFromPython(directory: string, embeddings: Embeddings): Promise<FaissStore>;
+    static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, dbConfig?: {
+        docstore?: InMemoryDocstore;
+    }): Promise<FaissStore>;
+    static fromDocuments(docs: Document[], embeddings: Embeddings, dbConfig?: {
+        docstore?: InMemoryDocstore;
+    }): Promise<FaissStore>;
+    static importFaiss(): Promise<{
+        IndexFlatL2: typeof IndexFlatL2;
+    }>;
+    static importPickleparser(): Promise<{
+        Parser: typeof Parser;
+        NameRegistry: typeof NameRegistry;
+    }>;
+}

package/dist/vectorstores/faiss.js

@@ -0,0 +1,218 @@
+import * as uuid from "uuid";
+import { SaveableVectorStore } from "./base.js";
+import { Document } from "../document.js";
+import { InMemoryDocstore } from "../docstore/index.js";
+export class FaissStore extends SaveableVectorStore {
+    constructor(embeddings, args) {
+        super(embeddings, args);
+        Object.defineProperty(this, "_index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_mapping", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "docstore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "args", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.args = args;
+        this._index = args.index;
+        this._mapping = args.mapping ?? {};
+        this.embeddings = embeddings;
+        this.docstore = args?.docstore ?? new InMemoryDocstore();
+    }
+    async addDocuments(documents) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+    }
+    get index() {
+        if (!this._index) {
+            throw new Error("Vector store not initialised yet. Try calling `fromTexts` or `fromDocuments` first.");
+        }
+        return this._index;
+    }
+    set index(index) {
+        this._index = index;
+    }
+    async addVectors(vectors, documents) {
+        if (vectors.length === 0) {
+            return;
+        }
+        if (vectors.length !== documents.length) {
+            throw new Error(`Vectors and documents must have the same length`);
+        }
+        const dv = vectors[0].length;
+        if (!this._index) {
+            const { IndexFlatL2 } = await FaissStore.importFaiss();
+            this._index = new IndexFlatL2(dv);
+        }
+        const d = this.index.getDimension();
+        if (dv !== d) {
+            throw new Error(`Vectors must have the same length as the number of dimensions (${d})`);
+        }
+        const docstoreSize = this.docstore.count;
+        for (let i = 0; i < vectors.length; i += 1) {
+            const documentId = uuid.v4();
+            const id = docstoreSize + i;
+            this.index.add(vectors[i]);
+            this._mapping[id] = documentId;
+            this.docstore.add({ [documentId]: documents[i] });
+        }
+    }
+    async similaritySearchVectorWithScore(query, k) {
+        const d = this.index.getDimension();
+        if (query.length !== d) {
+            throw new Error(`Query vector must have the same length as the number of dimensions (${d})`);
+        }
+        if (k > this.index.ntotal()) {
+            const total = this.index.ntotal();
+            console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`);
+            // eslint-disable-next-line no-param-reassign
+            k = total;
+        }
+        const result = this.index.search(query, k);
+        return result.labels.map((id, index) => {
+            const uuid = this._mapping[id];
+            return [this.docstore.search(uuid), result.distances[index]];
+        });
+    }
+    async save(directory) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        await fs.mkdir(directory, { recursive: true });
+        await Promise.all([
+            this.index.write(path.join(directory, "faiss.index")),
+            await fs.writeFile(path.join(directory, "docstore.json"), JSON.stringify([
+                Array.from(this.docstore._docs.entries()),
+                this._mapping,
+            ])),
+        ]);
+    }
+    static async load(directory, embeddings) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        const readStore = (directory) => fs
+            .readFile(path.join(directory, "docstore.json"), "utf8")
+            .then(JSON.parse);
+        const readIndex = async (directory) => {
+            const { IndexFlatL2 } = await this.importFaiss();
+            return IndexFlatL2.read(path.join(directory, "faiss.index"));
+        };
+        const [[docstoreFiles, mapping], index] = await Promise.all([
+            readStore(directory),
+            readIndex(directory),
+        ]);
+        const docstore = new InMemoryDocstore(new Map(docstoreFiles));
+        return new this(embeddings, { docstore, index, mapping });
+    }
+    static async loadFromPython(directory, embeddings) {
+        const fs = await import("node:fs/promises");
+        const path = await import("node:path");
+        const { Parser, NameRegistry } = await this.importPickleparser();
+        class PyDocument extends Map {
+            toDocument() {
+                return new Document({
+                    pageContent: this.get("page_content"),
+                    metadata: this.get("metadata"),
+                });
+            }
+        }
+        class PyInMemoryDocstore {
+            constructor() {
+                Object.defineProperty(this, "_dict", {
+                    enumerable: true,
+                    configurable: true,
+                    writable: true,
+                    value: void 0
+                });
+            }
+            toInMemoryDocstore() {
+                const s = new InMemoryDocstore();
+                for (const [key, value] of Object.entries(this._dict)) {
+                    s._docs.set(key, value.toDocument());
+                }
+                return s;
+            }
+        }
+        const readStore = async (directory) => {
+            const pkl = await fs.readFile(path.join(directory, "index.pkl"), "binary");
+            const buffer = Buffer.from(pkl, "binary");
+            const registry = new NameRegistry()
+                .register("langchain.docstore.in_memory", "InMemoryDocstore", PyInMemoryDocstore)
+                .register("langchain.schema", "Document", PyDocument)
+                .register("pathlib", "WindowsPath", (...args) => args.join("\\"))
+                .register("pathlib", "PosixPath", (...args) => args.join("/"));
+            const pickleparser = new Parser({
+                nameResolver: registry,
+            });
+            const [rawStore, mapping] = pickleparser.parse(buffer);
+            const store = rawStore.toInMemoryDocstore();
+            return { store, mapping };
+        };
+        const readIndex = async (directory) => {
+            const { IndexFlatL2 } = await this.importFaiss();
+            return IndexFlatL2.read(path.join(directory, "index.faiss"));
+        };
+        const [store, index] = await Promise.all([
+            readStore(directory),
+            readIndex(directory),
+        ]);
+        return new this(embeddings, {
+            docstore: store.store,
+            index,
+            mapping: store.mapping,
+        });
+    }
+    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
+        const docs = [];
+        for (let i = 0; i < texts.length; i += 1) {
+            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
+            const newDoc = new Document({
+                pageContent: texts[i],
+                metadata,
+            });
+            docs.push(newDoc);
+        }
+        return this.fromDocuments(docs, embeddings, dbConfig);
+    }
+    static async fromDocuments(docs, embeddings, dbConfig) {
+        const args = {
+            docstore: dbConfig?.docstore,
+        };
+        const instance = new this(embeddings, args);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+    static async importFaiss() {
+        try {
+            const { default: { IndexFlatL2 }, } = await import("faiss-node");
+            return { IndexFlatL2 };
+        }
+        catch (err) {
+            throw new Error("Please install faiss-node as a dependency with, e.g. `npm install -S faiss-node`");
+        }
+    }
+    static async importPickleparser() {
+        try {
+            const { default: { Parser, NameRegistry }, } = await import("pickleparser");
+            return { Parser, NameRegistry };
+        }
+        catch (err) {
+            throw new Error("Please install pickleparser as a dependency with, e.g. `npm install -S pickleparser`");
+        }
+    }
+}

package/llms/sagemaker_endpoint.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/llms/sagemaker_endpoint.cjs');

package/llms/sagemaker_endpoint.d.ts

@@ -0,0 +1 @@
+export * from '../dist/llms/sagemaker_endpoint.js'

package/llms/sagemaker_endpoint.js

@@ -0,0 +1 @@
+export * from '../dist/llms/sagemaker_endpoint.js'

package/output_parsers/expression.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/output_parsers/expression.cjs');

package/output_parsers/expression.d.ts

@@ -0,0 +1 @@
+export * from '../dist/output_parsers/expression.js'

package/output_parsers/expression.js

@@ -0,0 +1 @@
+export * from '../dist/output_parsers/expression.js'