langchain 0.0.76 → 0.0.77

package/dist/output_parsers/structured.js

@@ -59,30 +59,87 @@ export class JsonMarkdownStructuredOutputParser extends StructuredOutputParser {
     }
     _schemaToInstruction(schemaInput, indent = 2) {
         const schema = schemaInput;
-        let nullable = false;
-        if (Array.isArray(schema.type)) {
-            const [actualType, nullStr] = schema.type;
-            nullable = nullStr === "null";
-            schema.type = actualType;
-        }
-        if (schema.type === "object" && schema.properties) {
+        if ("type" in schema) {
+            let nullable = false;
+            let type;
+            if (Array.isArray(schema.type)) {
+                const nullIdx = schema.type.findIndex((type) => type === "null");
+                if (nullIdx !== -1) {
+                    nullable = true;
+                    schema.type.splice(nullIdx, 1);
+                }
+                type = schema.type.join(" | ");
+            }
+            else {
+                type = schema.type;
+            }
+            if (schema.type === "object" && schema.properties) {
+                const description = schema.description
+                    ? ` // ${schema.description}`
+                    : "";
+                const properties = Object.entries(schema.properties)
+                    .map(([key, value]) => {
+                    const isOptional = schema.required?.includes(key)
+                        ? ""
+                        : " (optional)";
+                    return `${" ".repeat(indent)}"${key}": ${this._schemaToInstruction(value, indent + 2)}${isOptional}`;
+                })
+                    .join("\n");
+                return `{\n${properties}\n${" ".repeat(indent - 2)}}${description}`;
+            }
+            if (schema.type === "array" && schema.items) {
+                const description = schema.description
+                    ? ` // ${schema.description}`
+                    : "";
+                return `array[\n${" ".repeat(indent)}${this._schemaToInstruction(schema.items, indent + 2)}\n${" ".repeat(indent - 2)}] ${description}`;
+            }
+            const isNullable = nullable ? " (nullable)" : "";
             const description = schema.description ? ` // ${schema.description}` : "";
-            const properties = Object.entries(schema.properties)
-                .map(([key, value]) => {
-                const isOptional = schema.required?.includes(key)
-                    ? ""
-                    : " (optional)";
-                return `${" ".repeat(indent)}"${key}": ${this._schemaToInstruction(value, indent + 2)}${isOptional}`;
-            })
-                .join("\n");
-            return `{\n${properties}\n${" ".repeat(indent - 2)}}${description}`;
+            return `${type}${description}${isNullable}`;
         }
-        if (schema.type === "array" && schema.items) {
-            const description = schema.description ? ` // ${schema.description}` : "";
-            return `array[\n${" ".repeat(indent)}${this._schemaToInstruction(schema.items)}\n${" ".repeat(indent - 2)}] ${description}`;
+        if ("anyOf" in schema) {
+            return schema.anyOf
+                .map((s) => this._schemaToInstruction(s, indent))
+                .join(`\n${" ".repeat(indent - 2)}`);
         }
-        const isNullable = nullable ? " (nullable)" : "";
-        const description = schema.description ? ` // ${schema.description}` : "";
-        return `${schema.type}${description}${isNullable}`;
+        throw new Error("unsupported schema type");
+    }
+    static fromZodSchema(schema) {
+        return new this(schema);
+    }
+    static fromNamesAndDescriptions(schemas) {
+        const zodSchema = z.object(Object.fromEntries(Object.entries(schemas).map(([name, description]) => [name, z.string().describe(description)])));
+        return new this(zodSchema);
+    }
+}
+export class AsymmetricStructuredOutputParser extends BaseOutputParser {
+    constructor(inputSchema) {
+        super();
+        Object.defineProperty(this, "inputSchema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: inputSchema
+        });
+        Object.defineProperty(this, "structuredInputParser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.structuredInputParser = new JsonMarkdownStructuredOutputParser(inputSchema);
+    }
+    async parse(text) {
+        let parsedInput;
+        try {
+            parsedInput = await this.structuredInputParser.parse(text);
+        }
+        catch (e) {
+            throw new OutputParserException(`Failed to parse. Text: "${text}". Error: ${e}`, text);
+        }
+        return this.outputProcessor(parsedInput);
+    }
+    getFormatInstructions() {
+        return this.structuredInputParser.getFormatInstructions();
     }
 }

package/dist/retrievers/self_query/index.cjs

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SelfQueryRetriever = exports.BasicTranslator = exports.BaseTranslator = void 0;
+const index_js_1 = require("../../chains/query_constructor/index.cjs");
+const index_js_2 = require("../../schema/index.cjs");
+const translator_js_1 = require("./translator.cjs");
+Object.defineProperty(exports, "BaseTranslator", { enumerable: true, get: function () { return translator_js_1.BaseTranslator; } });
+Object.defineProperty(exports, "BasicTranslator", { enumerable: true, get: function () { return translator_js_1.BasicTranslator; } });
+class SelfQueryRetriever extends index_js_2.BaseRetriever {
+    constructor(options) {
+        super();
+        Object.defineProperty(this, "vectorStore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "llmChain", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "verbose", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "structuredQueryTranslator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "searchParams", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: { k: 4 }
+        });
+        this.vectorStore = options.vectorStore;
+        this.llmChain = options.llmChain;
+        this.verbose = options.verbose ?? false;
+        this.searchParams = options.searchParams ?? this.searchParams;
+        this.structuredQueryTranslator = options.structuredQueryTranslator;
+    }
+    async getRelevantDocuments(query) {
+        const { [this.llmChain.outputKey]: output } = await this.llmChain.call({
+            [this.llmChain.inputKeys[0]]: query,
+        });
+        const nextArg = this.structuredQueryTranslator.visitStructuredQuery(output);
+        if (nextArg.filter) {
+            return this.vectorStore.similaritySearch(query, this.searchParams?.k, nextArg.filter);
+        }
+        else {
+            return this.vectorStore.similaritySearch(query, this.searchParams?.k, this.searchParams?.filter);
+        }
+    }
+    static fromLLM(opts) {
+        const { structuredQueryTranslator } = opts;
+        const allowedComparators = opts.allowedComparators ?? structuredQueryTranslator.allowedComparators;
+        const allowedOperators = opts.allowedOperators ?? structuredQueryTranslator.allowedOperators;
+        const llmChain = (0, index_js_1.loadQueryContstructorChain)({
+            llm: opts.llm,
+            documentContents: opts.documentContents,
+            attributeInfo: opts.attributeInfo,
+            examples: opts.examples,
+            allowedComparators,
+            allowedOperators,
+        });
+        return new SelfQueryRetriever({
+            vectorStore: opts.vectorStore,
+            llmChain,
+            structuredQueryTranslator,
+        });
+    }
+}
+exports.SelfQueryRetriever = SelfQueryRetriever;

package/dist/retrievers/self_query/index.d.ts

@@ -0,0 +1,33 @@
+import { LLMChain } from "../../chains/llm_chain.js";
+import { QueryConstructorChainOptions } from "../../chains/query_constructor/index.js";
+import { Document } from "../../document.js";
+import { BaseRetriever } from "../../schema/index.js";
+import { VectorStore } from "../../vectorstores/base.js";
+import { BaseTranslator, BasicTranslator } from "./translator.js";
+export { BaseTranslator, BasicTranslator };
+export type SelfQueryRetrieverArgs = {
+    vectorStore: VectorStore;
+    llmChain: LLMChain;
+    structuredQueryTranslator: BaseTranslator;
+    verbose?: boolean;
+    searchParams?: {
+        k?: number;
+        filter?: VectorStore["FilterType"];
+    };
+};
+export declare class SelfQueryRetriever extends BaseRetriever implements SelfQueryRetrieverArgs {
+    vectorStore: VectorStore;
+    llmChain: LLMChain;
+    verbose?: boolean;
+    structuredQueryTranslator: BaseTranslator;
+    searchParams?: {
+        k?: number;
+        filter?: VectorStore["FilterType"];
+    };
+    constructor(options: SelfQueryRetrieverArgs);
+    getRelevantDocuments(query: string): Promise<Document<Record<string, unknown>>[]>;
+    static fromLLM(opts: QueryConstructorChainOptions & {
+        vectorStore: VectorStore;
+        structuredQueryTranslator: BaseTranslator;
+    }): SelfQueryRetriever;
+}

package/dist/retrievers/self_query/index.js

@@ -0,0 +1,74 @@
+import { loadQueryContstructorChain, } from "../../chains/query_constructor/index.js";
+import { BaseRetriever } from "../../schema/index.js";
+import { BaseTranslator, BasicTranslator } from "./translator.js";
+export { BaseTranslator, BasicTranslator };
+export class SelfQueryRetriever extends BaseRetriever {
+    constructor(options) {
+        super();
+        Object.defineProperty(this, "vectorStore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "llmChain", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "verbose", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "structuredQueryTranslator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "searchParams", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: { k: 4 }
+        });
+        this.vectorStore = options.vectorStore;
+        this.llmChain = options.llmChain;
+        this.verbose = options.verbose ?? false;
+        this.searchParams = options.searchParams ?? this.searchParams;
+        this.structuredQueryTranslator = options.structuredQueryTranslator;
+    }
+    async getRelevantDocuments(query) {
+        const { [this.llmChain.outputKey]: output } = await this.llmChain.call({
+            [this.llmChain.inputKeys[0]]: query,
+        });
+        const nextArg = this.structuredQueryTranslator.visitStructuredQuery(output);
+        if (nextArg.filter) {
+            return this.vectorStore.similaritySearch(query, this.searchParams?.k, nextArg.filter);
+        }
+        else {
+            return this.vectorStore.similaritySearch(query, this.searchParams?.k, this.searchParams?.filter);
+        }
+    }
+    static fromLLM(opts) {
+        const { structuredQueryTranslator } = opts;
+        const allowedComparators = opts.allowedComparators ?? structuredQueryTranslator.allowedComparators;
+        const allowedOperators = opts.allowedOperators ?? structuredQueryTranslator.allowedOperators;
+        const llmChain = loadQueryContstructorChain({
+            llm: opts.llm,
+            documentContents: opts.documentContents,
+            attributeInfo: opts.attributeInfo,
+            examples: opts.examples,
+            allowedComparators,
+            allowedOperators,
+        });
+        return new SelfQueryRetriever({
+            vectorStore: opts.vectorStore,
+            llmChain,
+            structuredQueryTranslator,
+        });
+    }
+}

package/dist/retrievers/self_query/translator.cjs

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BasicTranslator = exports.BaseTranslator = void 0;
+const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
+class BaseTranslator extends ir_js_1.Visitor {
+}
+exports.BaseTranslator = BaseTranslator;
+class BasicTranslator extends BaseTranslator {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "allowedOperators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: [ir_js_1.Operators.and, ir_js_1.Operators.or]
+        });
+        Object.defineProperty(this, "allowedComparators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: [
+                ir_js_1.Comparators.eq,
+                ir_js_1.Comparators.neq,
+                ir_js_1.Comparators.gt,
+                ir_js_1.Comparators.gte,
+                ir_js_1.Comparators.lt,
+                ir_js_1.Comparators.lte,
+            ]
+        });
+    }
+    formatFunction(func) {
+        if (func in ir_js_1.Comparators) {
+            if (this.allowedComparators.length > 0 &&
+                this.allowedComparators.indexOf(func) === -1) {
+                throw new Error(`Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(", ")}`);
+            }
+        }
+        else if (func in ir_js_1.Operators) {
+            if (this.allowedOperators.length > 0 &&
+                this.allowedOperators.indexOf(func) === -1) {
+                throw new Error(`Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(", ")}`);
+            }
+        }
+        else {
+            throw new Error("Unknown comparator or operator");
+        }
+        return `$${func}`;
+    }
+    visitOperation(operation) {
+        const args = operation.args?.map((arg) => arg.accept(this));
+        return {
+            [this.formatFunction(operation.operator)]: args,
+        };
+    }
+    visitComparison(comparison) {
+        return {
+            [comparison.attribute]: {
+                [this.formatFunction(comparison.comparator)]: comparison.value,
+            },
+        };
+    }
+    visitStructuredQuery(query) {
+        let nextArg = {};
+        if (query.filter) {
+            nextArg = {
+                filter: query.filter.accept(this),
+            };
+        }
+        return nextArg;
+    }
+}
+exports.BasicTranslator = BasicTranslator;

package/dist/retrievers/self_query/translator.d.ts

@@ -0,0 +1,14 @@
+import { Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor, VisitorComparisonResult, VisitorOperationResult, VisitorStructuredQueryResult } from "../../chains/query_constructor/ir.js";
+export declare abstract class BaseTranslator extends Visitor {
+    abstract allowedOperators: Operator[];
+    abstract allowedComparators: Comparator[];
+    abstract formatFunction(func: Operator | Comparator): string;
+}
+export declare class BasicTranslator extends BaseTranslator {
+    allowedOperators: Operator[];
+    allowedComparators: Comparator[];
+    formatFunction(func: Operator | Comparator): string;
+    visitOperation(operation: Operation): VisitorOperationResult;
+    visitComparison(comparison: Comparison): VisitorComparisonResult;
+    visitStructuredQuery(query: StructuredQuery): VisitorStructuredQueryResult;
+}

package/dist/retrievers/self_query/translator.js

@@ -0,0 +1,67 @@
+import { Comparators, Operators, Visitor, } from "../../chains/query_constructor/ir.js";
+export class BaseTranslator extends Visitor {
+}
+export class BasicTranslator extends BaseTranslator {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "allowedOperators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: [Operators.and, Operators.or]
+        });
+        Object.defineProperty(this, "allowedComparators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: [
+                Comparators.eq,
+                Comparators.neq,
+                Comparators.gt,
+                Comparators.gte,
+                Comparators.lt,
+                Comparators.lte,
+            ]
+        });
+    }
+    formatFunction(func) {
+        if (func in Comparators) {
+            if (this.allowedComparators.length > 0 &&
+                this.allowedComparators.indexOf(func) === -1) {
+                throw new Error(`Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(", ")}`);
+            }
+        }
+        else if (func in Operators) {
+            if (this.allowedOperators.length > 0 &&
+                this.allowedOperators.indexOf(func) === -1) {
+                throw new Error(`Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(", ")}`);
+            }
+        }
+        else {
+            throw new Error("Unknown comparator or operator");
+        }
+        return `$${func}`;
+    }
+    visitOperation(operation) {
+        const args = operation.args?.map((arg) => arg.accept(this));
+        return {
+            [this.formatFunction(operation.operator)]: args,
+        };
+    }
+    visitComparison(comparison) {
+        return {
+            [comparison.attribute]: {
+                [this.formatFunction(comparison.comparator)]: comparison.value,
+            },
+        };
+    }
+    visitStructuredQuery(query) {
+        let nextArg = {};
+        if (query.filter) {
+            nextArg = {
+                filter: query.filter.accept(this),
+            };
+        }
+        return nextArg;
+    }
+}

package/dist/schema/query_constructor.cjs

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AttributeInfo = void 0;
+class AttributeInfo {
+    constructor(name, type, description) {
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: name
+        });
+        Object.defineProperty(this, "type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: type
+        });
+        Object.defineProperty(this, "description", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: description
+        });
+    }
+}
+exports.AttributeInfo = AttributeInfo;

package/dist/schema/query_constructor.d.ts

@@ -0,0 +1,6 @@
+export declare class AttributeInfo {
+    name: string;
+    type: string;
+    description: string;
+    constructor(name: string, type: string, description: string);
+}

package/dist/schema/query_constructor.js

@@ -0,0 +1,22 @@
+export class AttributeInfo {
+    constructor(name, type, description) {
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: name
+        });
+        Object.defineProperty(this, "type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: type
+        });
+        Object.defineProperty(this, "description", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: description
+        });
+    }
+}

package/dist/tools/json.cjs

@@ -27,7 +27,9 @@ class JsonSpec {
         const pointer = jsonpointer_1.default.compile(input);
         const res = pointer.get(this.obj);
         if (typeof res === "object" && !Array.isArray(res) && res !== null) {
-            return Object.keys(res).join(", ");
+            return Object.keys(res)
+                .map((i) => i.replaceAll("~", "~0").replaceAll("/", "~1"))
+                .join(", ");
         }
         throw new Error(`Value at ${input} is not a dictionary, get the value directly instead.`);
     }

package/dist/tools/json.js

@@ -21,7 +21,9 @@ export class JsonSpec {
         const pointer = jsonpointer.compile(input);
         const res = pointer.get(this.obj);
         if (typeof res === "object" && !Array.isArray(res) && res !== null) {
-            return Object.keys(res).join(", ");
+            return Object.keys(res)
+                .map((i) => i.replaceAll("~", "~0").replaceAll("/", "~1"))
+                .join(", ");
         }
         throw new Error(`Value at ${input} is not a dictionary, get the value directly instead.`);
     }

package/dist/util/event-source-parse.cjs

@@ -16,9 +16,18 @@ exports.EventStreamContentType = "text/event-stream";
  */
 async function getBytes(stream, onChunk) {
     const reader = stream.getReader();
-    let result;
-    // eslint-disable-next-line no-cond-assign
-    while (!(result = await reader.read()).done) {
+    // CHANGED: Introduced a "flush" mechanism to process potential pending messages when the stream ends.
+    //          This change is essential to ensure that we capture every last piece of information from streams,
+    //          such as those from Azure OpenAI, which may not terminate with a blank line. Without this
+    //          mechanism, we risk ignoring a possibly significant last message.
+    //          See https://github.com/hwchase17/langchainjs/issues/1299 for details.
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = await reader.read();
+        if (result.done) {
+            onChunk(new Uint8Array(), true);
+            break;
+        }
         onChunk(result.value);
     }
 }
@@ -35,7 +44,11 @@ function getLines(onLine) {
     let fieldLength; // length of the `field` portion of the line
     let discardTrailingNewline = false;
     // return a function that can process each incoming byte chunk:
-    return function onChunk(arr) {
+    return function onChunk(arr, flush) {
+        if (flush) {
+            onLine(arr, 0, true);
+            return;
+        }
         if (buffer === undefined) {
             buffer = arr;
             position = 0;
@@ -107,7 +120,14 @@ function getMessages(onMessage, onId, onRetry) {
     let message = newMessage();
     const decoder = new TextDecoder();
     // return a function that can process each incoming line buffer:
-    return function onLine(line, fieldLength) {
+    return function onLine(line, fieldLength, flush) {
+        if (flush) {
+            if (!isEmpty(message)) {
+                onMessage?.(message);
+                message = newMessage();
+            }
+            return;
+        }
         if (line.length === 0) {
             // empty line denotes end of message. Trigger the callback and start a new message:
             onMessage?.(message);
@@ -163,3 +183,9 @@ function newMessage() {
         retry: undefined,
     };
 }
+function isEmpty(message) {
+    return (message.data === "" &&
+        message.event === "" &&
+        message.id === "" &&
+        message.retry === undefined);
+}

package/dist/util/event-source-parse.d.ts

@@ -19,14 +19,14 @@ export interface EventSourceMessage {
  * @param onChunk A function that will be called on each new byte chunk in the stream.
  * @returns {Promise<void>} A promise that will be resolved when the stream closes.
  */
-export declare function getBytes(stream: ReadableStream<Uint8Array>, onChunk: (arr: Uint8Array) => void): Promise<void>;
+export declare function getBytes(stream: ReadableStream<Uint8Array>, onChunk: (arr: Uint8Array, flush?: boolean) => void): Promise<void>;
 /**
  * Parses arbitary byte chunks into EventSource line buffers.
  * Each line should be of the format "field: value" and ends with \r, \n, or \r\n.
  * @param onLine A function that will be called on each new EventSource line.
  * @returns A function that should be called for each incoming byte chunk.
  */
-export declare function getLines(onLine: (line: Uint8Array, fieldLength: number) => void): (arr: Uint8Array) => void;
+export declare function getLines(onLine: (line: Uint8Array, fieldLength: number, flush?: boolean) => void): (arr: Uint8Array, flush?: boolean) => void;
 /**
  * Parses line buffers into EventSourceMessages.
  * @param onId A function that will be called on each `id` field.
@@ -34,4 +34,4 @@ export declare function getLines(onLine: (line: Uint8Array, fieldLength: number)
  * @param onMessage A function that will be called on each message.
  * @returns A function that should be called for each incoming line buffer.
  */
-export declare function getMessages(onMessage?: (msg: EventSourceMessage) => void, onId?: (id: string) => void, onRetry?: (retry: number) => void): (line: Uint8Array, fieldLength: number) => void;
+export declare function getMessages(onMessage?: (msg: EventSourceMessage) => void, onId?: (id: string) => void, onRetry?: (retry: number) => void): (line: Uint8Array, fieldLength: number, flush?: boolean) => void;