langchain 0.0.154 → 0.0.156

package/dist/chat_models/bedrock.js

@@ -0,0 +1,254 @@
+import { SignatureV4 } from "@smithy/signature-v4";
+import { defaultProvider } from "@aws-sdk/credential-provider-node";
+import { HttpRequest } from "@smithy/protocol-http";
+import { EventStreamCodec } from "@smithy/eventstream-codec";
+import { fromUtf8, toUtf8 } from "@smithy/util-utf8";
+import { Sha256 } from "@aws-crypto/sha256-js";
+import { BedrockLLMInputOutputAdapter, } from "../util/bedrock.js";
+import { getEnvironmentVariable } from "../util/env.js";
+import { SimpleChatModel } from "./base.js";
+import { AIMessageChunk, AIMessage, ChatGenerationChunk, ChatMessage, } from "../schema/index.js";
+function convertOneMessageToText(message, humanPrompt, aiPrompt) {
+    if (message._getType() === "human") {
+        return `${humanPrompt} ${message.content}`;
+    }
+    else if (message._getType() === "ai") {
+        return `${aiPrompt} ${message.content}`;
+    }
+    else if (message._getType() === "system") {
+        return `${humanPrompt} <admin>${message.content}</admin>`;
+    }
+    else if (ChatMessage.isInstance(message)) {
+        return `\n\n${message.role[0].toUpperCase() + message.role.slice(1)}: {message.content}`;
+    }
+    throw new Error(`Unknown role: ${message._getType()}`);
+}
+export function convertMessagesToPromptAnthropic(messages, humanPrompt = "\n\nHuman:", aiPrompt = "\n\nAssistant:") {
+    const messagesCopy = [...messages];
+    if (messagesCopy.length === 0 ||
+        messagesCopy[messagesCopy.length - 1]._getType() !== "ai") {
+        messagesCopy.push(new AIMessage({ content: "" }));
+    }
+    return messagesCopy
+        .map((message) => convertOneMessageToText(message, humanPrompt, aiPrompt))
+        .join("");
+}
+/**
+ * Function that converts an array of messages into a single string prompt
+ * that can be used as input for a chat model. It delegates the conversion
+ * logic to the appropriate provider-specific function.
+ * @param messages Array of messages to be converted.
+ * @param options Options to be used during the conversion.
+ * @returns A string prompt that can be used as input for a chat model.
+ */
+export function convertMessagesToPrompt(messages, provider) {
+    if (provider === "anthropic") {
+        return convertMessagesToPromptAnthropic(messages);
+    }
+    throw new Error(`Provider ${provider} does not support chat.`);
+}
+/**
+ * A type of Large Language Model (LLM) that interacts with the Bedrock
+ * service. It extends the base `LLM` class and implements the
+ * `BaseBedrockInput` interface. The class is designed to authenticate and
+ * interact with the Bedrock service, which is a part of Amazon Web
+ * Services (AWS). It uses AWS credentials for authentication and can be
+ * configured with various parameters such as the model to use, the AWS
+ * region, and the maximum number of tokens to generate.
+ */
+export class ChatBedrock extends SimpleChatModel {
+    get lc_secrets() {
+        return {};
+    }
+    _llmType() {
+        return "bedrock";
+    }
+    static lc_name() {
+        return "ChatBedrock";
+    }
+    constructor(fields) {
+        super(fields ?? {});
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "amazon.titan-tg1-large"
+        });
+        Object.defineProperty(this, "region", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "credentials", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "temperature", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "maxTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "fetchFn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "endpointHost", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "stopSequences", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "modelKwargs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "codec", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new EventStreamCodec(toUtf8, fromUtf8)
+        });
+        this.model = fields?.model ?? this.model;
+        const allowedModels = ["ai21", "anthropic", "amazon"];
+        if (!allowedModels.includes(this.model.split(".")[0])) {
+            throw new Error(`Unknown model: '${this.model}', only these are supported: ${allowedModels}`);
+        }
+        const region = fields?.region ?? getEnvironmentVariable("AWS_DEFAULT_REGION");
+        if (!region) {
+            throw new Error("Please set the AWS_DEFAULT_REGION environment variable or pass it to the constructor as the region field.");
+        }
+        this.region = region;
+        this.credentials = fields?.credentials ?? defaultProvider();
+        this.temperature = fields?.temperature ?? this.temperature;
+        this.maxTokens = fields?.maxTokens ?? this.maxTokens;
+        this.fetchFn = fields?.fetchFn ?? fetch;
+        this.endpointHost = fields?.endpointHost ?? fields?.endpointUrl;
+        this.stopSequences = fields?.stopSequences;
+        this.modelKwargs = fields?.modelKwargs;
+    }
+    /** Call out to Bedrock service model.
+      Arguments:
+        prompt: The prompt to pass into the model.
+  
+      Returns:
+        The string generated by the model.
+  
+      Example:
+        response = model.call("Tell me a joke.")
+    */
+    async _call(messages, options, runManager) {
+        const chunks = [];
+        for await (const chunk of this._streamResponseChunks(messages, options, runManager)) {
+            chunks.push(chunk);
+        }
+        return chunks.map((chunk) => chunk.text).join("");
+    }
+    async *_streamResponseChunks(messages, options, runManager) {
+        const provider = this.model.split(".")[0];
+        const service = "bedrock-runtime";
+        const inputBody = BedrockLLMInputOutputAdapter.prepareInput(provider, convertMessagesToPromptAnthropic(messages), this.maxTokens, this.temperature, this.stopSequences, this.modelKwargs);
+        const endpointHost = this.endpointHost ?? `${service}.${this.region}.amazonaws.com`;
+        const amazonMethod = provider === "anthropic" ? "invoke-with-response-stream" : "invoke";
+        const url = new URL(`https://${endpointHost}/model/${this.model}/${amazonMethod}`);
+        const request = new HttpRequest({
+            hostname: url.hostname,
+            path: url.pathname,
+            protocol: url.protocol,
+            method: "POST",
+            body: JSON.stringify(inputBody),
+            query: Object.fromEntries(url.searchParams.entries()),
+            headers: {
+                // host is required by AWS Signature V4: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+                host: url.host,
+                accept: "application/json",
+                "content-type": "application/json",
+            },
+        });
+        const signer = new SignatureV4({
+            credentials: this.credentials,
+            service: "bedrock",
+            region: this.region,
+            sha256: Sha256,
+        });
+        const signedRequest = await signer.sign(request);
+        // Send request to AWS using the low-level fetch API
+        const response = await this.caller.callWithOptions({ signal: options.signal }, async () => this.fetchFn(url, {
+            headers: signedRequest.headers,
+            body: signedRequest.body,
+            method: signedRequest.method,
+        }));
+        if (response.status < 200 || response.status >= 300) {
+            throw Error(`Failed to access underlying url '${url}': got ${response.status} ${response.statusText}: ${await response.text()}`);
+        }
+        if (provider === "anthropic") {
+            const reader = response.body?.getReader();
+            const decoder = new TextDecoder();
+            for await (const chunk of this._readChunks(reader)) {
+                const event = this.codec.decode(chunk);
+                if ((event.headers[":event-type"] !== undefined &&
+                    event.headers[":event-type"].value !== "chunk") ||
+                    event.headers[":content-type"].value !== "application/json") {
+                    throw Error(`Failed to get event chunk: got ${chunk}`);
+                }
+                // console.log(decoder.decode(event.body));
+                const body = JSON.parse(decoder.decode(event.body));
+                if (body.message) {
+                    throw new Error(body.message);
+                }
+                if (body.bytes !== undefined) {
+                    const chunkResult = JSON.parse(Buffer.from(body.bytes, "base64").toString());
+                    const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, chunkResult);
+                    yield new ChatGenerationChunk({
+                        text,
+                        message: new AIMessageChunk({ content: text }),
+                    });
+                    await runManager?.handleLLMNewToken(text);
+                }
+            }
+        }
+        else {
+            const json = await response.json();
+            const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, json);
+            yield new ChatGenerationChunk({
+                text,
+                message: new AIMessageChunk({ content: text }),
+            });
+            await runManager?.handleLLMNewToken(text);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    _readChunks(reader) {
+        return {
+            async *[Symbol.asyncIterator]() {
+                let readResult = await reader.read();
+                while (!readResult.done) {
+                    yield readResult.value;
+                    readResult = await reader.read();
+                }
+            },
+        };
+    }
+    _combineLLMOutput() {
+        return {};
+    }
+}

package/dist/document_loaders/web/notionapi.cjs

@@ -119,11 +119,15 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
      * @returns The string of the title.
      */
     getTitle(obj) {
-        if ((0, exports.isPage)(obj) && obj.properties.title.type === "title") {
-            return obj.properties.title.title[0]?.plain_text;
+        if ((0, exports.isPage)(obj)) {
+            const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
+            if (titleProp)
+                return this.getPropValue(titleProp);
         }
         if ((0, exports.isDatabase)(obj))
-            return obj.title[0]?.plain_text;
+            return obj.title
+                .map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
+                .join("");
         return null;
     }
     /**
@@ -284,7 +288,7 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
         });
         this.documents.push(pageDocument);
         this.pageCompleted.push(pageId);
-        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
+        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
     }
     /**
      * Loads a Notion database and adds it's pages to the queue.

package/dist/document_loaders/web/notionapi.js

@@ -111,11 +111,15 @@ export class NotionAPILoader extends BaseDocumentLoader {
      * @returns The string of the title.
      */
     getTitle(obj) {
-        if (isPage(obj) && obj.properties.title.type === "title") {
-            return obj.properties.title.title[0]?.plain_text;
+        if (isPage(obj)) {
+            const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
+            if (titleProp)
+                return this.getPropValue(titleProp);
         }
         if (isDatabase(obj))
-            return obj.title[0]?.plain_text;
+            return obj.title
+                .map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
+                .join("");
         return null;
     }
     /**
@@ -276,7 +280,7 @@ export class NotionAPILoader extends BaseDocumentLoader {
         });
         this.documents.push(pageDocument);
         this.pageCompleted.push(pageId);
-        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
+        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
     }
     /**
      * Loads a Notion database and adds it's pages to the queue.

package/dist/document_loaders/web/searchapi.cjs

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchApiLoader = void 0;
+const env_js_1 = require("../../util/env.cjs");
+const document_js_1 = require("../../document.cjs");
+const base_js_1 = require("../base.cjs");
+/**
+ * Class representing a document loader for loading search results from
+ * the SearchApi. It extends the BaseDocumentLoader class.
+ */
+class SearchApiLoader extends base_js_1.BaseDocumentLoader {
+    constructor(params) {
+        super();
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { apiKey = (0, env_js_1.getEnvironmentVariable)("SEARCHAPI_API_KEY") } = params;
+        if (typeof apiKey !== "string") {
+            throw new Error("Invalid type for apiKey. Expected string.");
+        }
+        if (!apiKey) {
+            throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
+        }
+        this.apiKey = apiKey;
+        this.parameters = { ...params };
+    }
+    /**
+     * Builds the URL for the SearchApi search request.
+     * @returns The URL for the search request.
+     */
+    buildUrl() {
+        this.parameters = {
+            engine: "google",
+            api_key: this.apiKey,
+            ...this.parameters,
+        };
+        const preparedParams = Object.entries(this.parameters)
+            .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
+            .map(([key, value]) => [key, `${value}`]);
+        const searchParams = new URLSearchParams(preparedParams);
+        return `https://www.searchapi.io/api/v1/search?${searchParams}`;
+    }
+    /**
+     * Extracts documents from the provided output.
+     * @param output - The output to extract documents from.
+     * @param responseType - The type of the response to extract documents from.
+     * @returns An array of Documents.
+     */
+    extractDocuments(output, responseType) {
+        const documents = [];
+        const results = Array.isArray(output) ? output : [output];
+        if (responseType === "transcripts") {
+            const pageContent = results.map((result) => result.text).join("\n");
+            const metadata = {
+                source: "SearchApi",
+                responseType,
+            };
+            documents.push(new document_js_1.Document({ pageContent, metadata }));
+        }
+        else {
+            for (const result of results) {
+                const pageContent = JSON.stringify(result);
+                const metadata = {
+                    source: "SearchApi",
+                    responseType,
+                };
+                documents.push(new document_js_1.Document({ pageContent, metadata }));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
+     * @param data - The response data from the SearchApi search request.
+     * @returns An array of Documents.
+     */
+    processResponseData(data) {
+        const documents = [];
+        const responseTypes = [
+            "answer_box",
+            "shopping_results",
+            "knowledge_graph",
+            "organic_results",
+            "transcripts",
+        ];
+        for (const responseType of responseTypes) {
+            if (responseType in data) {
+                documents.push(...this.extractDocuments(data[responseType], responseType));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Fetches the data from the provided URL and returns it as a JSON object.
+     * If an error occurs during the fetch operation, an exception is thrown with the error message.
+     * @param url - The URL to fetch data from.
+     * @returns A promise that resolves to the fetched data as a JSON object.
+     * @throws An error if the fetch operation fails.
+     */
+    async fetchData(url) {
+        const response = await fetch(url);
+        const data = await response.json();
+        if (data.error) {
+            throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
+        }
+        return data;
+    }
+    /**
+     * Loads the search results from the SearchApi.
+     * @returns An array of Documents representing the search results.
+     * @throws An error if the search results could not be loaded.
+     */
+    async load() {
+        const url = this.buildUrl();
+        const data = await this.fetchData(url);
+        try {
+            return this.processResponseData(data);
+        }
+        catch (error) {
+            console.error(error);
+            throw new Error(`Failed to process search results from SearchApi: ${error}`);
+        }
+    }
+}
+exports.SearchApiLoader = SearchApiLoader;

package/dist/document_loaders/web/searchapi.d.ts

@@ -0,0 +1,65 @@
+import { Document } from "../../document.js";
+import { BaseDocumentLoader } from "../base.js";
+type JSONPrimitive = string | number | boolean | null;
+type JSONValue = JSONPrimitive | JSONObject | JSONArray;
+interface JSONObject {
+    [key: string]: JSONValue;
+}
+interface JSONArray extends Array<JSONValue> {
+}
+/**
+ * SearchApiParameters Type Definition.
+ *
+ * For more parameters and supported search engines, refer specific engine documentation:
+ * Google - https://www.searchapi.io/docs/google
+ * Google News - https://www.searchapi.io/docs/google-news
+ * Google Scholar - https://www.searchapi.io/docs/google-scholar
+ * YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts
+ * and others.
+ *
+ */
+type SearchApiParameters = {
+    [key: string]: JSONValue;
+};
+/**
+ * Class representing a document loader for loading search results from
+ * the SearchApi. It extends the BaseDocumentLoader class.
+ */
+export declare class SearchApiLoader extends BaseDocumentLoader {
+    private apiKey;
+    private parameters;
+    constructor(params: SearchApiParameters);
+    /**
+     * Builds the URL for the SearchApi search request.
+     * @returns The URL for the search request.
+     */
+    buildUrl(): string;
+    /**
+     * Extracts documents from the provided output.
+     * @param output - The output to extract documents from.
+     * @param responseType - The type of the response to extract documents from.
+     * @returns An array of Documents.
+     */
+    private extractDocuments;
+    /**
+     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
+     * @param data - The response data from the SearchApi search request.
+     * @returns An array of Documents.
+     */
+    processResponseData(data: Record<string, unknown>): Document[];
+    /**
+     * Fetches the data from the provided URL and returns it as a JSON object.
+     * If an error occurs during the fetch operation, an exception is thrown with the error message.
+     * @param url - The URL to fetch data from.
+     * @returns A promise that resolves to the fetched data as a JSON object.
+     * @throws An error if the fetch operation fails.
+     */
+    private fetchData;
+    /**
+     * Loads the search results from the SearchApi.
+     * @returns An array of Documents representing the search results.
+     * @throws An error if the search results could not be loaded.
+     */
+    load(): Promise<Document[]>;
+}
+export {};

package/dist/document_loaders/web/searchapi.js

@@ -0,0 +1,130 @@
+import { getEnvironmentVariable } from "../../util/env.js";
+import { Document } from "../../document.js";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Class representing a document loader for loading search results from
+ * the SearchApi. It extends the BaseDocumentLoader class.
+ */
+export class SearchApiLoader extends BaseDocumentLoader {
+    constructor(params) {
+        super();
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { apiKey = getEnvironmentVariable("SEARCHAPI_API_KEY") } = params;
+        if (typeof apiKey !== "string") {
+            throw new Error("Invalid type for apiKey. Expected string.");
+        }
+        if (!apiKey) {
+            throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
+        }
+        this.apiKey = apiKey;
+        this.parameters = { ...params };
+    }
+    /**
+     * Builds the URL for the SearchApi search request.
+     * @returns The URL for the search request.
+     */
+    buildUrl() {
+        this.parameters = {
+            engine: "google",
+            api_key: this.apiKey,
+            ...this.parameters,
+        };
+        const preparedParams = Object.entries(this.parameters)
+            .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
+            .map(([key, value]) => [key, `${value}`]);
+        const searchParams = new URLSearchParams(preparedParams);
+        return `https://www.searchapi.io/api/v1/search?${searchParams}`;
+    }
+    /**
+     * Extracts documents from the provided output.
+     * @param output - The output to extract documents from.
+     * @param responseType - The type of the response to extract documents from.
+     * @returns An array of Documents.
+     */
+    extractDocuments(output, responseType) {
+        const documents = [];
+        const results = Array.isArray(output) ? output : [output];
+        if (responseType === "transcripts") {
+            const pageContent = results.map((result) => result.text).join("\n");
+            const metadata = {
+                source: "SearchApi",
+                responseType,
+            };
+            documents.push(new Document({ pageContent, metadata }));
+        }
+        else {
+            for (const result of results) {
+                const pageContent = JSON.stringify(result);
+                const metadata = {
+                    source: "SearchApi",
+                    responseType,
+                };
+                documents.push(new Document({ pageContent, metadata }));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
+     * @param data - The response data from the SearchApi search request.
+     * @returns An array of Documents.
+     */
+    processResponseData(data) {
+        const documents = [];
+        const responseTypes = [
+            "answer_box",
+            "shopping_results",
+            "knowledge_graph",
+            "organic_results",
+            "transcripts",
+        ];
+        for (const responseType of responseTypes) {
+            if (responseType in data) {
+                documents.push(...this.extractDocuments(data[responseType], responseType));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Fetches the data from the provided URL and returns it as a JSON object.
+     * If an error occurs during the fetch operation, an exception is thrown with the error message.
+     * @param url - The URL to fetch data from.
+     * @returns A promise that resolves to the fetched data as a JSON object.
+     * @throws An error if the fetch operation fails.
+     */
+    async fetchData(url) {
+        const response = await fetch(url);
+        const data = await response.json();
+        if (data.error) {
+            throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
+        }
+        return data;
+    }
+    /**
+     * Loads the search results from the SearchApi.
+     * @returns An array of Documents representing the search results.
+     * @throws An error if the search results could not be loaded.
+     */
+    async load() {
+        const url = this.buildUrl();
+        const data = await this.fetchData(url);
+        try {
+            return this.processResponseData(data);
+        }
+        catch (error) {
+            console.error(error);
+            throw new Error(`Failed to process search results from SearchApi: ${error}`);
+        }
+    }
+}