@nocobase/plugin-ai 1.6.0-beta.17

package/dist/node_modules/@langchain/core/dist/indexing/base.js

@@ -0,0 +1,263 @@
+import { v5 as uuidv5 } from "uuid";
+import { UUIDV5_NAMESPACE } from "./record_manager.js";
+import { insecureHash } from "../utils/hash.js";
+import { Document } from "../documents/document.js";
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+export class _HashedDocument {
+    constructor(fields) {
+        Object.defineProperty(this, "uid", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "hash_", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contentHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.uid = fields.uid;
+        this.pageContent = fields.pageContent;
+        this.metadata = fields.metadata;
+    }
+    calculateHashes() {
+        const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
+        for (const key of forbiddenKeys) {
+            if (key in this.metadata) {
+                throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
+            }
+        }
+        const contentHash = this._hashStringToUUID(this.pageContent);
+        try {
+            const metadataHash = this._hashNestedDictToUUID(this.metadata);
+            this.contentHash = contentHash;
+            this.metadataHash = metadataHash;
+        }
+        catch (e) {
+            throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
+        }
+        this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
+        if (!this.uid) {
+            this.uid = this.hash_;
+        }
+    }
+    toDocument() {
+        return new Document({
+            pageContent: this.pageContent,
+            metadata: this.metadata,
+        });
+    }
+    static fromDocument(document, uid) {
+        const doc = new this({
+            pageContent: document.pageContent,
+            metadata: document.metadata,
+            uid: uid || document.uid,
+        });
+        doc.calculateHashes();
+        return doc;
+    }
+    _hashStringToUUID(inputString) {
+        const hash_value = insecureHash(inputString);
+        return uuidv5(hash_value, UUIDV5_NAMESPACE);
+    }
+    _hashNestedDictToUUID(data) {
+        const serialized_data = JSON.stringify(data, Object.keys(data).sort());
+        const hash_value = insecureHash(serialized_data);
+        return uuidv5(hash_value, UUIDV5_NAMESPACE);
+    }
+}
+export function _batch(size, iterable) {
+    const batches = [];
+    let currentBatch = [];
+    iterable.forEach((item) => {
+        currentBatch.push(item);
+        if (currentBatch.length >= size) {
+            batches.push(currentBatch);
+            currentBatch = [];
+        }
+    });
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+}
+export function _deduplicateInOrder(hashedDocuments) {
+    const seen = new Set();
+    const deduplicated = [];
+    for (const hashedDoc of hashedDocuments) {
+        if (!hashedDoc.hash_) {
+            throw new Error("Hashed document does not have a hash");
+        }
+        if (!seen.has(hashedDoc.hash_)) {
+            seen.add(hashedDoc.hash_);
+            deduplicated.push(hashedDoc);
+        }
+    }
+    return deduplicated;
+}
+export function _getSourceIdAssigner(sourceIdKey) {
+    if (sourceIdKey === null) {
+        return (_doc) => null;
+    }
+    else if (typeof sourceIdKey === "string") {
+        return (doc) => doc.metadata[sourceIdKey];
+    }
+    else if (typeof sourceIdKey === "function") {
+        return sourceIdKey;
+    }
+    else {
+        throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
+    }
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const _isBaseDocumentLoader = (arg) => {
+    if ("load" in arg &&
+        typeof arg.load === "function" &&
+        "loadAndSplit" in arg &&
+        typeof arg.loadAndSplit === "function") {
+        return true;
+    }
+    return false;
+};
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+export async function index(args) {
+    const { docsSource, recordManager, vectorStore, options } = args;
+    const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
+    if (cleanup === "incremental" && !sourceIdKey) {
+        throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
+    }
+    const docs = _isBaseDocumentLoader(docsSource)
+        ? await docsSource.load()
+        : docsSource;
+    const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
+    const indexStartDt = await recordManager.getTime();
+    let numAdded = 0;
+    let numDeleted = 0;
+    let numUpdated = 0;
+    let numSkipped = 0;
+    const batches = _batch(batchSize ?? 100, docs);
+    for (const batch of batches) {
+        const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
+        const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
+        if (cleanup === "incremental") {
+            hashedDocs.forEach((_hashedDoc, index) => {
+                const source = sourceIds[index];
+                if (source === null) {
+                    throw new Error("sourceIdKey must be provided when cleanup is incremental");
+                }
+            });
+        }
+        const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
+        const uids = [];
+        const docsToIndex = [];
+        const docsToUpdate = [];
+        const seenDocs = new Set();
+        hashedDocs.forEach((hashedDoc, i) => {
+            const docExists = batchExists[i];
+            if (docExists) {
+                if (forceUpdate) {
+                    seenDocs.add(hashedDoc.uid);
+                }
+                else {
+                    docsToUpdate.push(hashedDoc.uid);
+                    return;
+                }
+            }
+            uids.push(hashedDoc.uid);
+            docsToIndex.push(hashedDoc.toDocument());
+        });
+        if (docsToUpdate.length > 0) {
+            await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
+            numSkipped += docsToUpdate.length;
+        }
+        if (docsToIndex.length > 0) {
+            await vectorStore.addDocuments(docsToIndex, { ids: uids });
+            numAdded += docsToIndex.length - seenDocs.size;
+            numUpdated += seenDocs.size;
+        }
+        await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
+        if (cleanup === "incremental") {
+            sourceIds.forEach((sourceId) => {
+                if (!sourceId)
+                    throw new Error("Source id cannot be null");
+            });
+            const uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                groupIds: sourceIds,
+            });
+            if (uidsToDelete.length > 0) {
+                await vectorStore.delete({ ids: uidsToDelete });
+                await recordManager.deleteKeys(uidsToDelete);
+                numDeleted += uidsToDelete.length;
+            }
+        }
+    }
+    if (cleanup === "full") {
+        let uidsToDelete = await recordManager.listKeys({
+            before: indexStartDt,
+            limit: cleanupBatchSize,
+        });
+        while (uidsToDelete.length > 0) {
+            await vectorStore.delete({ ids: uidsToDelete });
+            await recordManager.deleteKeys(uidsToDelete);
+            numDeleted += uidsToDelete.length;
+            uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                limit: cleanupBatchSize,
+            });
+        }
+    }
+    return {
+        numAdded,
+        numDeleted,
+        numUpdated,
+        numSkipped,
+    };
+}

package/dist/node_modules/@langchain/core/dist/indexing/index.cjs

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./record_manager.cjs"), exports);
+__exportStar(require("./base.cjs"), exports);

package/dist/node_modules/@langchain/core/dist/indexing/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./record_manager.js";
+export * from "./base.js";

package/dist/node_modules/@langchain/core/dist/indexing/index.js

@@ -0,0 +1,2 @@
+export * from "./record_manager.js";
+export * from "./base.js";

package/dist/node_modules/@langchain/core/dist/indexing/record_manager.cjs

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordManager = exports.UUIDV5_NAMESPACE = void 0;
+const serializable_js_1 = require("../load/serializable.cjs");
+// Arbitrary value, used for generating namespaced UUIDs.
+exports.UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+class RecordManager extends serializable_js_1.Serializable {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "recordmanagers"]
+        });
+    }
+}
+exports.RecordManager = RecordManager;

package/dist/node_modules/@langchain/core/dist/indexing/record_manager.d.ts

@@ -0,0 +1,64 @@
+import { Serializable } from "../load/serializable.js";
+export declare const UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+export type UpdateOptions = {
+    groupIds?: (string | null)[];
+    timeAtLeast?: number;
+};
+export type ListKeyOptions = {
+    before?: number;
+    after?: number;
+    groupIds?: (string | null)[];
+    limit?: number;
+};
+export interface RecordManagerInterface {
+    /**
+     * Creates schema in the record manager.
+     * @returns Promise
+     */
+    createSchema(): Promise<void>;
+    /**
+     * Returns current time from the record manager.
+     * @returns Current time
+     */
+    getTime(): Promise<number>;
+    /**
+     * Updates keys in the record manager.
+     * @param keys List of keys to update
+     * @param groupIds List of groupIds to update
+     * @param timeAtLeast Update only if current time is at least this value
+     * @returns Promise
+     * @throws Error if timeAtLeast is provided and current time is less than timeAtLeast
+     * @throws Error if number of keys does not match number of groupIds
+     */
+    update(keys: string[], updateOptions: UpdateOptions): Promise<void>;
+    /**
+     * Checks if keys exist in the record manager.
+     * @param keys List of keys to check
+     * @returns List of booleans indicating if key exists in same order as provided keys
+     */
+    exists(keys: string[]): Promise<boolean[]>;
+    /**
+     * Lists keys from the record manager.
+     * @param before List keys before this timestamp
+     * @param after List keys after this timestamp
+     * @param groupIds List keys with these groupIds
+     * @param limit Limit the number of keys returned
+     * @returns List of keys
+     *
+     */
+    listKeys(options: ListKeyOptions): Promise<string[]>;
+    /**
+     * Deletes keys from the record manager.
+     * @param keys List of keys to delete
+     */
+    deleteKeys(keys: string[]): Promise<void>;
+}
+export declare abstract class RecordManager extends Serializable implements RecordManagerInterface {
+    lc_namespace: string[];
+    abstract createSchema(): Promise<void>;
+    abstract getTime(): Promise<number>;
+    abstract update(keys: string[], updateOptions?: UpdateOptions): Promise<void>;
+    abstract exists(keys: string[]): Promise<boolean[]>;
+    abstract listKeys(options?: ListKeyOptions): Promise<string[]>;
+    abstract deleteKeys(keys: string[]): Promise<void>;
+}

package/dist/node_modules/@langchain/core/dist/indexing/record_manager.js

@@ -0,0 +1,14 @@
+import { Serializable } from "../load/serializable.js";
+// Arbitrary value, used for generating namespaced UUIDs.
+export const UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+export class RecordManager extends Serializable {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "recordmanagers"]
+        });
+    }
+}

package/dist/node_modules/@langchain/core/dist/language_models/base.cjs

@@ -0,0 +1,283 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseLanguageModel = exports.BaseLangChain = exports.calculateMaxTokens = exports.isOpenAITool = exports.getModelContextSize = exports.getEmbeddingContextSize = exports.getModelNameForTiktoken = void 0;
+const base_js_1 = require("../caches/base.cjs");
+const prompt_values_js_1 = require("../prompt_values.cjs");
+const utils_js_1 = require("../messages/utils.cjs");
+const async_caller_js_1 = require("../utils/async_caller.cjs");
+const tiktoken_js_1 = require("../utils/tiktoken.cjs");
+const base_js_2 = require("../runnables/base.cjs");
+// https://www.npmjs.com/package/js-tiktoken
+const getModelNameForTiktoken = (modelName) => {
+    if (modelName.startsWith("gpt-3.5-turbo-16k")) {
+        return "gpt-3.5-turbo-16k";
+    }
+    if (modelName.startsWith("gpt-3.5-turbo-")) {
+        return "gpt-3.5-turbo";
+    }
+    if (modelName.startsWith("gpt-4-32k")) {
+        return "gpt-4-32k";
+    }
+    if (modelName.startsWith("gpt-4-")) {
+        return "gpt-4";
+    }
+    if (modelName.startsWith("gpt-4o")) {
+        return "gpt-4o";
+    }
+    return modelName;
+};
+exports.getModelNameForTiktoken = getModelNameForTiktoken;
+const getEmbeddingContextSize = (modelName) => {
+    switch (modelName) {
+        case "text-embedding-ada-002":
+            return 8191;
+        default:
+            return 2046;
+    }
+};
+exports.getEmbeddingContextSize = getEmbeddingContextSize;
+const getModelContextSize = (modelName) => {
+    switch ((0, exports.getModelNameForTiktoken)(modelName)) {
+        case "gpt-3.5-turbo-16k":
+            return 16384;
+        case "gpt-3.5-turbo":
+            return 4096;
+        case "gpt-4-32k":
+            return 32768;
+        case "gpt-4":
+            return 8192;
+        case "text-davinci-003":
+            return 4097;
+        case "text-curie-001":
+            return 2048;
+        case "text-babbage-001":
+            return 2048;
+        case "text-ada-001":
+            return 2048;
+        case "code-davinci-002":
+            return 8000;
+        case "code-cushman-001":
+            return 2048;
+        default:
+            return 4097;
+    }
+};
+exports.getModelContextSize = getModelContextSize;
+/**
+ * Whether or not the input matches the OpenAI tool definition.
+ * @param {unknown} tool The input to check.
+ * @returns {boolean} Whether the input is an OpenAI tool definition.
+ */
+function isOpenAITool(tool) {
+    if (typeof tool !== "object" || !tool)
+        return false;
+    if ("type" in tool &&
+        tool.type === "function" &&
+        "function" in tool &&
+        typeof tool.function === "object" &&
+        tool.function &&
+        "name" in tool.function &&
+        "parameters" in tool.function) {
+        return true;
+    }
+    return false;
+}
+exports.isOpenAITool = isOpenAITool;
+const calculateMaxTokens = async ({ prompt, modelName, }) => {
+    let numTokens;
+    try {
+        numTokens = (await (0, tiktoken_js_1.encodingForModel)((0, exports.getModelNameForTiktoken)(modelName))).encode(prompt).length;
+    }
+    catch (error) {
+        console.warn("Failed to calculate number of tokens, falling back to approximate count");
+        // fallback to approximate calculation if tiktoken is not available
+        // each token is ~4 characters: https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them#
+        numTokens = Math.ceil(prompt.length / 4);
+    }
+    const maxTokens = (0, exports.getModelContextSize)(modelName);
+    return maxTokens - numTokens;
+};
+exports.calculateMaxTokens = calculateMaxTokens;
+const getVerbosity = () => false;
+/**
+ * Base class for language models, chains, tools.
+ */
+class BaseLangChain extends base_js_2.Runnable {
+    get lc_attributes() {
+        return {
+            callbacks: undefined,
+            verbose: undefined,
+        };
+    }
+    constructor(params) {
+        super(params);
+        /**
+         * Whether to print out response text.
+         */
+        Object.defineProperty(this, "verbose", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "callbacks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tags", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.verbose = params.verbose ?? getVerbosity();
+        this.callbacks = params.callbacks;
+        this.tags = params.tags ?? [];
+        this.metadata = params.metadata ?? {};
+    }
+}
+exports.BaseLangChain = BaseLangChain;
+/**
+ * Base class for language models.
+ */
+class BaseLanguageModel extends BaseLangChain {
+    /**
+     * Keys that the language model accepts as call options.
+     */
+    get callKeys() {
+        return ["stop", "timeout", "signal", "tags", "metadata", "callbacks"];
+    }
+    constructor({ callbacks, callbackManager, ...params }) {
+        const { cache, ...rest } = params;
+        super({
+            callbacks: callbacks ?? callbackManager,
+            ...rest,
+        });
+        /**
+         * The async caller should be used by subclasses to make any async calls,
+         * which will thus benefit from the concurrency and retry logic.
+         */
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_encoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (typeof cache === "object") {
+            this.cache = cache;
+        }
+        else if (cache) {
+            this.cache = base_js_1.InMemoryCache.global();
+        }
+        else {
+            this.cache = undefined;
+        }
+        this.caller = new async_caller_js_1.AsyncCaller(params ?? {});
+    }
+    async getNumTokens(content) {
+        // TODO: Figure out correct value.
+        if (typeof content !== "string") {
+            return 0;
+        }
+        // fallback to approximate calculation if tiktoken is not available
+        let numTokens = Math.ceil(content.length / 4);
+        if (!this._encoding) {
+            try {
+                this._encoding = await (0, tiktoken_js_1.encodingForModel)("modelName" in this
+                    ? (0, exports.getModelNameForTiktoken)(this.modelName)
+                    : "gpt2");
+            }
+            catch (error) {
+                console.warn("Failed to calculate number of tokens, falling back to approximate count", error);
+            }
+        }
+        if (this._encoding) {
+            try {
+                numTokens = this._encoding.encode(content).length;
+            }
+            catch (error) {
+                console.warn("Failed to calculate number of tokens, falling back to approximate count", error);
+            }
+        }
+        return numTokens;
+    }
+    static _convertInputToPromptValue(input) {
+        if (typeof input === "string") {
+            return new prompt_values_js_1.StringPromptValue(input);
+        }
+        else if (Array.isArray(input)) {
+            return new prompt_values_js_1.ChatPromptValue(input.map(utils_js_1.coerceMessageLikeToMessage));
+        }
+        else {
+            return input;
+        }
+    }
+    /**
+     * Get the identifying parameters of the LLM.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    _identifyingParams() {
+        return {};
+    }
+    /**
+     * Create a unique cache key for a specific call to a specific language model.
+     * @param callOptions Call options for the model
+     * @returns A unique cache key.
+     */
+    _getSerializedCacheKeyParametersForCall(
+    // TODO: Fix when we remove the RunnableLambda backwards compatibility shim.
+    { config, ...callOptions }) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const params = {
+            ...this._identifyingParams(),
+            ...callOptions,
+            _type: this._llmType(),
+            _model: this._modelType(),
+        };
+        const filteredEntries = Object.entries(params).filter(([_, value]) => value !== undefined);
+        const serializedEntries = filteredEntries
+            .map(([key, value]) => `${key}:${JSON.stringify(value)}`)
+            .sort()
+            .join(",");
+        return serializedEntries;
+    }
+    /**
+     * @deprecated
+     * Return a json-like object representing this LLM.
+     */
+    serialize() {
+        return {
+            ...this._identifyingParams(),
+            _type: this._llmType(),
+            _model: this._modelType(),
+        };
+    }
+    /**
+     * @deprecated
+     * Load an LLM from a json-like object describing it.
+     */
+    static async deserialize(_data) {
+        throw new Error("Use .toJSON() instead");
+    }
+}
+exports.BaseLanguageModel = BaseLanguageModel;