langchain 0.0.142 → 0.0.144

package/dist/document_loaders/web/github.d.ts

@@ -29,8 +29,22 @@ export interface GithubFile {
  * properties specific to the GitHub repository loader.
  */
 export interface GithubRepoLoaderParams extends AsyncCallerParams {
+    /**
+     * The base URL of the GitHub instance.
+     * To be used when you are not targeting github.com, e.g. a GitHub Enterprise instance.
+     */
+    baseUrl?: string;
+    /**
+     * The API endpoint URL of the GitHub instance.
+     * To be used when you are not targeting github.com, e.g. a GitHub Enterprise instance.
+     */
+    apiUrl?: string;
     branch?: string;
     recursive?: boolean;
+    /**
+     * Set to true to recursively process submodules. Is only effective, when recursive=true.
+     */
+    processSubmodules?: boolean;
     unknown?: UnknownHandling;
     accessToken?: string;
     ignoreFiles?: (string | RegExp)[];
@@ -52,19 +66,26 @@ export interface GithubRepoLoaderParams extends AsyncCallerParams {
  * loading files from a GitHub repository.
  */
 export declare class GithubRepoLoader extends BaseDocumentLoader implements GithubRepoLoaderParams {
+    baseUrl: string;
+    apiUrl: string;
     private readonly owner;
     private readonly repo;
     private readonly initialPath;
     private headers;
     branch: string;
     recursive: boolean;
+    processSubmodules: boolean;
     unknown: UnknownHandling;
     accessToken?: string;
     ignoreFiles: (string | RegExp)[];
     ignore?: Ignore;
     verbose?: boolean;
+    maxConcurrency?: number;
+    maxRetries?: number;
     protected caller: AsyncCaller;
-    constructor(githubUrl: string, { accessToken, branch, recursive, unknown, ignoreFiles, ignorePaths, verbose, maxConcurrency, maxRetries, ...rest }?: GithubRepoLoaderParams);
+    ignorePaths?: string[];
+    private submoduleInfos;
+    constructor(githubUrl: string, { accessToken, baseUrl, apiUrl, branch, recursive, processSubmodules, unknown, ignoreFiles, ignorePaths, verbose, maxConcurrency, maxRetries, ...rest }?: GithubRepoLoaderParams);
     /**
      * Extracts the owner, repository, and path from a GitHub URL.
      * @param url The GitHub URL to extract information from.
@@ -78,6 +99,22 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * @returns A promise that resolves to an array of Document instances.
      */
     load(): Promise<Document[]>;
+    /**
+     * Loads the information about Git submodules from the repository, if available.
+     */
+    private getSubmoduleInfo;
+    /**
+     * Parses the given content of a .gitmodules file. Furthermore, queries the current SHA ref of all submodules.
+     * Returns the submodule information as array.
+     * @param gitmodulesContent the content of a .gitmodules file
+     */
+    private parseGitmodules;
+    /**
+     * Loads the documents of the given submodule. Uses the same parameters as for the current repository.
+     * External submodules, i.e. submodules pointing to another GitHub instance, are ignored.
+     * @param submoduleInfo the info about the submodule to be loaded
+     */
+    private loadSubmodule;
     /**
      * Determines whether a file or directory should be ignored based on its
      * path and type.
@@ -109,6 +146,7 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
     private processDirectory;
     /**
      * Fetches the files from a GitHub repository.
+     * If the path denotes a single file, the resulting array contains only one element.
      * @param path The path of the repository to fetch the files from.
      * @returns A promise that resolves to an array of GithubFile instances.
      */
@@ -125,4 +163,9 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * @returns void
      */
     private handleError;
+    /**
+     * Logs the given message to the console, if parameter 'verbose' is set to true.
+     * @param message the message to be logged.
+     */
+    private log;
 }

package/dist/document_loaders/web/github.js

@@ -22,8 +22,20 @@ function isBinaryPath(name) {
  * loading files from a GitHub repository.
  */
 export class GithubRepoLoader extends BaseDocumentLoader {
-    constructor(githubUrl, { accessToken = getEnvironmentVariable("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
+    constructor(githubUrl, { accessToken = getEnvironmentVariable("GITHUB_ACCESS_TOKEN"), baseUrl = "https://github.com", apiUrl = "https://api.github.com", branch = "main", recursive = true, processSubmodules = false, unknown = UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
         super();
+        Object.defineProperty(this, "baseUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "apiUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "owner", {
             enumerable: true,
             configurable: true,
@@ -60,6 +72,12 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "processSubmodules", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "unknown", {
             enumerable: true,
             configurable: true,
@@ -90,22 +108,55 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "maxConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxRetries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "caller", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "ignorePaths", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "submoduleInfos", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.baseUrl = baseUrl;
+        this.apiUrl = apiUrl;
         const { owner, repo, path } = this.extractOwnerAndRepoAndPath(githubUrl);
         this.owner = owner;
         this.repo = repo;
         this.initialPath = path;
         this.branch = branch;
         this.recursive = recursive;
+        // processing submodules without processing contents of other directories makes no sense
+        if (processSubmodules && !recursive) {
+            throw new Error(`Input property "recursive" must be true if "processSubmodules" is true.`);
+        }
+        this.processSubmodules = processSubmodules;
         this.unknown = unknown;
         this.accessToken = accessToken;
         this.ignoreFiles = ignoreFiles;
         this.verbose = verbose;
+        this.maxConcurrency = maxConcurrency;
+        this.maxRetries = maxRetries;
         this.headers = {
             "User-Agent": "langchain",
         };
@@ -114,6 +165,7 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             maxRetries,
             ...rest,
         });
+        this.ignorePaths = ignorePaths;
         if (ignorePaths) {
             this.ignore = ignore.default().add(ignorePaths);
         }
@@ -130,7 +182,7 @@ export class GithubRepoLoader extends BaseDocumentLoader {
      * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
      */
     extractOwnerAndRepoAndPath(url) {
-        const match = url.match(/https:\/\/github.com\/([^/]+)\/([^/]+)(\/tree\/[^/]+\/(.+))?/i);
+        const match = url.match(new RegExp(`${this.baseUrl}/([^/]+)/([^/]+)(/tree/[^/]+/(.+))?`, "i"));
         if (!match) {
             throw new Error("Invalid GitHub URL format.");
         }
@@ -143,10 +195,127 @@ export class GithubRepoLoader extends BaseDocumentLoader {
      * @returns A promise that resolves to an array of Document instances.
      */
     async load() {
-        return (await this.processRepo()).map((fileResponse) => new Document({
+        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
+        // process repository without submodules
+        const documents = (await this.processRepo()).map((fileResponse) => new Document({
             pageContent: fileResponse.contents,
             metadata: fileResponse.metadata,
         }));
+        if (this.processSubmodules) {
+            // process submodules
+            await this.getSubmoduleInfo();
+            for (const submoduleInfo of this.submoduleInfos) {
+                documents.push(...(await this.loadSubmodule(submoduleInfo)));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Loads the information about Git submodules from the repository, if available.
+     */
+    async getSubmoduleInfo() {
+        this.log("Loading info about submodules...");
+        // we have to fetch the files of the root directory to get the download url of the .gitmodules file
+        // however, we cannot reuse the files retrieved in processRepo() as initialPath may be != ""
+        // so it may be that we end up fetching this file list twice
+        const repoFiles = await this.fetchRepoFiles("");
+        const gitmodulesFile = repoFiles.filter(({ name }) => name === ".gitmodules")?.[0];
+        if (gitmodulesFile) {
+            const gitmodulesContent = await this.fetchFileContent({
+                download_url: gitmodulesFile.download_url,
+            });
+            this.submoduleInfos = await this.parseGitmodules(gitmodulesContent);
+        }
+        else {
+            this.submoduleInfos = [];
+        }
+        this.log(`Found ${this.submoduleInfos.length} submodules:`);
+        for (const submoduleInfo of this.submoduleInfos) {
+            this.log(JSON.stringify(submoduleInfo));
+        }
+    }
+    /**
+     * Parses the given content of a .gitmodules file. Furthermore, queries the current SHA ref of all submodules.
+     * Returns the submodule information as array.
+     * @param gitmodulesContent the content of a .gitmodules file
+     */
+    async parseGitmodules(gitmodulesContent) {
+        // catches the initial line of submodule entries
+        const submodulePattern = /\[submodule "(.*?)"]\n((\s+.*?\s*=\s*.*?\n)*)/g;
+        // catches the properties of a submodule
+        const keyValuePattern = /\s+(.*?)\s*=\s*(.*?)\s/g;
+        const submoduleInfos = [];
+        for (const [, name, propertyLines] of gitmodulesContent.matchAll(submodulePattern)) {
+            if (!name || !propertyLines) {
+                throw new Error("Could not parse submodule entry");
+            }
+            const submodulePropertyLines = propertyLines.matchAll(keyValuePattern);
+            let path;
+            let url;
+            for (const [, key, value] of submodulePropertyLines) {
+                if (!key || !value) {
+                    throw new Error(`Could not parse key/value pairs for submodule ${name}`);
+                }
+                switch (key) {
+                    case "path":
+                        path = value;
+                        break;
+                    case "url":
+                        url = value;
+                        if (url.endsWith(".git")) {
+                            url = url.substring(0, url.length - 4);
+                        }
+                        break;
+                    default:
+                    // ignoring unused keys
+                }
+            }
+            if (!path || !url) {
+                throw new Error(`Missing properties for submodule ${name}`);
+            }
+            // fetch the current ref of the submodule
+            const files = await this.fetchRepoFiles(path);
+            const submoduleInfo = {
+                name,
+                path,
+                url,
+                ref: files[0].sha,
+            };
+            submoduleInfos.push(submoduleInfo);
+        }
+        return submoduleInfos;
+    }
+    /**
+     * Loads the documents of the given submodule. Uses the same parameters as for the current repository.
+     * External submodules, i.e. submodules pointing to another GitHub instance, are ignored.
+     * @param submoduleInfo the info about the submodule to be loaded
+     */
+    async loadSubmodule(submoduleInfo) {
+        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
+            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
+            return [];
+        }
+        else if (!submoduleInfo.path.startsWith(this.initialPath)) {
+            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
+            return [];
+        }
+        else {
+            this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
+            return new GithubRepoLoader(submoduleInfo.url, {
+                accessToken: this.accessToken,
+                apiUrl: this.apiUrl,
+                baseUrl: this.baseUrl,
+                branch: submoduleInfo.ref,
+                recursive: this.recursive,
+                processSubmodules: this.processSubmodules,
+                unknown: this.unknown,
+                ignoreFiles: this.ignoreFiles,
+                ignorePaths: this.ignorePaths,
+                verbose: this.verbose,
+                maxConcurrency: this.maxConcurrency,
+                maxRetries: this.maxRetries,
+            }).load();
+        }
     }
     /**
      * Determines whether a file or directory should be ignored based on its
@@ -186,7 +355,11 @@ export class GithubRepoLoader extends BaseDocumentLoader {
         });
         return {
             contents: fileContent || "",
-            metadata: { source: file.path },
+            metadata: {
+                source: file.path,
+                repository: `${this.baseUrl}/${this.owner}/${this.repo}`,
+                branch: this.branch,
+            },
         };
     }
     /**
@@ -197,19 +370,24 @@ export class GithubRepoLoader extends BaseDocumentLoader {
         // Directories have nested files / directories, which is why this is a list of promises of promises
         const currentDirectoryDirectoryPromises = [];
         for (const file of files) {
-            if (!this.shouldIgnore(file.path, file.type)) {
-                if (file.type !== "dir") {
-                    try {
-                        currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
-                    }
-                    catch (e) {
-                        this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
-                    }
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file" && file.size === 0) {
+                // this is a submodule. ignoring for the moment. submodule processing is done separately
+                continue;
+            }
+            if (file.type !== "dir") {
+                try {
+                    currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
                 }
-                else if (this.recursive) {
-                    currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
+                catch (e) {
+                    this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
                 }
             }
+            else if (this.recursive) {
+                currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
+            }
         }
         const curDirDirectories = await Promise.all(currentDirectoryDirectoryPromises);
         return [...currentDirectoryFilePromises, ...curDirDirectories.flat()];
@@ -248,24 +426,25 @@ export class GithubRepoLoader extends BaseDocumentLoader {
     }
     /**
      * Fetches the files from a GitHub repository.
+     * If the path denotes a single file, the resulting array contains only one element.
      * @param path The path of the repository to fetch the files from.
      * @returns A promise that resolves to an array of GithubFile instances.
      */
     async fetchRepoFiles(path) {
-        const url = `https://api.github.com/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
+        const url = `${this.apiUrl}/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
         return this.caller.call(async () => {
-            if (this.verbose) {
-                console.log("Fetching", url);
-            }
+            this.log(`Fetching ${url}`);
             const response = await fetch(url, { headers: this.headers });
             const data = await response.json();
             if (!response.ok) {
                 throw new Error(`Unable to fetch repository files: ${response.status} ${JSON.stringify(data)}`);
             }
-            if (!Array.isArray(data)) {
-                throw new Error("Unable to fetch repository files.");
+            if (Array.isArray(data)) {
+                return data;
+            }
+            else {
+                return [data];
             }
-            return data;
         });
     }
     /**
@@ -275,9 +454,7 @@ export class GithubRepoLoader extends BaseDocumentLoader {
      */
     async fetchFileContent(file) {
         return this.caller.call(async () => {
-            if (this.verbose) {
-                console.log("Fetching", file.download_url);
-            }
+            this.log(`Fetching ${file.download_url}`);
             const response = await fetch(file.download_url, {
                 headers: this.headers,
             });
@@ -302,4 +479,13 @@ export class GithubRepoLoader extends BaseDocumentLoader {
                 throw new Error(`Unknown unknown handling: ${this.unknown}`);
         }
     }
+    /**
+     * Logs the given message to the console, if parameter 'verbose' is set to true.
+     * @param message the message to be logged.
+     */
+    log(message) {
+        if (this.verbose) {
+            console.log(message);
+        }
+    }
 }

package/dist/document_loaders/web/recursive_url.cjs

@@ -84,6 +84,19 @@ class RecursiveUrlLoader extends base_js_1.BaseDocumentLoader {
             if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
                 invalidSuffixes.some((suffix) => link.endsWith(suffix)))
                 continue;
+            let standardizedLink;
+            if (link.startsWith("http")) {
+                standardizedLink = link;
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                standardizedLink = base.protocol + link;
+            }
+            else {
+                standardizedLink = new URL(link, baseUrl).href;
+            }
+            if (this.excludeDirs.some((exDir) => standardizedLink.startsWith(exDir)))
+                continue;
             if (link.startsWith("http")) {
                 const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
                 if (isAllowed)

package/dist/document_loaders/web/recursive_url.js

@@ -81,6 +81,19 @@ export class RecursiveUrlLoader extends BaseDocumentLoader {
             if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
                 invalidSuffixes.some((suffix) => link.endsWith(suffix)))
                 continue;
+            let standardizedLink;
+            if (link.startsWith("http")) {
+                standardizedLink = link;
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                standardizedLink = base.protocol + link;
+            }
+            else {
+                standardizedLink = new URL(link, baseUrl).href;
+            }
+            if (this.excludeDirs.some((exDir) => standardizedLink.startsWith(exDir)))
+                continue;
             if (link.startsWith("http")) {
                 const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
                 if (isAllowed)

package/dist/embeddings/hf_transformers.cjs

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HuggingFaceTransformersEmbeddings = void 0;
+const transformers_1 = require("@xenova/transformers");
+const chunk_js_1 = require("../util/chunk.cjs");
+const base_js_1 = require("./base.cjs");
+class HuggingFaceTransformersEmbeddings extends base_js_1.Embeddings {
+    constructor(fields) {
+        super(fields ?? {});
+        Object.defineProperty(this, "modelName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "Xenova/all-MiniLM-L6-v2"
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 512
+        });
+        Object.defineProperty(this, "stripNewLines", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pipelinePromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.modelName = fields?.modelName ?? this.modelName;
+        this.stripNewLines = fields?.stripNewLines ?? this.stripNewLines;
+        this.timeout = fields?.timeout;
+    }
+    async embedDocuments(texts) {
+        const batches = (0, chunk_js_1.chunkArray)(this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts, this.batchSize);
+        const batchRequests = batches.map((batch) => this.runEmbedding(batch));
+        const batchResponses = await Promise.all(batchRequests);
+        const embeddings = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batchResponse = batchResponses[i];
+            for (let j = 0; j < batchResponse.length; j += 1) {
+                embeddings.push(batchResponse[j]);
+            }
+        }
+        return embeddings;
+    }
+    async embedQuery(text) {
+        const data = await this.runEmbedding([
+            this.stripNewLines ? text.replace(/\n/g, " ") : text,
+        ]);
+        return data[0];
+    }
+    async runEmbedding(texts) {
+        const pipe = await (this.pipelinePromise ??= (0, transformers_1.pipeline)("feature-extraction", this.modelName));
+        return this.caller.call(async () => {
+            const output = await pipe(texts, { pooling: "mean", normalize: true });
+            return output.tolist();
+        });
+    }
+}
+exports.HuggingFaceTransformersEmbeddings = HuggingFaceTransformersEmbeddings;

package/dist/embeddings/hf_transformers.d.ts

@@ -0,0 +1,29 @@
+import { Embeddings, EmbeddingsParams } from "./base.js";
+export interface HuggingFaceTransformersEmbeddingsParams extends EmbeddingsParams {
+    /** Model name to use */
+    modelName: string;
+    /**
+     * Timeout to use when making requests to OpenAI.
+     */
+    timeout?: number;
+    /**
+     * The maximum number of documents to embed in a single request.
+     */
+    batchSize?: number;
+    /**
+     * Whether to strip new lines from the input text. This is recommended by
+     * OpenAI, but may not be suitable for all use cases.
+     */
+    stripNewLines?: boolean;
+}
+export declare class HuggingFaceTransformersEmbeddings extends Embeddings implements HuggingFaceTransformersEmbeddingsParams {
+    modelName: string;
+    batchSize: number;
+    stripNewLines: boolean;
+    timeout?: number;
+    private pipelinePromise;
+    constructor(fields?: Partial<HuggingFaceTransformersEmbeddingsParams>);
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private runEmbedding;
+}

package/dist/embeddings/hf_transformers.js

@@ -0,0 +1,67 @@
+import { pipeline } from "@xenova/transformers";
+import { chunkArray } from "../util/chunk.js";
+import { Embeddings } from "./base.js";
+export class HuggingFaceTransformersEmbeddings extends Embeddings {
+    constructor(fields) {
+        super(fields ?? {});
+        Object.defineProperty(this, "modelName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "Xenova/all-MiniLM-L6-v2"
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 512
+        });
+        Object.defineProperty(this, "stripNewLines", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pipelinePromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.modelName = fields?.modelName ?? this.modelName;
+        this.stripNewLines = fields?.stripNewLines ?? this.stripNewLines;
+        this.timeout = fields?.timeout;
+    }
+    async embedDocuments(texts) {
+        const batches = chunkArray(this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts, this.batchSize);
+        const batchRequests = batches.map((batch) => this.runEmbedding(batch));
+        const batchResponses = await Promise.all(batchRequests);
+        const embeddings = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batchResponse = batchResponses[i];
+            for (let j = 0; j < batchResponse.length; j += 1) {
+                embeddings.push(batchResponse[j]);
+            }
+        }
+        return embeddings;
+    }
+    async embedQuery(text) {
+        const data = await this.runEmbedding([
+            this.stripNewLines ? text.replace(/\n/g, " ") : text,
+        ]);
+        return data[0];
+    }
+    async runEmbedding(texts) {
+        const pipe = await (this.pipelinePromise ??= pipeline("feature-extraction", this.modelName));
+        return this.caller.call(async () => {
+            const output = await pipe(texts, { pooling: "mean", normalize: true });
+            return output.tolist();
+        });
+    }
+}

package/dist/embeddings/openai.cjs

@@ -82,7 +82,7 @@ class OpenAIEmbeddings extends base_js_1.Embeddings {
             writable: true,
             value: void 0
         });
-        const apiKey = fieldsWithDefaults?.openAIApiKey ??
+        let apiKey = fieldsWithDefaults?.openAIApiKey ??
             (0, env_js_1.getEnvironmentVariable)("OPENAI_API_KEY");
         const azureApiKey = fieldsWithDefaults?.azureOpenAIApiKey ??
             (0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_KEY");
@@ -120,6 +120,7 @@ class OpenAIEmbeddings extends base_js_1.Embeddings {
             if (!this.azureOpenAIApiVersion) {
                 throw new Error("Azure OpenAI API version not found");
             }
+            apiKey = apiKey ?? "";
         }
         this.clientConfig = {
             apiKey,

package/dist/embeddings/openai.js

@@ -79,7 +79,7 @@ export class OpenAIEmbeddings extends Embeddings {
             writable: true,
             value: void 0
         });
-        const apiKey = fieldsWithDefaults?.openAIApiKey ??
+        let apiKey = fieldsWithDefaults?.openAIApiKey ??
             getEnvironmentVariable("OPENAI_API_KEY");
         const azureApiKey = fieldsWithDefaults?.azureOpenAIApiKey ??
             getEnvironmentVariable("AZURE_OPENAI_API_KEY");
@@ -117,6 +117,7 @@ export class OpenAIEmbeddings extends Embeddings {
             if (!this.azureOpenAIApiVersion) {
                 throw new Error("Azure OpenAI API version not found");
             }
+            apiKey = apiKey ?? "";
         }
         this.clientConfig = {
             apiKey,