@oracle/langchain-oracledb 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2023 LangChain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,139 @@
+# @oracle/langchain-oracledb
+
+This package contains the LangChain.js integrations for Oracle Database.
+
+## Installation
+
+```bash npm2yarn
+npm install @oracle/langchain-oracledb @langchain/core
+```
+
+This package, along with the main LangChain package, depends on [`@langchain/core`](https://npmjs.com/package/@langchain/core/).
+If you are using this package with other LangChain packages, you should make sure that all of the packages depend on the same instance of @langchain/core.
+You can do so by adding an appropriate field to your project's `package.json` like this:
+
+```json
+{
+  "name": "your-project",
+  "version": "0.1.0",
+  "dependencies": {
+    "@oracle/langchain-oracledb": "^0.1.1",
+    "@langchain/core": "^1.0.0"
+  },
+  "resolutions": {
+    "@langchain/core": "^1.0.0"
+  },
+  "overrides": {
+    "@langchain/core": "^1.0.0"
+  },
+  "pnpm": {
+    "overrides": {
+      "@langchain/core": "^1.0.0"
+    }
+  }
+}
+```
+
+The field you need depends on the package manager you're using, but we recommend adding a field for the common `yarn`, `npm`, and `pnpm` to maximize compatibility.
+
+## Document Loaders
+
+This package includes a document loader for loading documents from different sources and file formats.
+
+```typescript
+import {OracleDocLoader} from "@oracle/langchain-oracledb";
+
+const loader = new OracleDocLoader(conn, loader_params);
+const docs = await loader.load();
+```
+
+## Text Splitter
+
+This package includes a text splitter for chunking documents using the database.
+
+```typescript
+import {OracleTextSplitter} from "@oracle/langchain-oracledb";
+
+const splitter = new OracleTextSplitter(conn, splitter_params);
+let chunks = await splitter.splitText(doc.pageContent);
+```
+
+## Embeddings
+
+This package includes a class for generating embeddings either inside or outside of the database.
+
+```typescript
+import {OracleEmbeddings} from "@oracle/langchain-oracledb";
+
+const embedder = new OracleEmbeddings(conn, embedder_params, proxy);
+const embed = await embedder.embedQuery(chunk);
+```
+
+## Summary
+
+This package includes a class for generating summaries either inside or outside of the database.
+
+```typescript
+import {OracleSummary} from "@oracle/langchain-oracledb";
+
+const model = new OracleSummary(conn, summary_params, proxy);
+const summary = await model.getSummary(doc.pageContent);
+```
+
+## Vector Store
+
+This package includes a vector store for storing, indexing, and querying data in the database.
+
+```typescript
+import {OracleVS} from "@oracle/langchain-oracledb";
+
+oraclevs = new OracleVS(embedder, dbConfig);
+await oraclevs.initialize();
+
+await oraclevs.addDocuments(docs);
+const results = await oraclevs.similaritySearch("hello!", 3);
+```
+
+## Development
+
+To develop the `@oracle/langchain-oracledb` package, you'll need to follow these instructions:
+
+### Install dependencies
+
+```bash
+pnpm install
+```
+
+### Build the package
+
+```bash
+pnpm build
+```
+
+Or from the repo root:
+
+```bash
+pnpm build --filter @oracle/langchain-oracledb
+```
+
+### Run tests
+
+Test files should live within a `tests/` file in the `src/` folder. Unit tests should end in `.test.ts` and integration tests should
+end in `.int.test.ts`:
+
+```bash
+$ pnpm test
+$ pnpm test:int
+```
+
+### Lint & Format
+
+Run the linter & formatter to ensure your code is up to standard:
+
+```bash
+pnpm lint && pnpm format
+```
+
+### Adding new entrypoints
+
+If you add a new file to be exported, either import & re-export from `src/index.ts`, or add it to the `exports` field in the `package.json` file and run `pnpm build` to generate the new entrypoint.

package/dist/document_loaders.cjs

@@ -0,0 +1,211 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OracleDocLoader = void 0;
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+const documents_1 = require("@langchain/core/documents");
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const oracledb_1 = __importDefault(require("oracledb"));
+const htmlparser2 = __importStar(require("htmlparser2"));
+const base_1 = require("@langchain/core/document_loaders/base");
+function* listDir(dir) {
+    const files = node_fs_1.default.readdirSync(dir, { withFileTypes: true });
+    for (const file of files) {
+        if (file.isDirectory()) {
+            yield* listDir(node_path_1.default.join(dir, file.name));
+        }
+        else {
+            yield node_path_1.default.join(dir, file.name);
+        }
+    }
+}
+/**
+ * Load documents from a file, a directory, or a table
+ * If the document isn't a plain text file such as a PDF,
+ * a plain text version will be extracted using utl_to_text
+ * @example
+ * ```typescript
+ * const loader = new OracleDocLoader(conn, params);
+ * const docs = await loader.load();
+ * ```
+ */
+class OracleDocLoader extends base_1.BaseDocumentLoader {
+    constructor(conn, pref) {
+        super();
+        Object.defineProperty(this, "conn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pref", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.conn = conn;
+        this.pref = pref;
+    }
+    /**
+     * A method that loads the text file or blob and returns a promise that
+     * resolves to an array of `Document` instances. It reads the text from
+     * the file or blob using the `readFile` function from the
+     * `node:fs/promises` module or the `text()` method of the blob. It then
+     * parses the text using the `parse()` method and creates a `Document`
+     * instance for each parsed page. The metadata includes the source of the
+     * text (file path or blob) and, if there are multiple pages, the line
+     * number of each page.
+     * @returns A promise that resolves to an array of `Document` instances.
+     */
+    async load() {
+        const docs = [];
+        if ("file" in this.pref) {
+            const doc = await this._loadFromFile(this.pref.file);
+            if (doc != null) {
+                docs.push(doc);
+            }
+        }
+        else if ("dir" in this.pref) {
+            for (const file of listDir(this.pref.dir)) {
+                const doc = await this._loadFromFile(file);
+                if (doc != null) {
+                    docs.push(doc);
+                }
+            }
+        }
+        else if ("tablename" in this.pref) {
+            if (!("owner" in this.pref) || !("colname" in this.pref)) {
+                throw new Error("Invalid preferences: missing owner or colname");
+            }
+            docs.push(...(await this._loadFromTable(this.pref.owner, this.pref.tablename, this.pref.colname)));
+        }
+        else {
+            throw new Error("Invalid preferences: missing file, dir, or tablename");
+        }
+        return docs;
+    }
+    // load from file
+    async _loadFromFile(filename) {
+        let doc = null;
+        // don't specify an encoding to use binary
+        const data = node_fs_1.default.readFileSync(filename);
+        const result = await this.conn.execute(`select dbms_vector_chain.utl_to_text(:content, :pref) text,\
+                        dbms_vector_chain.utl_to_text(:content, json('{"plaintext": "false"}')) metadata from dual`, {
+            content: { val: data, dir: oracledb_1.default.BIND_IN, type: oracledb_1.default.BLOB },
+            pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
+        }, {
+            resultSet: true,
+            fetchInfo: {
+                TEXT: { type: oracledb_1.default.STRING },
+                METADATA: { type: oracledb_1.default.STRING },
+            },
+        });
+        const resultSet = result.resultSet;
+        try {
+            if (resultSet) {
+                for await (const row of resultSet) {
+                    const [plain_text, metadata] = await this._extract(row);
+                    doc = new documents_1.Document({
+                        pageContent: plain_text,
+                        metadata: metadata,
+                    });
+                }
+            }
+        }
+        finally {
+            if (resultSet) {
+                await resultSet.close();
+            }
+        }
+        return doc;
+    }
+    // load from table
+    async _loadFromTable(owner, table, col) {
+        const docs = [];
+        // Check if names are invalid
+        const qn = `${owner}.${table}`;
+        try {
+            const sql = `select sys.dbms_assert.simple_sql_name(:col),\
+                            sys.dbms_assert.qualified_sql_name(:qn) from dual`;
+            const binds = [col, qn];
+            await this.conn.execute(sql, binds);
+        }
+        catch {
+            throw new Error("Invalid owner, table, or column name");
+        }
+        const result = await this.conn.execute((`select dbms_vector_chain.utl_to_text(t.${this.pref.colname}, :pref) text,\
+                  dbms_vector_chain.utl_to_text(t.${this.pref.colname}, json('{"plaintext": "false"}')) metadata\
+                  from ${owner}.${table} t`), {
+            pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
+        }, {
+            resultSet: true,
+            fetchInfo: {
+                TEXT: { type: oracledb_1.default.STRING },
+                METADATA: { type: oracledb_1.default.STRING },
+            },
+        });
+        const resultSet = result.resultSet;
+        try {
+            if (resultSet) {
+                for await (const row of resultSet) {
+                    const [plain_text, metadata] = await this._extract(row);
+                    docs.push(new documents_1.Document({
+                        pageContent: plain_text,
+                        metadata: metadata,
+                    }));
+                }
+            }
+        }
+        finally {
+            if (resultSet) {
+                await resultSet.close();
+            }
+        }
+        return docs;
+    }
+    // extract plain text and metadata from a row
+    async _extract(row) {
+        const [text, htmlMetadata] = row;
+        const metadata = {};
+        const parser = new htmlparser2.Parser({
+            onopentag(name, attrs) {
+                if (name === "meta" && attrs.name) {
+                    metadata[attrs.name] = attrs.content;
+                }
+            },
+        });
+        parser.write(htmlMetadata);
+        parser.end();
+        return [text, metadata];
+    }
+}
+exports.OracleDocLoader = OracleDocLoader;
+//# sourceMappingURL=document_loaders.cjs.map

package/dist/document_loaders.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"document_loaders.cjs","sourceRoot":"","sources":["../src/document_loaders.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sDAAsD;AACtD,uDAAuD;AACvD,yDAAqD;AACrD,sDAAyB;AACzB,0DAA6B;AAC7B,wDAAgC;AAChC,yDAA2C;AAC3C,gEAA2E;AAE3E,QAAQ,CAAC,CAAC,OAAO,CAAC,GAAW;IAC3B,MAAM,KAAK,GAAG,iBAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,KAAK,CAAC,CAAC,OAAO,CAAC,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3C;aAAM;YACL,MAAM,mBAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SACjC;KACF;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAa,eAAgB,SAAQ,yBAAkB;IAKrD,YAAY,IAAyB,EAAE,IAAyB;QAC9D,KAAK,EAAE,CAAC;QALA;;;;;WAA0B;QAE1B;;;;;WAA0B;QAIlC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,IAAI,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE;YACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,IAAI,EAAE;gBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAChB;SACF;aAAM,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE;YAC7B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACzC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,GAAG,IAAI,IAAI,EAAE;oBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBAChB;aACF;SACF;aAAM,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE;YACnC,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE;gBACxD,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;aAClE;YACD,IAAI,CAAC,IAAI,CACP,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,CAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,SAAS,EACnB,IAAI,CAAC,IAAI,CAAC,OAAO,CAClB,CAAC,CACH,CAAC;SACH;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;SACzE;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB;IACT,KAAK,CAAC,aAAa,CAAC,QAAgB;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC;QAEf,0CAA0C;QAC1C,MAAM,IAAI,GAAG,iBAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B;mHACqG,EACpF;YACvB,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,kBAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,kBAAQ,CAAC,IAAI,EAAE;YAClE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;SACtD,EACkC;YACjC,SAAS,EAAE,IAAI;YACf,SAAS,EAAE;gBACT,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE;gBAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE;aACpC;SACD,CACH,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,IAAI;YACF,IAAI,SAAS,EAAE;gBACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,SAAkC,EAAE;oBAC1D,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxD,GAAG,GAAG,IAAI,oBAAQ,CAAC;wBACjB,WAAW,EAAU,UAAU;wBAC/B,QAAQ,EAAuB,QAAQ;qBACxC,CAAC,CAAC;iBACJ;aACF;SACF;gBAAS;YACR,IAAI,SAAS,EAAE;gBACb,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aACzB;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAED,kBAAkB;IACV,KAAK,CAAC,cAAc,CAC1B,KAAa,EACb,KAAa,EACb,GAAW;QAEX,MAAM,IAAI,GAAG,EAAE,CAAC;QAEhB,6BAA6B;QAC7B,MAAM,EAAE,GAAG,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;QAC/B,IAAI;YACF,MAAM,GAAG,GAAG;8EAC4D,CAAC;YACzE,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACrC;QAAC,MAAM;YACN,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;SACzD;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,0CAA0C,IAAI,CAAC,IAAI,CAAC,OAAO;oDACf,IAAI,CAAC,IAAI,CAAC,OAAO;yBAC5C,KAAK,IAAI,KAAK,IAAI,CACpC,EACwB;YACvB,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;SACtD,EACkC;YACjC,SAAS,EAAE,IAAI;YACf,SAAS,EAAE;gBACT,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE;gBAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE;aACpC;SACD,CACH,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,IAAI;YACF,IAAI,SAAS,EAAE;gBACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,SAAkC,EAAE;oBAC1D,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxD,IAAI,CAAC,IAAI,CACP,IAAI,oBAAQ,CAAC;wBACX,WAAW,EAAU,UAAU;wBAC/B,QAAQ,EAAuB,QAAQ;qBACxC,CAAC,CACH,CAAC;iBACH;aACF;SACF;gBAAS;YACR,IAAI,SAAS,EAAE;gBACb,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aACzB;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6CAA6C;IACrC,KAAK,CAAC,QAAQ,CAAC,GAAQ;QAC7B,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,GAAG,GAAG,CAAC;QACjC,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC;YACpC,SAAS,CAAC,IAAI,EAAE,KAAK;gBACnB,IAAI,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE;oBACjC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;iBACtC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC3B,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1B,CAAC;CACF;AA9KD,0CA8KC"}

package/dist/document_loaders.d.cts

@@ -0,0 +1,33 @@
+import { Document } from "@langchain/core/documents";
+import oracledb from "oracledb";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+/**
+ * Load documents from a file, a directory, or a table
+ * If the document isn't a plain text file such as a PDF,
+ * a plain text version will be extracted using utl_to_text
+ * @example
+ * ```typescript
+ * const loader = new OracleDocLoader(conn, params);
+ * const docs = await loader.load();
+ * ```
+ */
+export declare class OracleDocLoader extends BaseDocumentLoader {
+    protected conn: oracledb.Connection;
+    protected pref: Record<string, any>;
+    constructor(conn: oracledb.Connection, pref: Record<string, any>);
+    /**
+     * A method that loads the text file or blob and returns a promise that
+     * resolves to an array of `Document` instances. It reads the text from
+     * the file or blob using the `readFile` function from the
+     * `node:fs/promises` module or the `text()` method of the blob. It then
+     * parses the text using the `parse()` method and creates a `Document`
+     * instance for each parsed page. The metadata includes the source of the
+     * text (file path or blob) and, if there are multiple pages, the line
+     * number of each page.
+     * @returns A promise that resolves to an array of `Document` instances.
+     */
+    load(): Promise<Document[]>;
+    private _loadFromFile;
+    private _loadFromTable;
+    private _extract;
+}

package/dist/document_loaders.d.ts

@@ -0,0 +1,33 @@
+import { Document } from "@langchain/core/documents";
+import oracledb from "oracledb";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+/**
+ * Load documents from a file, a directory, or a table
+ * If the document isn't a plain text file such as a PDF,
+ * a plain text version will be extracted using utl_to_text
+ * @example
+ * ```typescript
+ * const loader = new OracleDocLoader(conn, params);
+ * const docs = await loader.load();
+ * ```
+ */
+export declare class OracleDocLoader extends BaseDocumentLoader {
+    protected conn: oracledb.Connection;
+    protected pref: Record<string, any>;
+    constructor(conn: oracledb.Connection, pref: Record<string, any>);
+    /**
+     * A method that loads the text file or blob and returns a promise that
+     * resolves to an array of `Document` instances. It reads the text from
+     * the file or blob using the `readFile` function from the
+     * `node:fs/promises` module or the `text()` method of the blob. It then
+     * parses the text using the `parse()` method and creates a `Document`
+     * instance for each parsed page. The metadata includes the source of the
+     * text (file path or blob) and, if there are multiple pages, the line
+     * number of each page.
+     * @returns A promise that resolves to an array of `Document` instances.
+     */
+    load(): Promise<Document[]>;
+    private _loadFromFile;
+    private _loadFromTable;
+    private _extract;
+}

package/dist/document_loaders.js

@@ -0,0 +1,181 @@
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { Document } from "@langchain/core/documents";
+import fs from "node:fs";
+import path from "node:path";
+import oracledb from "oracledb";
+import * as htmlparser2 from "htmlparser2";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+function* listDir(dir) {
+    const files = fs.readdirSync(dir, { withFileTypes: true });
+    for (const file of files) {
+        if (file.isDirectory()) {
+            yield* listDir(path.join(dir, file.name));
+        }
+        else {
+            yield path.join(dir, file.name);
+        }
+    }
+}
+/**
+ * Load documents from a file, a directory, or a table
+ * If the document isn't a plain text file such as a PDF,
+ * a plain text version will be extracted using utl_to_text
+ * @example
+ * ```typescript
+ * const loader = new OracleDocLoader(conn, params);
+ * const docs = await loader.load();
+ * ```
+ */
+export class OracleDocLoader extends BaseDocumentLoader {
+    constructor(conn, pref) {
+        super();
+        Object.defineProperty(this, "conn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pref", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.conn = conn;
+        this.pref = pref;
+    }
+    /**
+     * A method that loads the text file or blob and returns a promise that
+     * resolves to an array of `Document` instances. It reads the text from
+     * the file or blob using the `readFile` function from the
+     * `node:fs/promises` module or the `text()` method of the blob. It then
+     * parses the text using the `parse()` method and creates a `Document`
+     * instance for each parsed page. The metadata includes the source of the
+     * text (file path or blob) and, if there are multiple pages, the line
+     * number of each page.
+     * @returns A promise that resolves to an array of `Document` instances.
+     */
+    async load() {
+        const docs = [];
+        if ("file" in this.pref) {
+            const doc = await this._loadFromFile(this.pref.file);
+            if (doc != null) {
+                docs.push(doc);
+            }
+        }
+        else if ("dir" in this.pref) {
+            for (const file of listDir(this.pref.dir)) {
+                const doc = await this._loadFromFile(file);
+                if (doc != null) {
+                    docs.push(doc);
+                }
+            }
+        }
+        else if ("tablename" in this.pref) {
+            if (!("owner" in this.pref) || !("colname" in this.pref)) {
+                throw new Error("Invalid preferences: missing owner or colname");
+            }
+            docs.push(...(await this._loadFromTable(this.pref.owner, this.pref.tablename, this.pref.colname)));
+        }
+        else {
+            throw new Error("Invalid preferences: missing file, dir, or tablename");
+        }
+        return docs;
+    }
+    // load from file
+    async _loadFromFile(filename) {
+        let doc = null;
+        // don't specify an encoding to use binary
+        const data = fs.readFileSync(filename);
+        const result = await this.conn.execute(`select dbms_vector_chain.utl_to_text(:content, :pref) text,\
+                        dbms_vector_chain.utl_to_text(:content, json('{"plaintext": "false"}')) metadata from dual`, {
+            content: { val: data, dir: oracledb.BIND_IN, type: oracledb.BLOB },
+            pref: { val: this.pref, type: oracledb.DB_TYPE_JSON },
+        }, {
+            resultSet: true,
+            fetchInfo: {
+                TEXT: { type: oracledb.STRING },
+                METADATA: { type: oracledb.STRING },
+            },
+        });
+        const resultSet = result.resultSet;
+        try {
+            if (resultSet) {
+                for await (const row of resultSet) {
+                    const [plain_text, metadata] = await this._extract(row);
+                    doc = new Document({
+                        pageContent: plain_text,
+                        metadata: metadata,
+                    });
+                }
+            }
+        }
+        finally {
+            if (resultSet) {
+                await resultSet.close();
+            }
+        }
+        return doc;
+    }
+    // load from table
+    async _loadFromTable(owner, table, col) {
+        const docs = [];
+        // Check if names are invalid
+        const qn = `${owner}.${table}`;
+        try {
+            const sql = `select sys.dbms_assert.simple_sql_name(:col),\
+                            sys.dbms_assert.qualified_sql_name(:qn) from dual`;
+            const binds = [col, qn];
+            await this.conn.execute(sql, binds);
+        }
+        catch {
+            throw new Error("Invalid owner, table, or column name");
+        }
+        const result = await this.conn.execute((`select dbms_vector_chain.utl_to_text(t.${this.pref.colname}, :pref) text,\
+                  dbms_vector_chain.utl_to_text(t.${this.pref.colname}, json('{"plaintext": "false"}')) metadata\
+                  from ${owner}.${table} t`), {
+            pref: { val: this.pref, type: oracledb.DB_TYPE_JSON },
+        }, {
+            resultSet: true,
+            fetchInfo: {
+                TEXT: { type: oracledb.STRING },
+                METADATA: { type: oracledb.STRING },
+            },
+        });
+        const resultSet = result.resultSet;
+        try {
+            if (resultSet) {
+                for await (const row of resultSet) {
+                    const [plain_text, metadata] = await this._extract(row);
+                    docs.push(new Document({
+                        pageContent: plain_text,
+                        metadata: metadata,
+                    }));
+                }
+            }
+        }
+        finally {
+            if (resultSet) {
+                await resultSet.close();
+            }
+        }
+        return docs;
+    }
+    // extract plain text and metadata from a row
+    async _extract(row) {
+        const [text, htmlMetadata] = row;
+        const metadata = {};
+        const parser = new htmlparser2.Parser({
+            onopentag(name, attrs) {
+                if (name === "meta" && attrs.name) {
+                    metadata[attrs.name] = attrs.content;
+                }
+            },
+        });
+        parser.write(htmlMetadata);
+        parser.end();
+        return [text, metadata];
+    }
+}
+//# sourceMappingURL=document_loaders.js.map

package/dist/document_loaders.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document_loaders.js","sourceRoot":"","sources":["../src/document_loaders.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,uDAAuD;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,WAAW,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAE3E,QAAQ,CAAC,CAAC,OAAO,CAAC,GAAW;IAC3B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3C;aAAM;YACL,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SACjC;KACF;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,eAAgB,SAAQ,kBAAkB;IAKrD,YAAY,IAAyB,EAAE,IAAyB;QAC9D,KAAK,EAAE,CAAC;QALA;;;;;WAA0B;QAE1B;;;;;WAA0B;QAIlC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,IAAI,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE;YACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,IAAI,EAAE;gBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aAChB;SACF;aAAM,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE;YAC7B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACzC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,GAAG,IAAI,IAAI,EAAE;oBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBAChB;aACF;SACF;aAAM,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE;YACnC,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE;gBACxD,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;aAClE;YACD,IAAI,CAAC,IAAI,CACP,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,CAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,SAAS,EACnB,IAAI,CAAC,IAAI,CAAC,OAAO,CAClB,CAAC,CACH,CAAC;SACH;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;SACzE;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iBAAiB;IACT,KAAK,CAAC,aAAa,CAAC,QAAgB;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC;QAEf,0CAA0C;QAC1C,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B;mHACqG,EACpF;YACvB,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;YAClE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE;SACtD,EACkC;YACjC,SAAS,EAAE,IAAI;YACf,SAAS,EAAE;gBACT,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE;gBAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE;aACpC;SACD,CACH,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,IAAI;YACF,IAAI,SAAS,EAAE;gBACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,SAAkC,EAAE;oBAC1D,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxD,GAAG,GAAG,IAAI,QAAQ,CAAC;wBACjB,WAAW,EAAU,UAAU;wBAC/B,QAAQ,EAAuB,QAAQ;qBACxC,CAAC,CAAC;iBACJ;aACF;SACF;gBAAS;YACR,IAAI,SAAS,EAAE;gBACb,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aACzB;SACF;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAED,kBAAkB;IACV,KAAK,CAAC,cAAc,CAC1B,KAAa,EACb,KAAa,EACb,GAAW;QAEX,MAAM,IAAI,GAAG,EAAE,CAAC;QAEhB,6BAA6B;QAC7B,MAAM,EAAE,GAAG,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;QAC/B,IAAI;YACF,MAAM,GAAG,GAAG;8EAC4D,CAAC;YACzE,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACrC;QAAC,MAAM;YACN,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;SACzD;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,0CAA0C,IAAI,CAAC,IAAI,CAAC,OAAO;oDACf,IAAI,CAAC,IAAI,CAAC,OAAO;yBAC5C,KAAK,IAAI,KAAK,IAAI,CACpC,EACwB;YACvB,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE;SACtD,EACkC;YACjC,SAAS,EAAE,IAAI;YACf,SAAS,EAAE;gBACT,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE;gBAC/B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE;aACpC;SACD,CACH,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,IAAI;YACF,IAAI,SAAS,EAAE;gBACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,SAAkC,EAAE;oBAC1D,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACxD,IAAI,CAAC,IAAI,CACP,IAAI,QAAQ,CAAC;wBACX,WAAW,EAAU,UAAU;wBAC/B,QAAQ,EAAuB,QAAQ;qBACxC,CAAC,CACH,CAAC;iBACH;aACF;SACF;gBAAS;YACR,IAAI,SAAS,EAAE;gBACb,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aACzB;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6CAA6C;IACrC,KAAK,CAAC,QAAQ,CAAC,GAAQ;QAC7B,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,GAAG,GAAG,CAAC;QACjC,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAE5C,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC;YACpC,SAAS,CAAC,IAAI,EAAE,KAAK;gBACnB,IAAI,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE;oBACjC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;iBACtC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC3B,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1B,CAAC;CACF"}