@oracle/langchain-oracledb 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +139 -0
- package/dist/document_loaders.cjs +211 -0
- package/dist/document_loaders.cjs.map +1 -0
- package/dist/document_loaders.d.cts +33 -0
- package/dist/document_loaders.d.ts +33 -0
- package/dist/document_loaders.js +181 -0
- package/dist/document_loaders.js.map +1 -0
- package/dist/embeddings.cjs +129 -0
- package/dist/embeddings.cjs.map +1 -0
- package/dist/embeddings.d.cts +35 -0
- package/dist/embeddings.d.ts +35 -0
- package/dist/embeddings.js +122 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/index.cjs +22 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/summary.cjs +63 -0
- package/dist/summary.cjs.map +1 -0
- package/dist/summary.d.cts +16 -0
- package/dist/summary.d.ts +16 -0
- package/dist/summary.js +56 -0
- package/dist/summary.js.map +1 -0
- package/dist/text_splitter.cjs +58 -0
- package/dist/text_splitter.cjs.map +1 -0
- package/dist/text_splitter.d.cts +17 -0
- package/dist/text_splitter.d.ts +17 -0
- package/dist/text_splitter.js +51 -0
- package/dist/text_splitter.js.map +1 -0
- package/dist/vectorstores.cjs +658 -0
- package/dist/vectorstores.cjs.map +1 -0
- package/dist/vectorstores.d.cts +98 -0
- package/dist/vectorstores.d.ts +98 -0
- package/dist/vectorstores.js +647 -0
- package/dist/vectorstores.js.map +1 -0
- package/index.cjs +1 -0
- package/index.d.cts +1 -0
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/package.json +88 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OracleEmbeddings = void 0;
|
|
7
|
+
/* eslint-disable import/no-extraneous-dependencies */
|
|
8
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
9
|
+
const embeddings_1 = require("@langchain/core/embeddings");
|
|
10
|
+
const oracledb_1 = __importDefault(require("oracledb"));
|
|
11
|
+
/**
|
|
12
|
+
* Generate embeddings using models through Oracle
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* const embedder = new OracleEmbeddings(conn, params, proxy);
|
|
16
|
+
* const embed = await embedder.embedQuery(chunk);
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
class OracleEmbeddings extends embeddings_1.Embeddings {
|
|
20
|
+
constructor(conn, pref, proxy = "", fields = {}) {
|
|
21
|
+
super(fields ?? {});
|
|
22
|
+
Object.defineProperty(this, "conn", {
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
writable: true,
|
|
26
|
+
value: void 0
|
|
27
|
+
});
|
|
28
|
+
Object.defineProperty(this, "pref", {
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true,
|
|
32
|
+
value: void 0
|
|
33
|
+
});
|
|
34
|
+
Object.defineProperty(this, "proxy", {
|
|
35
|
+
enumerable: true,
|
|
36
|
+
configurable: true,
|
|
37
|
+
writable: true,
|
|
38
|
+
value: void 0
|
|
39
|
+
});
|
|
40
|
+
this.conn = conn;
|
|
41
|
+
this.pref = pref;
|
|
42
|
+
this.proxy = proxy;
|
|
43
|
+
}
|
|
44
|
+
static async loadOnnxModel(conn, dir, onnx_file, model_name) {
|
|
45
|
+
await conn.execute(`begin
|
|
46
|
+
dbms_data_mining.drop_model(model_name => :model, force => true);
|
|
47
|
+
dbms_vector.load_onnx_model(:path, :filename, :model,
|
|
48
|
+
json('{"function" : "embedding", "embeddingOutput" : "embedding" , "input": {"input": ["DATA"]}}'));
|
|
49
|
+
end;`, { path: dir, filename: onnx_file, model: model_name });
|
|
50
|
+
}
|
|
51
|
+
async _embed(texts) {
|
|
52
|
+
// replace newlines, which can negatively affect performance.
|
|
53
|
+
const clean_texts = texts.map((text) => text.replace(/\n/g, " "));
|
|
54
|
+
if (this.proxy) {
|
|
55
|
+
await this.conn.execute("begin utl_http.set_proxy(:proxy); end;", {
|
|
56
|
+
proxy: this.proxy,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
const embeddings = [];
|
|
60
|
+
if (oracledb_1.default.thin) {
|
|
61
|
+
// thin mode, can't use batching
|
|
62
|
+
for (const clean_text of clean_texts) {
|
|
63
|
+
const result = await this.conn.execute((`select t.column_value as data from dbms_vector_chain.utl_to_embeddings(:content, :pref) t`), {
|
|
64
|
+
content: clean_text,
|
|
65
|
+
pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
|
|
66
|
+
}, { fetchInfo: { DATA: { type: oracledb_1.default.STRING } } });
|
|
67
|
+
const rows = result.rows;
|
|
68
|
+
if (Symbol.iterator in Object(rows)) {
|
|
69
|
+
for (const row of rows) {
|
|
70
|
+
const [chunk_str] = row;
|
|
71
|
+
const chunk = JSON.parse(chunk_str);
|
|
72
|
+
const vec = JSON.parse(chunk.embed_vector);
|
|
73
|
+
embeddings.push(vec);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
// thick mode, can use batching
|
|
80
|
+
const chunks = [];
|
|
81
|
+
for (const [i, clean_text] of clean_texts.entries()) {
|
|
82
|
+
const chunk = {
|
|
83
|
+
chunk_id: i,
|
|
84
|
+
chunk_data: clean_text,
|
|
85
|
+
};
|
|
86
|
+
chunks.push(JSON.stringify(chunk));
|
|
87
|
+
}
|
|
88
|
+
const VectorArrayT = await this.conn.getDbObjectClass("SYS.VECTOR_ARRAY_T");
|
|
89
|
+
const inputs = new VectorArrayT(chunks);
|
|
90
|
+
const result = await this.conn.execute((`select t.column_value as data from dbms_vector_chain.utl_to_embeddings(:content, :pref) t`), {
|
|
91
|
+
content: inputs,
|
|
92
|
+
pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
|
|
93
|
+
}, { fetchInfo: { DATA: { type: oracledb_1.default.STRING } } });
|
|
94
|
+
const rows = result.rows;
|
|
95
|
+
if (Symbol.iterator in Object(rows)) {
|
|
96
|
+
for (const row of rows) {
|
|
97
|
+
const [chunk_str] = row;
|
|
98
|
+
const chunk = JSON.parse(chunk_str);
|
|
99
|
+
const vec = JSON.parse(chunk.embed_vector);
|
|
100
|
+
embeddings.push(vec);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return embeddings;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Method that takes a document as input and returns a promise that
|
|
108
|
+
* resolves to an embedding for the document. It calls the _embed method
|
|
109
|
+
* with the document as the input and returns the first embedding in the
|
|
110
|
+
* resulting array.
|
|
111
|
+
* @param document Document to generate an embedding for.
|
|
112
|
+
* @returns Promise that resolves to an embedding for the document.
|
|
113
|
+
*/
|
|
114
|
+
embedQuery(document) {
|
|
115
|
+
return this._embed([document]).then((embeddings) => embeddings[0]);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Method that takes an array of documents as input and returns a promise
|
|
119
|
+
* that resolves to a 2D array of embeddings for each document. It calls
|
|
120
|
+
* the _embed method with the documents as the input.
|
|
121
|
+
* @param documents Array of documents to generate embeddings for.
|
|
122
|
+
* @returns Promise that resolves to a 2D array of embeddings for each document.
|
|
123
|
+
*/
|
|
124
|
+
embedDocuments(documents) {
|
|
125
|
+
return this._embed(documents);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.OracleEmbeddings = OracleEmbeddings;
|
|
129
|
+
//# sourceMappingURL=embeddings.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.cjs","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":";;;;;;AAAA,sDAAsD;AACtD,uDAAuD;AACvD,2DAA0E;AAC1E,wDAAgC;AAEhC;;;;;;;GAOG;AACH,MAAa,gBAAiB,SAAQ,uBAAU;IAO9C,YACE,IAAyB,EACzB,IAA6B,EAC7B,KAAK,GAAG,EAAE,EACV,SAA2B,EAAE;QAE7B,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAZZ;;;;;WAA0B;QAE1B;;;;;WAA8B;QAE9B;;;;;WAAc;QAStB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAyB,EACzB,GAAW,EACX,SAAiB,EACjB,UAAkB;QAElB,MAAM,IAAI,CAAC,OAAO,CAChB;;;;YAIM,EACN,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CACtD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,6DAA6D;QAC7D,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAElE,IAAI,IAAI,CAAC,KAAK,EAAE;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wCAAwC,EAAE;gBAChE,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;SACJ;QAED,MAAM,UAAU,GAAG,EAAE,CAAC;QAEtB,IAAI,kBAAQ,CAAC,IAAI,EAAE;YACjB,gCAAgC;YAEhC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;gBACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,2FAA2F,CAC5F,EACwB;oBACvB,OAAO,EAAE,UAAU;oBACnB,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;iBACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;gBACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;gBAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;oBACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;wBACtB,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;wBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;wBAC3C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBACtB;iBACF;aACF;SACF;aAAM;YACL,+BAA+B;YAE/B,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE;gBACnD,MAAM,KAAK,GAAG;oBACZ,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,UAAU;iBACvB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;aACpC;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CACnD,oBAAoB,CACrB,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;YAExC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,2FAA2F,CAC5F,EACwB;gBACvB,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;aACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;YACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;YAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;gBACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;oBACtB,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;oBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBAC3C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACtB;aACF;SACF;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,UAAU,CAAC,QAAgB;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,SAAmB;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;CACF;AA1ID,4CA0IC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Embeddings, EmbeddingsParams } from "@langchain/core/embeddings";
|
|
2
|
+
import oracledb from "oracledb";
|
|
3
|
+
/**
|
|
4
|
+
* Generate embeddings using models through Oracle
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* const embedder = new OracleEmbeddings(conn, params, proxy);
|
|
8
|
+
* const embed = await embedder.embedQuery(chunk);
|
|
9
|
+
* ```
|
|
10
|
+
*/
|
|
11
|
+
export declare class OracleEmbeddings extends Embeddings {
|
|
12
|
+
protected conn: oracledb.Connection;
|
|
13
|
+
protected pref: Record<string, unknown>;
|
|
14
|
+
protected proxy: string;
|
|
15
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, proxy?: string, fields?: EmbeddingsParams);
|
|
16
|
+
static loadOnnxModel(conn: oracledb.Connection, dir: string, onnx_file: string, model_name: string): Promise<void>;
|
|
17
|
+
_embed(texts: string[]): Promise<any[]>;
|
|
18
|
+
/**
|
|
19
|
+
* Method that takes a document as input and returns a promise that
|
|
20
|
+
* resolves to an embedding for the document. It calls the _embed method
|
|
21
|
+
* with the document as the input and returns the first embedding in the
|
|
22
|
+
* resulting array.
|
|
23
|
+
* @param document Document to generate an embedding for.
|
|
24
|
+
* @returns Promise that resolves to an embedding for the document.
|
|
25
|
+
*/
|
|
26
|
+
embedQuery(document: string): Promise<any>;
|
|
27
|
+
/**
|
|
28
|
+
* Method that takes an array of documents as input and returns a promise
|
|
29
|
+
* that resolves to a 2D array of embeddings for each document. It calls
|
|
30
|
+
* the _embed method with the documents as the input.
|
|
31
|
+
* @param documents Array of documents to generate embeddings for.
|
|
32
|
+
* @returns Promise that resolves to a 2D array of embeddings for each document.
|
|
33
|
+
*/
|
|
34
|
+
embedDocuments(documents: string[]): Promise<any[]>;
|
|
35
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Embeddings, EmbeddingsParams } from "@langchain/core/embeddings";
|
|
2
|
+
import oracledb from "oracledb";
|
|
3
|
+
/**
|
|
4
|
+
* Generate embeddings using models through Oracle
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* const embedder = new OracleEmbeddings(conn, params, proxy);
|
|
8
|
+
* const embed = await embedder.embedQuery(chunk);
|
|
9
|
+
* ```
|
|
10
|
+
*/
|
|
11
|
+
export declare class OracleEmbeddings extends Embeddings {
|
|
12
|
+
protected conn: oracledb.Connection;
|
|
13
|
+
protected pref: Record<string, unknown>;
|
|
14
|
+
protected proxy: string;
|
|
15
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, proxy?: string, fields?: EmbeddingsParams);
|
|
16
|
+
static loadOnnxModel(conn: oracledb.Connection, dir: string, onnx_file: string, model_name: string): Promise<void>;
|
|
17
|
+
_embed(texts: string[]): Promise<any[]>;
|
|
18
|
+
/**
|
|
19
|
+
* Method that takes a document as input and returns a promise that
|
|
20
|
+
* resolves to an embedding for the document. It calls the _embed method
|
|
21
|
+
* with the document as the input and returns the first embedding in the
|
|
22
|
+
* resulting array.
|
|
23
|
+
* @param document Document to generate an embedding for.
|
|
24
|
+
* @returns Promise that resolves to an embedding for the document.
|
|
25
|
+
*/
|
|
26
|
+
embedQuery(document: string): Promise<any>;
|
|
27
|
+
/**
|
|
28
|
+
* Method that takes an array of documents as input and returns a promise
|
|
29
|
+
* that resolves to a 2D array of embeddings for each document. It calls
|
|
30
|
+
* the _embed method with the documents as the input.
|
|
31
|
+
* @param documents Array of documents to generate embeddings for.
|
|
32
|
+
* @returns Promise that resolves to a 2D array of embeddings for each document.
|
|
33
|
+
*/
|
|
34
|
+
embedDocuments(documents: string[]): Promise<any[]>;
|
|
35
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/* eslint-disable import/no-extraneous-dependencies */
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
|
+
import { Embeddings } from "@langchain/core/embeddings";
|
|
4
|
+
import oracledb from "oracledb";
|
|
5
|
+
/**
|
|
6
|
+
* Generate embeddings using models through Oracle
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* const embedder = new OracleEmbeddings(conn, params, proxy);
|
|
10
|
+
* const embed = await embedder.embedQuery(chunk);
|
|
11
|
+
* ```
|
|
12
|
+
*/
|
|
13
|
+
export class OracleEmbeddings extends Embeddings {
|
|
14
|
+
constructor(conn, pref, proxy = "", fields = {}) {
|
|
15
|
+
super(fields ?? {});
|
|
16
|
+
Object.defineProperty(this, "conn", {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true,
|
|
20
|
+
value: void 0
|
|
21
|
+
});
|
|
22
|
+
Object.defineProperty(this, "pref", {
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
writable: true,
|
|
26
|
+
value: void 0
|
|
27
|
+
});
|
|
28
|
+
Object.defineProperty(this, "proxy", {
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true,
|
|
32
|
+
value: void 0
|
|
33
|
+
});
|
|
34
|
+
this.conn = conn;
|
|
35
|
+
this.pref = pref;
|
|
36
|
+
this.proxy = proxy;
|
|
37
|
+
}
|
|
38
|
+
static async loadOnnxModel(conn, dir, onnx_file, model_name) {
|
|
39
|
+
await conn.execute(`begin
|
|
40
|
+
dbms_data_mining.drop_model(model_name => :model, force => true);
|
|
41
|
+
dbms_vector.load_onnx_model(:path, :filename, :model,
|
|
42
|
+
json('{"function" : "embedding", "embeddingOutput" : "embedding" , "input": {"input": ["DATA"]}}'));
|
|
43
|
+
end;`, { path: dir, filename: onnx_file, model: model_name });
|
|
44
|
+
}
|
|
45
|
+
async _embed(texts) {
|
|
46
|
+
// replace newlines, which can negatively affect performance.
|
|
47
|
+
const clean_texts = texts.map((text) => text.replace(/\n/g, " "));
|
|
48
|
+
if (this.proxy) {
|
|
49
|
+
await this.conn.execute("begin utl_http.set_proxy(:proxy); end;", {
|
|
50
|
+
proxy: this.proxy,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
const embeddings = [];
|
|
54
|
+
if (oracledb.thin) {
|
|
55
|
+
// thin mode, can't use batching
|
|
56
|
+
for (const clean_text of clean_texts) {
|
|
57
|
+
const result = await this.conn.execute((`select t.column_value as data from dbms_vector_chain.utl_to_embeddings(:content, :pref) t`), {
|
|
58
|
+
content: clean_text,
|
|
59
|
+
pref: { val: this.pref, type: oracledb.DB_TYPE_JSON },
|
|
60
|
+
}, { fetchInfo: { DATA: { type: oracledb.STRING } } });
|
|
61
|
+
const rows = result.rows;
|
|
62
|
+
if (Symbol.iterator in Object(rows)) {
|
|
63
|
+
for (const row of rows) {
|
|
64
|
+
const [chunk_str] = row;
|
|
65
|
+
const chunk = JSON.parse(chunk_str);
|
|
66
|
+
const vec = JSON.parse(chunk.embed_vector);
|
|
67
|
+
embeddings.push(vec);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
// thick mode, can use batching
|
|
74
|
+
const chunks = [];
|
|
75
|
+
for (const [i, clean_text] of clean_texts.entries()) {
|
|
76
|
+
const chunk = {
|
|
77
|
+
chunk_id: i,
|
|
78
|
+
chunk_data: clean_text,
|
|
79
|
+
};
|
|
80
|
+
chunks.push(JSON.stringify(chunk));
|
|
81
|
+
}
|
|
82
|
+
const VectorArrayT = await this.conn.getDbObjectClass("SYS.VECTOR_ARRAY_T");
|
|
83
|
+
const inputs = new VectorArrayT(chunks);
|
|
84
|
+
const result = await this.conn.execute((`select t.column_value as data from dbms_vector_chain.utl_to_embeddings(:content, :pref) t`), {
|
|
85
|
+
content: inputs,
|
|
86
|
+
pref: { val: this.pref, type: oracledb.DB_TYPE_JSON },
|
|
87
|
+
}, { fetchInfo: { DATA: { type: oracledb.STRING } } });
|
|
88
|
+
const rows = result.rows;
|
|
89
|
+
if (Symbol.iterator in Object(rows)) {
|
|
90
|
+
for (const row of rows) {
|
|
91
|
+
const [chunk_str] = row;
|
|
92
|
+
const chunk = JSON.parse(chunk_str);
|
|
93
|
+
const vec = JSON.parse(chunk.embed_vector);
|
|
94
|
+
embeddings.push(vec);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return embeddings;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Method that takes a document as input and returns a promise that
|
|
102
|
+
* resolves to an embedding for the document. It calls the _embed method
|
|
103
|
+
* with the document as the input and returns the first embedding in the
|
|
104
|
+
* resulting array.
|
|
105
|
+
* @param document Document to generate an embedding for.
|
|
106
|
+
* @returns Promise that resolves to an embedding for the document.
|
|
107
|
+
*/
|
|
108
|
+
embedQuery(document) {
|
|
109
|
+
return this._embed([document]).then((embeddings) => embeddings[0]);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Method that takes an array of documents as input and returns a promise
|
|
113
|
+
* that resolves to a 2D array of embeddings for each document. It calls
|
|
114
|
+
* the _embed method with the documents as the input.
|
|
115
|
+
* @param documents Array of documents to generate embeddings for.
|
|
116
|
+
* @returns Promise that resolves to a 2D array of embeddings for each document.
|
|
117
|
+
*/
|
|
118
|
+
embedDocuments(documents) {
|
|
119
|
+
return this._embed(documents);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=embeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,uDAAuD;AACvD,OAAO,EAAE,UAAU,EAAoB,MAAM,4BAA4B,CAAC;AAC1E,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC;;;;;;;GAOG;AACH,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAO9C,YACE,IAAyB,EACzB,IAA6B,EAC7B,KAAK,GAAG,EAAE,EACV,SAA2B,EAAE;QAE7B,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAZZ;;;;;WAA0B;QAE1B;;;;;WAA8B;QAE9B;;;;;WAAc;QAStB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAyB,EACzB,GAAW,EACX,SAAiB,EACjB,UAAkB;QAElB,MAAM,IAAI,CAAC,OAAO,CAChB;;;;YAIM,EACN,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CACtD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,6DAA6D;QAC7D,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAElE,IAAI,IAAI,CAAC,KAAK,EAAE;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wCAAwC,EAAE;gBAChE,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;SACJ;QAED,MAAM,UAAU,GAAG,EAAE,CAAC;QAEtB,IAAI,QAAQ,CAAC,IAAI,EAAE;YACjB,gCAAgC;YAEhC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;gBACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,2FAA2F,CAC5F,EACwB;oBACvB,OAAO,EAAE,UAAU;oBACnB,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE;iBACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;gBACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;gBAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;oBACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;wBACtB,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;wBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;wBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;wBAC3C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBACtB;iBACF;aACF;SACF;aAAM;YACL,+BAA+B;YAE/B,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE;gBACnD,MAAM,KAAK,GAAG;oBACZ,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,UAAU;iBACvB,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;aACpC;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CACnD,oBAAoB,CACrB,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;YAExC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,2FAA2F,CAC5F,EACwB;gBACvB,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE;aACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;YACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;YAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;gBACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;oBACtB,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;oBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;oBACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBAC3C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACtB;aACF;SACF;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,UAAU,CAAC,QAAgB;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,SAAmB;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;CACF"}
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./document_loaders.cjs"), exports);
|
|
18
|
+
__exportStar(require("./embeddings.cjs"), exports);
|
|
19
|
+
__exportStar(require("./summary.cjs"), exports);
|
|
20
|
+
__exportStar(require("./text_splitter.cjs"), exports);
|
|
21
|
+
__exportStar(require("./vectorstores.cjs"), exports);
|
|
22
|
+
//# sourceMappingURL=index.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.cjs","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wDAAsC;AACtC,kDAAgC;AAChC,+CAA6B;AAC7B,qDAAmC;AACnC,oDAAkC"}
|
package/dist/index.d.cts
ADDED
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC"}
|
package/dist/summary.cjs
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OracleSummary = void 0;
|
|
7
|
+
/* eslint-disable import/no-extraneous-dependencies */
|
|
8
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
9
|
+
const oracledb_1 = __importDefault(require("oracledb"));
|
|
10
|
+
/**
|
|
11
|
+
* Generate a summary using models through Oracle
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* const model = new OracleSummary(conn, params, proxy);
|
|
15
|
+
* let summary = await model.getSummary(doc.pageContent);
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
class OracleSummary {
|
|
19
|
+
constructor(conn, pref, proxy = "") {
|
|
20
|
+
Object.defineProperty(this, "conn", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
Object.defineProperty(this, "pref", {
|
|
27
|
+
enumerable: true,
|
|
28
|
+
configurable: true,
|
|
29
|
+
writable: true,
|
|
30
|
+
value: void 0
|
|
31
|
+
});
|
|
32
|
+
Object.defineProperty(this, "proxy", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
writable: true,
|
|
36
|
+
value: void 0
|
|
37
|
+
});
|
|
38
|
+
this.conn = conn;
|
|
39
|
+
this.pref = pref;
|
|
40
|
+
this.proxy = proxy;
|
|
41
|
+
}
|
|
42
|
+
async getSummary(text) {
|
|
43
|
+
if (this.proxy) {
|
|
44
|
+
await this.conn.execute("begin utl_http.set_proxy(:proxy); end;", {
|
|
45
|
+
proxy: this.proxy,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
let summary = "";
|
|
49
|
+
const result = await this.conn.execute((`select dbms_vector_chain.utl_to_summary(:content, :pref) data from dual`), {
|
|
50
|
+
content: { val: text, dir: oracledb_1.default.BIND_IN, type: oracledb_1.default.CLOB },
|
|
51
|
+
pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
|
|
52
|
+
}, { fetchInfo: { DATA: { type: oracledb_1.default.STRING } } });
|
|
53
|
+
const rows = result.rows;
|
|
54
|
+
if (Symbol.iterator in Object(rows)) {
|
|
55
|
+
for (const row of rows) {
|
|
56
|
+
[summary] = row;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return summary;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.OracleSummary = OracleSummary;
|
|
63
|
+
//# sourceMappingURL=summary.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summary.cjs","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":";;;;;;AAAA,sDAAsD;AACtD,uDAAuD;AACvD,wDAAgC;AAEhC;;;;;;;GAOG;AACH,MAAa,aAAa;IAOxB,YACE,IAAyB,EACzB,IAA6B,EAC7B,KAAK,GAAG,EAAE;QATF;;;;;WAA0B;QAE1B;;;;;WAA8B;QAE9B;;;;;WAAc;QAOtB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,CAAC,KAAK,EAAE;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wCAAwC,EAAE;gBAChE,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;SACJ;QAED,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,yEAAyE,CAC1E,EACwB;YACvB,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,kBAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,kBAAQ,CAAC,IAAI,EAAE;YAClE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;SACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;QACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;QAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;YACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACtB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC;aACjB;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AA7CD,sCA6CC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import oracledb from "oracledb";
|
|
2
|
+
/**
|
|
3
|
+
* Generate a summary using models through Oracle
|
|
4
|
+
* @example
|
|
5
|
+
* ```typescript
|
|
6
|
+
* const model = new OracleSummary(conn, params, proxy);
|
|
7
|
+
* let summary = await model.getSummary(doc.pageContent);
|
|
8
|
+
* ```
|
|
9
|
+
*/
|
|
10
|
+
export declare class OracleSummary {
|
|
11
|
+
protected conn: oracledb.Connection;
|
|
12
|
+
protected pref: Record<string, unknown>;
|
|
13
|
+
protected proxy: string;
|
|
14
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, proxy?: string);
|
|
15
|
+
getSummary(text: string): Promise<string>;
|
|
16
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import oracledb from "oracledb";
|
|
2
|
+
/**
|
|
3
|
+
* Generate a summary using models through Oracle
|
|
4
|
+
* @example
|
|
5
|
+
* ```typescript
|
|
6
|
+
* const model = new OracleSummary(conn, params, proxy);
|
|
7
|
+
* let summary = await model.getSummary(doc.pageContent);
|
|
8
|
+
* ```
|
|
9
|
+
*/
|
|
10
|
+
export declare class OracleSummary {
|
|
11
|
+
protected conn: oracledb.Connection;
|
|
12
|
+
protected pref: Record<string, unknown>;
|
|
13
|
+
protected proxy: string;
|
|
14
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, proxy?: string);
|
|
15
|
+
getSummary(text: string): Promise<string>;
|
|
16
|
+
}
|
package/dist/summary.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/* eslint-disable import/no-extraneous-dependencies */
|
|
2
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
3
|
+
import oracledb from "oracledb";
|
|
4
|
+
/**
|
|
5
|
+
* Generate a summary using models through Oracle
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* const model = new OracleSummary(conn, params, proxy);
|
|
9
|
+
* let summary = await model.getSummary(doc.pageContent);
|
|
10
|
+
* ```
|
|
11
|
+
*/
|
|
12
|
+
export class OracleSummary {
|
|
13
|
+
constructor(conn, pref, proxy = "") {
|
|
14
|
+
Object.defineProperty(this, "conn", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: void 0
|
|
19
|
+
});
|
|
20
|
+
Object.defineProperty(this, "pref", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
Object.defineProperty(this, "proxy", {
|
|
27
|
+
enumerable: true,
|
|
28
|
+
configurable: true,
|
|
29
|
+
writable: true,
|
|
30
|
+
value: void 0
|
|
31
|
+
});
|
|
32
|
+
this.conn = conn;
|
|
33
|
+
this.pref = pref;
|
|
34
|
+
this.proxy = proxy;
|
|
35
|
+
}
|
|
36
|
+
async getSummary(text) {
|
|
37
|
+
if (this.proxy) {
|
|
38
|
+
await this.conn.execute("begin utl_http.set_proxy(:proxy); end;", {
|
|
39
|
+
proxy: this.proxy,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
let summary = "";
|
|
43
|
+
const result = await this.conn.execute((`select dbms_vector_chain.utl_to_summary(:content, :pref) data from dual`), {
|
|
44
|
+
content: { val: text, dir: oracledb.BIND_IN, type: oracledb.CLOB },
|
|
45
|
+
pref: { val: this.pref, type: oracledb.DB_TYPE_JSON },
|
|
46
|
+
}, { fetchInfo: { DATA: { type: oracledb.STRING } } });
|
|
47
|
+
const rows = result.rows;
|
|
48
|
+
if (Symbol.iterator in Object(rows)) {
|
|
49
|
+
for (const row of rows) {
|
|
50
|
+
[summary] = row;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return summary;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=summary.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summary.js","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,uDAAuD;AACvD,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC;;;;;;;GAOG;AACH,MAAM,OAAO,aAAa;IAOxB,YACE,IAAyB,EACzB,IAA6B,EAC7B,KAAK,GAAG,EAAE;QATF;;;;;WAA0B;QAE1B;;;;;WAA8B;QAE9B;;;;;WAAc;QAOtB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,CAAC,KAAK,EAAE;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wCAAwC,EAAE;gBAChE,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;SACJ;QAED,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,yEAAyE,CAC1E,EACwB;YACvB,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;YAClE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,YAAY,EAAE;SACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;QACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;QAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;YACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACtB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC;aACjB;SACF;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OracleTextSplitter = void 0;
|
|
7
|
+
/* eslint-disable import/no-extraneous-dependencies */
|
|
8
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
9
|
+
const oracledb_1 = __importDefault(require("oracledb"));
|
|
10
|
+
const textsplitters_1 = require("@langchain/textsplitters");
|
|
11
|
+
/**
|
|
12
|
+
* Split text into smaller pieces
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* const splitter = new OracleTextSplitter(conn, params);
|
|
16
|
+
* let chunks = await splitter.splitText(doc.pageContent);
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
class OracleTextSplitter extends textsplitters_1.TextSplitter {
|
|
20
|
+
static lc_name() {
|
|
21
|
+
return "OracleTextSplitter";
|
|
22
|
+
}
|
|
23
|
+
constructor(conn, pref, fields) {
|
|
24
|
+
super(fields);
|
|
25
|
+
Object.defineProperty(this, "conn", {
|
|
26
|
+
enumerable: true,
|
|
27
|
+
configurable: true,
|
|
28
|
+
writable: true,
|
|
29
|
+
value: void 0
|
|
30
|
+
});
|
|
31
|
+
Object.defineProperty(this, "pref", {
|
|
32
|
+
enumerable: true,
|
|
33
|
+
configurable: true,
|
|
34
|
+
writable: true,
|
|
35
|
+
value: void 0
|
|
36
|
+
});
|
|
37
|
+
this.conn = conn;
|
|
38
|
+
this.pref = pref;
|
|
39
|
+
}
|
|
40
|
+
async splitText(text) {
|
|
41
|
+
const chunks = [];
|
|
42
|
+
const result = await this.conn.execute((`select t.column_value as data from dbms_vector_chain.utl_to_chunks(:content, :pref) t`), {
|
|
43
|
+
content: { val: text, dir: oracledb_1.default.BIND_IN, type: oracledb_1.default.CLOB },
|
|
44
|
+
pref: { val: this.pref, type: oracledb_1.default.DB_TYPE_JSON },
|
|
45
|
+
}, { fetchInfo: { DATA: { type: oracledb_1.default.STRING } } });
|
|
46
|
+
const rows = result.rows;
|
|
47
|
+
if (Symbol.iterator in Object(rows)) {
|
|
48
|
+
for (const row of rows) {
|
|
49
|
+
const [chunk_str] = row;
|
|
50
|
+
const chunk = JSON.parse(chunk_str);
|
|
51
|
+
chunks.push(chunk.chunk_data);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return chunks;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.OracleTextSplitter = OracleTextSplitter;
|
|
58
|
+
//# sourceMappingURL=text_splitter.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"text_splitter.cjs","sourceRoot":"","sources":["../src/text_splitter.ts"],"names":[],"mappings":";;;;;;AAAA,sDAAsD;AACtD,uDAAuD;AACvD,wDAAgC;AAChC,4DAA4E;AAE5E;;;;;;;GAOG;AACH,MAAa,kBAAmB,SAAQ,4BAAY;IAKlD,MAAM,CAAC,OAAO;QACZ,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,YACE,IAAyB,EACzB,IAA6B,EAC7B,MAA2B;QAE3B,KAAK,CAAC,MAAM,CAAC,CAAC;QAbN;;;;;WAA0B;QAE1B;;;;;WAA8B;QAYtC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC1B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAC5B,CACN,uFAAuF,CACxF,EACwB;YACvB,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,kBAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,kBAAQ,CAAC,IAAI,EAAE;YAClE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAQ,CAAC,YAAY,EAAE;SACtD,EAEW,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,kBAAQ,CAAC,MAAM,EAAE,EAAE,EAC3D,CACF,CAAC;QACF,MAAM,IAAI,GAAQ,MAAM,CAAC,IAAI,CAAC;QAC9B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE;YACnC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACtB,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;gBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;aAC/B;SACF;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA5CD,gDA4CC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import oracledb from "oracledb";
|
|
2
|
+
import { TextSplitter, TextSplitterParams } from "@langchain/textsplitters";
|
|
3
|
+
/**
|
|
4
|
+
* Split text into smaller pieces
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* const splitter = new OracleTextSplitter(conn, params);
|
|
8
|
+
* let chunks = await splitter.splitText(doc.pageContent);
|
|
9
|
+
* ```
|
|
10
|
+
*/
|
|
11
|
+
export declare class OracleTextSplitter extends TextSplitter {
|
|
12
|
+
protected conn: oracledb.Connection;
|
|
13
|
+
protected pref: Record<string, unknown>;
|
|
14
|
+
static lc_name(): string;
|
|
15
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, fields?: TextSplitterParams);
|
|
16
|
+
splitText(text: string): Promise<string[]>;
|
|
17
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import oracledb from "oracledb";
|
|
2
|
+
import { TextSplitter, TextSplitterParams } from "@langchain/textsplitters";
|
|
3
|
+
/**
|
|
4
|
+
* Split text into smaller pieces
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* const splitter = new OracleTextSplitter(conn, params);
|
|
8
|
+
* let chunks = await splitter.splitText(doc.pageContent);
|
|
9
|
+
* ```
|
|
10
|
+
*/
|
|
11
|
+
export declare class OracleTextSplitter extends TextSplitter {
|
|
12
|
+
protected conn: oracledb.Connection;
|
|
13
|
+
protected pref: Record<string, unknown>;
|
|
14
|
+
static lc_name(): string;
|
|
15
|
+
constructor(conn: oracledb.Connection, pref: Record<string, unknown>, fields?: TextSplitterParams);
|
|
16
|
+
splitText(text: string): Promise<string[]>;
|
|
17
|
+
}
|