@langchain/google-cloud-sql-pg 0.0.2 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/LICENSE +6 -6
- package/README.md +1 -1
- package/dist/_virtual/rolldown_runtime.cjs +25 -0
- package/dist/chat_message_history.cjs +104 -137
- package/dist/chat_message_history.cjs.map +1 -0
- package/dist/chat_message_history.d.cts +57 -0
- package/dist/chat_message_history.d.ts +53 -43
- package/dist/chat_message_history.js +103 -133
- package/dist/chat_message_history.js.map +1 -0
- package/dist/engine.cjs +188 -242
- package/dist/engine.cjs.map +1 -0
- package/dist/engine.d.cts +138 -0
- package/dist/engine.d.ts +102 -80
- package/dist/engine.js +186 -234
- package/dist/engine.js.map +1 -0
- package/dist/index.cjs +21 -20
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +7 -4
- package/dist/indexes.cjs +96 -168
- package/dist/indexes.cjs.map +1 -0
- package/dist/indexes.d.cts +68 -0
- package/dist/indexes.d.ts +50 -47
- package/dist/indexes.js +90 -161
- package/dist/indexes.js.map +1 -0
- package/dist/loader.cjs +159 -242
- package/dist/loader.cjs.map +1 -0
- package/dist/loader.d.cts +101 -0
- package/dist/loader.d.ts +40 -26
- package/dist/loader.js +157 -237
- package/dist/loader.js.map +1 -0
- package/dist/utils/utils.cjs +36 -65
- package/dist/utils/utils.cjs.map +1 -0
- package/dist/utils/utils.js +36 -62
- package/dist/utils/utils.js.map +1 -0
- package/dist/vectorstore.cjs +438 -593
- package/dist/vectorstore.cjs.map +1 -0
- package/dist/vectorstore.d.cts +300 -0
- package/dist/vectorstore.d.ts +147 -130
- package/dist/vectorstore.js +436 -588
- package/dist/vectorstore.js.map +1 -0
- package/package.json +41 -48
- package/dist/utils/utils.d.ts +0 -22
- package/index.cjs +0 -1
- package/index.d.cts +0 -1
- package/index.d.ts +0 -1
- package/index.js +0 -1
package/dist/loader.cjs
CHANGED
|
@@ -1,246 +1,163 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
2
|
+
const require_utils = require('./utils/utils.cjs');
|
|
3
|
+
const __langchain_core_document_loaders_base = require_rolldown_runtime.__toESM(require("@langchain/core/document_loaders/base"));
|
|
4
|
+
|
|
5
|
+
//#region src/loader.ts
|
|
6
6
|
const DEFAULT_METADATA_COL = "langchain_metadata";
|
|
7
|
-
function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter =
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
return { pageContent, metadata };
|
|
7
|
+
function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter = require_utils.textFormatter) {
|
|
8
|
+
const pageContent = formatter(row, contentColumns);
|
|
9
|
+
const metadata = {};
|
|
10
|
+
if (metadataJsonColumn && row[metadataJsonColumn]) Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {
|
|
11
|
+
metadata[k] = v;
|
|
12
|
+
});
|
|
13
|
+
metadataColumns.forEach((column) => {
|
|
14
|
+
if (column in row && column !== metadataJsonColumn) metadata[column] = row[column];
|
|
15
|
+
});
|
|
16
|
+
return {
|
|
17
|
+
pageContent,
|
|
18
|
+
metadata
|
|
19
|
+
};
|
|
21
20
|
}
|
|
22
21
|
/**
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
class PostgresLoader extends
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
formatFunc = utils_js_1.jsonFormatter;
|
|
164
|
-
}
|
|
165
|
-
else {
|
|
166
|
-
formatFunc = utils_js_1.textFormatter;
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
let queryStmt = query;
|
|
170
|
-
if (!queryStmt) {
|
|
171
|
-
queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
|
|
172
|
-
}
|
|
173
|
-
let result;
|
|
174
|
-
try {
|
|
175
|
-
result = await engine.pool.raw(queryStmt);
|
|
176
|
-
}
|
|
177
|
-
catch (error) {
|
|
178
|
-
if (typeof error === "string") {
|
|
179
|
-
throw Error(error);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
const columnNames = result.fields.map((field) => field.name);
|
|
183
|
-
const contentColumnNames = contentColumns || [columnNames[0]];
|
|
184
|
-
const metadataColumnNames = metadataColumns ||
|
|
185
|
-
columnNames.filter((col) => !contentColumnNames.includes(col));
|
|
186
|
-
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {
|
|
187
|
-
throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
188
|
-
}
|
|
189
|
-
let jsonColumnName = metadataJsonColumn;
|
|
190
|
-
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {
|
|
191
|
-
jsonColumnName = DEFAULT_METADATA_COL;
|
|
192
|
-
}
|
|
193
|
-
const allNames = [
|
|
194
|
-
...(contentColumnNames || []),
|
|
195
|
-
...(metadataColumnNames || []),
|
|
196
|
-
];
|
|
197
|
-
allNames.forEach((name) => {
|
|
198
|
-
if (!columnNames.includes(name)) {
|
|
199
|
-
throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
200
|
-
}
|
|
201
|
-
});
|
|
202
|
-
return new PostgresLoader(engine, {
|
|
203
|
-
contentColumns: contentColumnNames,
|
|
204
|
-
metadataColumns: metadataColumnNames,
|
|
205
|
-
query: queryStmt,
|
|
206
|
-
formatter: formatFunc,
|
|
207
|
-
metadataJsonColumn: jsonColumnName,
|
|
208
|
-
});
|
|
209
|
-
}
|
|
210
|
-
async load() {
|
|
211
|
-
const documents = [];
|
|
212
|
-
for await (const doc of this.lazyLoad()) {
|
|
213
|
-
documents.push(doc);
|
|
214
|
-
}
|
|
215
|
-
return documents;
|
|
216
|
-
}
|
|
217
|
-
async *lazyLoad() {
|
|
218
|
-
const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn, } = this;
|
|
219
|
-
try {
|
|
220
|
-
if (!query) {
|
|
221
|
-
throw new Error("Query is undefined");
|
|
222
|
-
}
|
|
223
|
-
const result = await this.engine.pool.raw(query);
|
|
224
|
-
for (const row of result.rows) {
|
|
225
|
-
const rowData = {};
|
|
226
|
-
const columnNames = [
|
|
227
|
-
...(contentColumns || []),
|
|
228
|
-
...(metadataColumns || []),
|
|
229
|
-
];
|
|
230
|
-
if (metadataJsonColumn) {
|
|
231
|
-
columnNames.push(metadataJsonColumn);
|
|
232
|
-
}
|
|
233
|
-
columnNames.forEach((column) => {
|
|
234
|
-
rowData[column] = row[column];
|
|
235
|
-
});
|
|
236
|
-
yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
catch (error) {
|
|
240
|
-
if (typeof error === "string") {
|
|
241
|
-
throw Error(error);
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
22
|
+
* Google Cloud SQL for PostgreSQL vector store integration.
|
|
23
|
+
*
|
|
24
|
+
* Setup:
|
|
25
|
+
* Install`@langchain/google-cloud-sql-pg`
|
|
26
|
+
*
|
|
27
|
+
* ```bash
|
|
28
|
+
* npm install @langchain/google-cloud-sql-pg
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* <details open >
|
|
32
|
+
* <summary><strong>Use with Table Name < /strong></summary >
|
|
33
|
+
*
|
|
34
|
+
* ```typescript
|
|
35
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
36
|
+
*
|
|
37
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
38
|
+
* tableName: "test_table_custom",
|
|
39
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
40
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
41
|
+
* format: "text"
|
|
42
|
+
* };
|
|
43
|
+
*
|
|
44
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
45
|
+
*
|
|
46
|
+
* const documents = await documentLoaderInstance.load();
|
|
47
|
+
* ```
|
|
48
|
+
* </details>
|
|
49
|
+
*
|
|
50
|
+
* <br />
|
|
51
|
+
*
|
|
52
|
+
* <details open >
|
|
53
|
+
* <summary><strong>Use with Query < /strong></summary >
|
|
54
|
+
*
|
|
55
|
+
* ```typescript
|
|
56
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
57
|
+
*
|
|
58
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
59
|
+
* query: "SELECT * FROM my_table WHERE organic = true;",
|
|
60
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
61
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
62
|
+
* format: "text"
|
|
63
|
+
* };
|
|
64
|
+
*
|
|
65
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
66
|
+
*
|
|
67
|
+
* for await (const doc of documentLoaderInstance.lazyLoad()) {
|
|
68
|
+
* console.log(doc);
|
|
69
|
+
* break; // break based on required condition
|
|
70
|
+
* }
|
|
71
|
+
* ```
|
|
72
|
+
* </details>
|
|
73
|
+
*
|
|
74
|
+
* <br />
|
|
75
|
+
*/
|
|
76
|
+
var PostgresLoader = class PostgresLoader extends __langchain_core_document_loaders_base.BaseDocumentLoader {
|
|
77
|
+
engine;
|
|
78
|
+
tableName;
|
|
79
|
+
schemaName;
|
|
80
|
+
contentColumns;
|
|
81
|
+
metadataColumns;
|
|
82
|
+
format;
|
|
83
|
+
formatter;
|
|
84
|
+
query;
|
|
85
|
+
metadataJsonColumn;
|
|
86
|
+
constructor(engine, options) {
|
|
87
|
+
super();
|
|
88
|
+
this.engine = engine;
|
|
89
|
+
this.contentColumns = options.contentColumns;
|
|
90
|
+
this.metadataColumns = options.metadataColumns;
|
|
91
|
+
this.query = options.query;
|
|
92
|
+
this.formatter = options.formatter;
|
|
93
|
+
this.metadataJsonColumn = options.metadataJsonColumn;
|
|
94
|
+
}
|
|
95
|
+
static async initialize(engine, { schemaName = "public", tableName, contentColumns, metadataColumns, format, query, formatter, metadataJsonColumn }) {
|
|
96
|
+
if (tableName && query) throw new Error("Only one of 'table_name' or 'query' should be specified.");
|
|
97
|
+
if (!tableName && !query) throw new Error("At least one of the parameters 'table_name' or 'query' needs to be provided");
|
|
98
|
+
if (format && formatter) throw new Error("Only one of 'format' or 'formatter' should be specified.");
|
|
99
|
+
if (format && ![
|
|
100
|
+
"csv",
|
|
101
|
+
"text",
|
|
102
|
+
"json",
|
|
103
|
+
"yaml"
|
|
104
|
+
].includes(format)) throw new Error("format must be type: 'csv', 'text', 'json', 'yaml'");
|
|
105
|
+
let formatFunc = formatter;
|
|
106
|
+
if (formatFunc === void 0) if (format === "csv") formatFunc = require_utils.csvFormatter;
|
|
107
|
+
else if (format === "yaml") formatFunc = require_utils.yamlFormatter;
|
|
108
|
+
else if (format === "json") formatFunc = require_utils.jsonFormatter;
|
|
109
|
+
else formatFunc = require_utils.textFormatter;
|
|
110
|
+
let queryStmt = query;
|
|
111
|
+
if (!queryStmt) queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
|
|
112
|
+
let result;
|
|
113
|
+
try {
|
|
114
|
+
result = await engine.pool.raw(queryStmt);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
if (typeof error === "string") throw Error(error);
|
|
117
|
+
}
|
|
118
|
+
const columnNames = result.fields.map((field) => field.name);
|
|
119
|
+
const contentColumnNames = contentColumns || [columnNames[0]];
|
|
120
|
+
const metadataColumnNames = metadataColumns || columnNames.filter((col) => !contentColumnNames.includes(col));
|
|
121
|
+
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
122
|
+
let jsonColumnName = metadataJsonColumn;
|
|
123
|
+
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) jsonColumnName = DEFAULT_METADATA_COL;
|
|
124
|
+
const allNames = [...contentColumnNames || [], ...metadataColumnNames || []];
|
|
125
|
+
allNames.forEach((name) => {
|
|
126
|
+
if (!columnNames.includes(name)) throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
127
|
+
});
|
|
128
|
+
return new PostgresLoader(engine, {
|
|
129
|
+
contentColumns: contentColumnNames,
|
|
130
|
+
metadataColumns: metadataColumnNames,
|
|
131
|
+
query: queryStmt,
|
|
132
|
+
formatter: formatFunc,
|
|
133
|
+
metadataJsonColumn: jsonColumnName
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
async load() {
|
|
137
|
+
const documents = [];
|
|
138
|
+
for await (const doc of this.lazyLoad()) documents.push(doc);
|
|
139
|
+
return documents;
|
|
140
|
+
}
|
|
141
|
+
async *lazyLoad() {
|
|
142
|
+
const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn } = this;
|
|
143
|
+
try {
|
|
144
|
+
if (!query) throw new Error("Query is undefined");
|
|
145
|
+
const result = await this.engine.pool.raw(query);
|
|
146
|
+
for (const row of result.rows) {
|
|
147
|
+
const rowData = {};
|
|
148
|
+
const columnNames = [...contentColumns || [], ...metadataColumns || []];
|
|
149
|
+
if (metadataJsonColumn) columnNames.push(metadataJsonColumn);
|
|
150
|
+
columnNames.forEach((column) => {
|
|
151
|
+
rowData[column] = row[column];
|
|
152
|
+
});
|
|
153
|
+
yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
|
|
154
|
+
}
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (typeof error === "string") throw Error(error);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
//#endregion
|
|
246
162
|
exports.PostgresLoader = PostgresLoader;
|
|
163
|
+
//# sourceMappingURL=loader.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.cjs","names":["contentColumns: string[]","metadataColumns: string[]","row: Row","metadataJsonColumn: string | null","formatter: (row: Row, contentColumns: string[]) => string","textFormatter","metadata: { [key: string]: string }","BaseDocumentLoader","engine: PostgresEngine","options: PostgresLoaderOptions","csvFormatter","yamlFormatter","jsonFormatter","field: { name: string }","col: string","documents: Document[]","rowData: Row"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACPA,gBACAC,iBACAC,KACAC,qBAAoC,sBACpCC,YAA4DC,6BAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAMC,WAAsC,CAAE;AAE9C,KAAI,sBAAsB,IAAI,qBAC5B,OAAO,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,KAAK;EAC1D,SAAS,KAAK;CACf,EAAC;CAGJ,gBAAgB,QAAQ,CAAC,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,oBAC9B,SAAS,UAAU,IAAI;CAE1B,EAAC;AAEF,QAAO;EAAE;EAAa;CAAU;AACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDD,IAAa,iBAAb,MAAa,uBAAuBC,0DAAmB;CACrD,AAAQ;CAER;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAYC,QAAwBC,SAAgC;EAClE,OAAO;EACP,KAAK,SAAS;EACd,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,kBAAkB,QAAQ;EAC/B,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;EACzB,KAAK,qBAAqB,QAAQ;CACnC;CAED,aAAa,WACXD,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,oBACsB,EACC;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR;AAGJ,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR;AAGJ,MAAI,UAAU,UACZ,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;EAAO,EAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM;EAGlB,IAAI,aAAa;AACjB,MAAI,eAAe,OACjB,KAAI,WAAW,OACb,aAAaE;WACJ,WAAW,QACpB,aAAaC;WACJ,WAAW,QACpB,aAAaC;OAEb,aAAaP;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,WACH,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,EAAE,UAAU,CAAC,CAAC;EAG5D,IAAI;AACJ,MAAI;GACF,SAAS,MAAM,OAAO,KAAK,IAAI,UAAU;EAC1C,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;EACD,MAAM,cAAc,OAAO,OAAO,IAChC,CAACQ,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,EAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,OAAO,CAACC,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,mBAAmB,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG5E,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,EAC/D,iBAAiB;EAGnB,MAAM,WAAW,CACf,GAAI,sBAAsB,CAAE,GAC5B,GAAI,uBAAuB,CAAE,CAC9B;EACD,SAAS,QAAQ,CAAC,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,KAAK,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG/D,EAAC;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;EACrB;CACF;CAED,MAAM,OAA4B;EAChC,MAAMC,YAAwB,CAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,EACrC,UAAU,KAAK,IAAI;AAErB,SAAO;CACR;CAED,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,oBACD,GAAG;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM;GAElB,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAMC,UAAe,CAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,CAAE,GACxB,GAAI,mBAAmB,CAAE,CAC1B;AACD,QAAI,oBACF,YAAY,KAAK,mBAAmB;IAEtC,YAAY,QAAQ,CAAC,WAAW;KAC9B,QAAQ,UAAU,IAAI;IACvB,EAAC;IAEF,MAAM,gBACJ,kBAAkB,CAAE,GACpB,mBAAmB,CAAE,GACrB,SACA,oBACA,UACD;GACF;EACF,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;CACF;AACF"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { PostgresEngine } from "./engine.cjs";
|
|
2
|
+
import { Document } from "@langchain/core/documents";
|
|
3
|
+
import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
|
|
4
|
+
|
|
5
|
+
//#region src/loader.d.ts
|
|
6
|
+
type Row = {
|
|
7
|
+
[key: string]: string;
|
|
8
|
+
};
|
|
9
|
+
// Options for PostgresLoader
|
|
10
|
+
interface PostgresLoaderOptions {
|
|
11
|
+
tableName?: string;
|
|
12
|
+
schemaName?: string;
|
|
13
|
+
contentColumns?: string[];
|
|
14
|
+
metadataColumns?: string[];
|
|
15
|
+
format?: "text" | "json" | "yaml" | "csv";
|
|
16
|
+
formatter?: (row: Row, contentColumns: string[]) => string;
|
|
17
|
+
query?: string;
|
|
18
|
+
metadataJsonColumn?: string | null;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Google Cloud SQL for PostgreSQL vector store integration.
|
|
22
|
+
*
|
|
23
|
+
* Setup:
|
|
24
|
+
* Install`@langchain/google-cloud-sql-pg`
|
|
25
|
+
*
|
|
26
|
+
* ```bash
|
|
27
|
+
* npm install @langchain/google-cloud-sql-pg
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* <details open >
|
|
31
|
+
* <summary><strong>Use with Table Name < /strong></summary >
|
|
32
|
+
*
|
|
33
|
+
* ```typescript
|
|
34
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
35
|
+
*
|
|
36
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
37
|
+
* tableName: "test_table_custom",
|
|
38
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
39
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
40
|
+
* format: "text"
|
|
41
|
+
* };
|
|
42
|
+
*
|
|
43
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
44
|
+
*
|
|
45
|
+
* const documents = await documentLoaderInstance.load();
|
|
46
|
+
* ```
|
|
47
|
+
* </details>
|
|
48
|
+
*
|
|
49
|
+
* <br />
|
|
50
|
+
*
|
|
51
|
+
* <details open >
|
|
52
|
+
* <summary><strong>Use with Query < /strong></summary >
|
|
53
|
+
*
|
|
54
|
+
* ```typescript
|
|
55
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
56
|
+
*
|
|
57
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
58
|
+
* query: "SELECT * FROM my_table WHERE organic = true;",
|
|
59
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
60
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
61
|
+
* format: "text"
|
|
62
|
+
* };
|
|
63
|
+
*
|
|
64
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
65
|
+
*
|
|
66
|
+
* for await (const doc of documentLoaderInstance.lazyLoad()) {
|
|
67
|
+
* console.log(doc);
|
|
68
|
+
* break; // break based on required condition
|
|
69
|
+
* }
|
|
70
|
+
* ```
|
|
71
|
+
* </details>
|
|
72
|
+
*
|
|
73
|
+
* <br />
|
|
74
|
+
*/
|
|
75
|
+
declare class PostgresLoader extends BaseDocumentLoader {
|
|
76
|
+
private engine;
|
|
77
|
+
tableName?: string;
|
|
78
|
+
schemaName?: string;
|
|
79
|
+
contentColumns?: string[];
|
|
80
|
+
metadataColumns?: string[];
|
|
81
|
+
format?: "text" | "json" | "yaml" | "csv";
|
|
82
|
+
formatter?: (row: Row, contentColumns: string[]) => string;
|
|
83
|
+
query?: string;
|
|
84
|
+
metadataJsonColumn?: string | null;
|
|
85
|
+
constructor(engine: PostgresEngine, options: PostgresLoaderOptions);
|
|
86
|
+
static initialize(engine: PostgresEngine, {
|
|
87
|
+
schemaName,
|
|
88
|
+
tableName,
|
|
89
|
+
contentColumns,
|
|
90
|
+
metadataColumns,
|
|
91
|
+
format,
|
|
92
|
+
query,
|
|
93
|
+
formatter,
|
|
94
|
+
metadataJsonColumn
|
|
95
|
+
}: PostgresLoaderOptions): Promise<PostgresLoader>;
|
|
96
|
+
load(): Promise<Document[]>;
|
|
97
|
+
lazyLoad(): AsyncGenerator<Document>;
|
|
98
|
+
}
|
|
99
|
+
//#endregion
|
|
100
|
+
export { PostgresLoader, PostgresLoaderOptions };
|
|
101
|
+
//# sourceMappingURL=loader.d.cts.map
|
package/dist/loader.d.ts
CHANGED
|
@@ -1,18 +1,21 @@
|
|
|
1
|
+
import { PostgresEngine } from "./engine.js";
|
|
1
2
|
import { Document } from "@langchain/core/documents";
|
|
2
3
|
import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
|
|
3
|
-
|
|
4
|
+
|
|
5
|
+
//#region src/loader.d.ts
|
|
4
6
|
type Row = {
|
|
5
|
-
|
|
7
|
+
[key: string]: string;
|
|
6
8
|
};
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
// Options for PostgresLoader
|
|
10
|
+
interface PostgresLoaderOptions {
|
|
11
|
+
tableName?: string;
|
|
12
|
+
schemaName?: string;
|
|
13
|
+
contentColumns?: string[];
|
|
14
|
+
metadataColumns?: string[];
|
|
15
|
+
format?: "text" | "json" | "yaml" | "csv";
|
|
16
|
+
formatter?: (row: Row, contentColumns: string[]) => string;
|
|
17
|
+
query?: string;
|
|
18
|
+
metadataJsonColumn?: string | null;
|
|
16
19
|
}
|
|
17
20
|
/**
|
|
18
21
|
* Google Cloud SQL for PostgreSQL vector store integration.
|
|
@@ -69,19 +72,30 @@ export interface PostgresLoaderOptions {
|
|
|
69
72
|
*
|
|
70
73
|
* <br />
|
|
71
74
|
*/
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
75
|
+
declare class PostgresLoader extends BaseDocumentLoader {
|
|
76
|
+
private engine;
|
|
77
|
+
tableName?: string;
|
|
78
|
+
schemaName?: string;
|
|
79
|
+
contentColumns?: string[];
|
|
80
|
+
metadataColumns?: string[];
|
|
81
|
+
format?: "text" | "json" | "yaml" | "csv";
|
|
82
|
+
formatter?: (row: Row, contentColumns: string[]) => string;
|
|
83
|
+
query?: string;
|
|
84
|
+
metadataJsonColumn?: string | null;
|
|
85
|
+
constructor(engine: PostgresEngine, options: PostgresLoaderOptions);
|
|
86
|
+
static initialize(engine: PostgresEngine, {
|
|
87
|
+
schemaName,
|
|
88
|
+
tableName,
|
|
89
|
+
contentColumns,
|
|
90
|
+
metadataColumns,
|
|
91
|
+
format,
|
|
92
|
+
query,
|
|
93
|
+
formatter,
|
|
94
|
+
metadataJsonColumn
|
|
95
|
+
}: PostgresLoaderOptions): Promise<PostgresLoader>;
|
|
96
|
+
load(): Promise<Document[]>;
|
|
97
|
+
lazyLoad(): AsyncGenerator<Document>;
|
|
86
98
|
}
|
|
87
|
-
|
|
99
|
+
//#endregion
|
|
100
|
+
export { PostgresLoader, PostgresLoaderOptions };
|
|
101
|
+
//# sourceMappingURL=loader.d.ts.map
|