@langchain/google-cloud-sql-pg 0.0.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/LICENSE +6 -6
- package/README.md +1 -1
- package/dist/_virtual/rolldown_runtime.cjs +25 -0
- package/dist/chat_message_history.cjs +104 -137
- package/dist/chat_message_history.cjs.map +1 -0
- package/dist/chat_message_history.d.cts +57 -0
- package/dist/chat_message_history.d.cts.map +1 -0
- package/dist/chat_message_history.d.ts +53 -43
- package/dist/chat_message_history.d.ts.map +1 -0
- package/dist/chat_message_history.js +103 -133
- package/dist/chat_message_history.js.map +1 -0
- package/dist/engine.cjs +188 -242
- package/dist/engine.cjs.map +1 -0
- package/dist/engine.d.cts +138 -0
- package/dist/engine.d.cts.map +1 -0
- package/dist/engine.d.ts +102 -80
- package/dist/engine.d.ts.map +1 -0
- package/dist/engine.js +186 -234
- package/dist/engine.js.map +1 -0
- package/dist/index.cjs +21 -20
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +7 -4
- package/dist/indexes.cjs +96 -168
- package/dist/indexes.cjs.map +1 -0
- package/dist/indexes.d.cts +68 -0
- package/dist/indexes.d.cts.map +1 -0
- package/dist/indexes.d.ts +50 -47
- package/dist/indexes.d.ts.map +1 -0
- package/dist/indexes.js +90 -161
- package/dist/indexes.js.map +1 -0
- package/dist/loader.cjs +159 -242
- package/dist/loader.cjs.map +1 -0
- package/dist/loader.d.cts +101 -0
- package/dist/loader.d.cts.map +1 -0
- package/dist/loader.d.ts +40 -26
- package/dist/loader.d.ts.map +1 -0
- package/dist/loader.js +157 -237
- package/dist/loader.js.map +1 -0
- package/dist/utils/utils.cjs +36 -65
- package/dist/utils/utils.cjs.map +1 -0
- package/dist/utils/utils.js +36 -62
- package/dist/utils/utils.js.map +1 -0
- package/dist/vectorstore.cjs +438 -593
- package/dist/vectorstore.cjs.map +1 -0
- package/dist/vectorstore.d.cts +300 -0
- package/dist/vectorstore.d.cts.map +1 -0
- package/dist/vectorstore.d.ts +147 -130
- package/dist/vectorstore.d.ts.map +1 -0
- package/dist/vectorstore.js +436 -588
- package/dist/vectorstore.js.map +1 -0
- package/package.json +38 -47
- package/dist/utils/utils.d.ts +0 -22
- package/index.cjs +0 -1
- package/index.d.cts +0 -1
- package/index.d.ts +0 -1
- package/index.js +0 -1
package/dist/loader.js
CHANGED
|
@@ -1,242 +1,162 @@
|
|
|
1
|
+
import { csvFormatter, jsonFormatter, textFormatter, yamlFormatter } from "./utils/utils.js";
|
|
1
2
|
import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
|
|
2
|
-
|
|
3
|
+
|
|
4
|
+
//#region src/loader.ts
|
|
3
5
|
const DEFAULT_METADATA_COL = "langchain_metadata";
|
|
4
6
|
function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter = textFormatter) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
return { pageContent, metadata };
|
|
7
|
+
const pageContent = formatter(row, contentColumns);
|
|
8
|
+
const metadata = {};
|
|
9
|
+
if (metadataJsonColumn && row[metadataJsonColumn]) Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {
|
|
10
|
+
metadata[k] = v;
|
|
11
|
+
});
|
|
12
|
+
metadataColumns.forEach((column) => {
|
|
13
|
+
if (column in row && column !== metadataJsonColumn) metadata[column] = row[column];
|
|
14
|
+
});
|
|
15
|
+
return {
|
|
16
|
+
pageContent,
|
|
17
|
+
metadata
|
|
18
|
+
};
|
|
18
19
|
}
|
|
19
20
|
/**
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
else {
|
|
163
|
-
formatFunc = textFormatter;
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
let queryStmt = query;
|
|
167
|
-
if (!queryStmt) {
|
|
168
|
-
queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
|
|
169
|
-
}
|
|
170
|
-
let result;
|
|
171
|
-
try {
|
|
172
|
-
result = await engine.pool.raw(queryStmt);
|
|
173
|
-
}
|
|
174
|
-
catch (error) {
|
|
175
|
-
if (typeof error === "string") {
|
|
176
|
-
throw Error(error);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
const columnNames = result.fields.map((field) => field.name);
|
|
180
|
-
const contentColumnNames = contentColumns || [columnNames[0]];
|
|
181
|
-
const metadataColumnNames = metadataColumns ||
|
|
182
|
-
columnNames.filter((col) => !contentColumnNames.includes(col));
|
|
183
|
-
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {
|
|
184
|
-
throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
185
|
-
}
|
|
186
|
-
let jsonColumnName = metadataJsonColumn;
|
|
187
|
-
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {
|
|
188
|
-
jsonColumnName = DEFAULT_METADATA_COL;
|
|
189
|
-
}
|
|
190
|
-
const allNames = [
|
|
191
|
-
...(contentColumnNames || []),
|
|
192
|
-
...(metadataColumnNames || []),
|
|
193
|
-
];
|
|
194
|
-
allNames.forEach((name) => {
|
|
195
|
-
if (!columnNames.includes(name)) {
|
|
196
|
-
throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
197
|
-
}
|
|
198
|
-
});
|
|
199
|
-
return new PostgresLoader(engine, {
|
|
200
|
-
contentColumns: contentColumnNames,
|
|
201
|
-
metadataColumns: metadataColumnNames,
|
|
202
|
-
query: queryStmt,
|
|
203
|
-
formatter: formatFunc,
|
|
204
|
-
metadataJsonColumn: jsonColumnName,
|
|
205
|
-
});
|
|
206
|
-
}
|
|
207
|
-
async load() {
|
|
208
|
-
const documents = [];
|
|
209
|
-
for await (const doc of this.lazyLoad()) {
|
|
210
|
-
documents.push(doc);
|
|
211
|
-
}
|
|
212
|
-
return documents;
|
|
213
|
-
}
|
|
214
|
-
async *lazyLoad() {
|
|
215
|
-
const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn, } = this;
|
|
216
|
-
try {
|
|
217
|
-
if (!query) {
|
|
218
|
-
throw new Error("Query is undefined");
|
|
219
|
-
}
|
|
220
|
-
const result = await this.engine.pool.raw(query);
|
|
221
|
-
for (const row of result.rows) {
|
|
222
|
-
const rowData = {};
|
|
223
|
-
const columnNames = [
|
|
224
|
-
...(contentColumns || []),
|
|
225
|
-
...(metadataColumns || []),
|
|
226
|
-
];
|
|
227
|
-
if (metadataJsonColumn) {
|
|
228
|
-
columnNames.push(metadataJsonColumn);
|
|
229
|
-
}
|
|
230
|
-
columnNames.forEach((column) => {
|
|
231
|
-
rowData[column] = row[column];
|
|
232
|
-
});
|
|
233
|
-
yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
catch (error) {
|
|
237
|
-
if (typeof error === "string") {
|
|
238
|
-
throw Error(error);
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
}
|
|
21
|
+
* Google Cloud SQL for PostgreSQL vector store integration.
|
|
22
|
+
*
|
|
23
|
+
* Setup:
|
|
24
|
+
* Install`@langchain/google-cloud-sql-pg`
|
|
25
|
+
*
|
|
26
|
+
* ```bash
|
|
27
|
+
* npm install @langchain/google-cloud-sql-pg
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* <details open >
|
|
31
|
+
* <summary><strong>Use with Table Name < /strong></summary >
|
|
32
|
+
*
|
|
33
|
+
* ```typescript
|
|
34
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
35
|
+
*
|
|
36
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
37
|
+
* tableName: "test_table_custom",
|
|
38
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
39
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
40
|
+
* format: "text"
|
|
41
|
+
* };
|
|
42
|
+
*
|
|
43
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
44
|
+
*
|
|
45
|
+
* const documents = await documentLoaderInstance.load();
|
|
46
|
+
* ```
|
|
47
|
+
* </details>
|
|
48
|
+
*
|
|
49
|
+
* <br />
|
|
50
|
+
*
|
|
51
|
+
* <details open >
|
|
52
|
+
* <summary><strong>Use with Query < /strong></summary >
|
|
53
|
+
*
|
|
54
|
+
* ```typescript
|
|
55
|
+
* import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
|
|
56
|
+
*
|
|
57
|
+
* const documentLoaderArgs: PostgresLoaderOptions = {
|
|
58
|
+
* query: "SELECT * FROM my_table WHERE organic = true;",
|
|
59
|
+
* contentColumns: [ "fruit_name", "variety"],
|
|
60
|
+
* metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
|
|
61
|
+
* format: "text"
|
|
62
|
+
* };
|
|
63
|
+
*
|
|
64
|
+
* const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
|
|
65
|
+
*
|
|
66
|
+
* for await (const doc of documentLoaderInstance.lazyLoad()) {
|
|
67
|
+
* console.log(doc);
|
|
68
|
+
* break; // break based on required condition
|
|
69
|
+
* }
|
|
70
|
+
* ```
|
|
71
|
+
* </details>
|
|
72
|
+
*
|
|
73
|
+
* <br />
|
|
74
|
+
*/
|
|
75
|
+
var PostgresLoader = class PostgresLoader extends BaseDocumentLoader {
|
|
76
|
+
engine;
|
|
77
|
+
tableName;
|
|
78
|
+
schemaName;
|
|
79
|
+
contentColumns;
|
|
80
|
+
metadataColumns;
|
|
81
|
+
format;
|
|
82
|
+
formatter;
|
|
83
|
+
query;
|
|
84
|
+
metadataJsonColumn;
|
|
85
|
+
constructor(engine, options) {
|
|
86
|
+
super();
|
|
87
|
+
this.engine = engine;
|
|
88
|
+
this.contentColumns = options.contentColumns;
|
|
89
|
+
this.metadataColumns = options.metadataColumns;
|
|
90
|
+
this.query = options.query;
|
|
91
|
+
this.formatter = options.formatter;
|
|
92
|
+
this.metadataJsonColumn = options.metadataJsonColumn;
|
|
93
|
+
}
|
|
94
|
+
static async initialize(engine, { schemaName = "public", tableName, contentColumns, metadataColumns, format, query, formatter, metadataJsonColumn }) {
|
|
95
|
+
if (tableName && query) throw new Error("Only one of 'table_name' or 'query' should be specified.");
|
|
96
|
+
if (!tableName && !query) throw new Error("At least one of the parameters 'table_name' or 'query' needs to be provided");
|
|
97
|
+
if (format && formatter) throw new Error("Only one of 'format' or 'formatter' should be specified.");
|
|
98
|
+
if (format && ![
|
|
99
|
+
"csv",
|
|
100
|
+
"text",
|
|
101
|
+
"json",
|
|
102
|
+
"yaml"
|
|
103
|
+
].includes(format)) throw new Error("format must be type: 'csv', 'text', 'json', 'yaml'");
|
|
104
|
+
let formatFunc = formatter;
|
|
105
|
+
if (formatFunc === void 0) if (format === "csv") formatFunc = csvFormatter;
|
|
106
|
+
else if (format === "yaml") formatFunc = yamlFormatter;
|
|
107
|
+
else if (format === "json") formatFunc = jsonFormatter;
|
|
108
|
+
else formatFunc = textFormatter;
|
|
109
|
+
let queryStmt = query;
|
|
110
|
+
if (!queryStmt) queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
|
|
111
|
+
let result;
|
|
112
|
+
try {
|
|
113
|
+
result = await engine.pool.raw(queryStmt);
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (typeof error === "string") throw Error(error);
|
|
116
|
+
}
|
|
117
|
+
const columnNames = result.fields.map((field) => field.name);
|
|
118
|
+
const contentColumnNames = contentColumns || [columnNames[0]];
|
|
119
|
+
const metadataColumnNames = metadataColumns || columnNames.filter((col) => !contentColumnNames.includes(col));
|
|
120
|
+
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
121
|
+
let jsonColumnName = metadataJsonColumn;
|
|
122
|
+
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) jsonColumnName = DEFAULT_METADATA_COL;
|
|
123
|
+
const allNames = [...contentColumnNames || [], ...metadataColumnNames || []];
|
|
124
|
+
allNames.forEach((name) => {
|
|
125
|
+
if (!columnNames.includes(name)) throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
126
|
+
});
|
|
127
|
+
return new PostgresLoader(engine, {
|
|
128
|
+
contentColumns: contentColumnNames,
|
|
129
|
+
metadataColumns: metadataColumnNames,
|
|
130
|
+
query: queryStmt,
|
|
131
|
+
formatter: formatFunc,
|
|
132
|
+
metadataJsonColumn: jsonColumnName
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
async load() {
|
|
136
|
+
const documents = [];
|
|
137
|
+
for await (const doc of this.lazyLoad()) documents.push(doc);
|
|
138
|
+
return documents;
|
|
139
|
+
}
|
|
140
|
+
async *lazyLoad() {
|
|
141
|
+
const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn } = this;
|
|
142
|
+
try {
|
|
143
|
+
if (!query) throw new Error("Query is undefined");
|
|
144
|
+
const result = await this.engine.pool.raw(query);
|
|
145
|
+
for (const row of result.rows) {
|
|
146
|
+
const rowData = {};
|
|
147
|
+
const columnNames = [...contentColumns || [], ...metadataColumns || []];
|
|
148
|
+
if (metadataJsonColumn) columnNames.push(metadataJsonColumn);
|
|
149
|
+
columnNames.forEach((column) => {
|
|
150
|
+
rowData[column] = row[column];
|
|
151
|
+
});
|
|
152
|
+
yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
|
|
153
|
+
}
|
|
154
|
+
} catch (error) {
|
|
155
|
+
if (typeof error === "string") throw Error(error);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
//#endregion
|
|
161
|
+
export { PostgresLoader };
|
|
162
|
+
//# sourceMappingURL=loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.js","names":["contentColumns: string[]","metadataColumns: string[]","row: Row","metadataJsonColumn: string | null","formatter: (row: Row, contentColumns: string[]) => string","metadata: { [key: string]: string }","engine: PostgresEngine","options: PostgresLoaderOptions","field: { name: string }","col: string","documents: Document[]","rowData: Row"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACPA,gBACAC,iBACAC,KACAC,qBAAoC,sBACpCC,YAA4D,eAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAMC,WAAsC,CAAE;AAE9C,KAAI,sBAAsB,IAAI,qBAC5B,OAAO,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,KAAK;EAC1D,SAAS,KAAK;CACf,EAAC;CAGJ,gBAAgB,QAAQ,CAAC,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,oBAC9B,SAAS,UAAU,IAAI;CAE1B,EAAC;AAEF,QAAO;EAAE;EAAa;CAAU;AACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDD,IAAa,iBAAb,MAAa,uBAAuB,mBAAmB;CACrD,AAAQ;CAER;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAYC,QAAwBC,SAAgC;EAClE,OAAO;EACP,KAAK,SAAS;EACd,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,kBAAkB,QAAQ;EAC/B,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;EACzB,KAAK,qBAAqB,QAAQ;CACnC;CAED,aAAa,WACXD,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,oBACsB,EACC;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR;AAGJ,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR;AAGJ,MAAI,UAAU,UACZ,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;EAAO,EAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM;EAGlB,IAAI,aAAa;AACjB,MAAI,eAAe,OACjB,KAAI,WAAW,OACb,aAAa;WACJ,WAAW,QACpB,aAAa;WACJ,WAAW,QACpB,aAAa;OAEb,aAAa;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,WACH,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,EAAE,UAAU,CAAC,CAAC;EAG5D,IAAI;AACJ,MAAI;GACF,SAAS,MAAM,OAAO,KAAK,IAAI,UAAU;EAC1C,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;EACD,MAAM,cAAc,OAAO,OAAO,IAChC,CAACE,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,EAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,OAAO,CAACC,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,mBAAmB,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG5E,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,EAC/D,iBAAiB;EAGnB,MAAM,WAAW,CACf,GAAI,sBAAsB,CAAE,GAC5B,GAAI,uBAAuB,CAAE,CAC9B;EACD,SAAS,QAAQ,CAAC,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,KAAK,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG/D,EAAC;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;EACrB;CACF;CAED,MAAM,OAA4B;EAChC,MAAMC,YAAwB,CAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,EACrC,UAAU,KAAK,IAAI;AAErB,SAAO;CACR;CAED,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,oBACD,GAAG;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM;GAElB,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAMC,UAAe,CAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,CAAE,GACxB,GAAI,mBAAmB,CAAE,CAC1B;AACD,QAAI,oBACF,YAAY,KAAK,mBAAmB;IAEtC,YAAY,QAAQ,CAAC,WAAW;KAC9B,QAAQ,UAAU,IAAI;IACvB,EAAC;IAEF,MAAM,gBACJ,kBAAkB,CAAE,GACpB,mBAAmB,CAAE,GACrB,SACA,oBACA,UACD;GACF;EACF,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;CACF;AACF"}
|
package/dist/utils/utils.cjs
CHANGED
|
@@ -1,78 +1,49 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.customZip = exports.getIAMPrincipalEmail = void 0;
|
|
4
|
-
exports.textFormatter = textFormatter;
|
|
5
|
-
exports.csvFormatter = csvFormatter;
|
|
6
|
-
exports.yamlFormatter = yamlFormatter;
|
|
7
|
-
exports.jsonFormatter = jsonFormatter;
|
|
1
|
+
|
|
2
|
+
//#region src/utils/utils.ts
|
|
8
3
|
/**
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
4
|
+
* Get email address associated with current authenticated IAM principal.
|
|
5
|
+
* Email will be used for automatic IAM database authentication to Cloud SQL.
|
|
6
|
+
*
|
|
7
|
+
* @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.
|
|
8
|
+
* @returns {string} email - email address associated with the current authenticated IAM principal
|
|
9
|
+
*/
|
|
15
10
|
const getIAMPrincipalEmail = async (auth) => {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
26
|
-
.then((res) => res.data);
|
|
27
|
-
if (!("email" in clientResponse)) {
|
|
28
|
-
throw new Error("Failed to automatically obtain authenticated IAM principal's " +
|
|
29
|
-
"email address using environment's ADC credentials!");
|
|
30
|
-
}
|
|
31
|
-
const { email } = clientResponse;
|
|
32
|
-
return email.replace(".gserviceaccount.com", "");
|
|
11
|
+
const credentials = await auth.getCredentials();
|
|
12
|
+
if ("client_email" in credentials && credentials.client_email !== void 0) return credentials.client_email.replace(".gserviceaccount.com", "");
|
|
13
|
+
const accessToken = await auth.getAccessToken();
|
|
14
|
+
const client = await auth.getClient();
|
|
15
|
+
const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;
|
|
16
|
+
const clientResponse = await client.request({ url }).then((res) => res.data);
|
|
17
|
+
if (!("email" in clientResponse)) throw new Error("Failed to automatically obtain authenticated IAM principal's email address using environment's ADC credentials!");
|
|
18
|
+
const { email } = clientResponse;
|
|
19
|
+
return email.replace(".gserviceaccount.com", "");
|
|
33
20
|
};
|
|
34
|
-
exports.getIAMPrincipalEmail = getIAMPrincipalEmail;
|
|
35
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
36
21
|
const customZip = (...arrays) => {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}
|
|
42
|
-
return result;
|
|
22
|
+
const minLength = Math.min(...arrays.map((arr) => arr.length));
|
|
23
|
+
const result = [];
|
|
24
|
+
for (let i = 0; i < minLength; i += 1) result.push(arrays.map((arr) => arr[i]));
|
|
25
|
+
return result;
|
|
43
26
|
};
|
|
44
|
-
exports.customZip = customZip;
|
|
45
|
-
/*
|
|
46
|
-
Formatter functions
|
|
47
|
-
*/
|
|
48
|
-
// txt document formatter.
|
|
49
27
|
function textFormatter(row, content_columns) {
|
|
50
|
-
|
|
51
|
-
.filter((column) => column in row)
|
|
52
|
-
.map((column) => String(row[column]))
|
|
53
|
-
.join(" ");
|
|
28
|
+
return content_columns.filter((column) => column in row).map((column) => String(row[column])).join(" ");
|
|
54
29
|
}
|
|
55
|
-
// CSV document formatter.
|
|
56
30
|
function csvFormatter(row, content_columns) {
|
|
57
|
-
|
|
58
|
-
.filter((column) => column in row)
|
|
59
|
-
.map((column) => String(row[column]))
|
|
60
|
-
.join(", ");
|
|
31
|
+
return content_columns.filter((column) => column in row).map((column) => String(row[column])).join(", ");
|
|
61
32
|
}
|
|
62
|
-
// YAML document formatter
|
|
63
33
|
function yamlFormatter(row, content_columns) {
|
|
64
|
-
|
|
65
|
-
.filter((column) => column in row)
|
|
66
|
-
.map((column) => `${column}: ${String(row[column])}`)
|
|
67
|
-
.join("\n");
|
|
34
|
+
return content_columns.filter((column) => column in row).map((column) => `${column}: ${String(row[column])}`).join("\n");
|
|
68
35
|
}
|
|
69
|
-
// JSON document formatter
|
|
70
36
|
function jsonFormatter(row, content_columns) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
dictionary[column] = row[column];
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return JSON.stringify(dictionary);
|
|
37
|
+
const dictionary = {};
|
|
38
|
+
for (const column of content_columns) if (column in row) dictionary[column] = row[column];
|
|
39
|
+
return JSON.stringify(dictionary);
|
|
78
40
|
}
|
|
41
|
+
|
|
42
|
+
//#endregion
|
|
43
|
+
exports.csvFormatter = csvFormatter;
|
|
44
|
+
exports.customZip = customZip;
|
|
45
|
+
exports.getIAMPrincipalEmail = getIAMPrincipalEmail;
|
|
46
|
+
exports.jsonFormatter = jsonFormatter;
|
|
47
|
+
exports.textFormatter = textFormatter;
|
|
48
|
+
exports.yamlFormatter = yamlFormatter;
|
|
49
|
+
//# sourceMappingURL=utils.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.cjs","names":["auth: GoogleAuth","res: { data: any }","row: { [key: string]: string }","content_columns: string[]","dictionary: { [key: string]: string }"],"sources":["../../src/utils/utils.ts"],"sourcesContent":["import { GoogleAuth } from \"google-auth-library\";\n\n/**\n * Get email address associated with current authenticated IAM principal.\n * Email will be used for automatic IAM database authentication to Cloud SQL.\n *\n * @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.\n * @returns {string} email - email address associated with the current authenticated IAM principal\n */\nexport const getIAMPrincipalEmail = async (\n auth: GoogleAuth\n): Promise<string> => {\n const credentials = await auth.getCredentials();\n\n if (\"client_email\" in credentials && credentials.client_email !== undefined) {\n return credentials.client_email.replace(\".gserviceaccount.com\", \"\");\n }\n\n const accessToken = await auth.getAccessToken();\n const client = await auth.getClient();\n\n const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;\n const clientResponse = await client\n .request({ url })\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n .then((res: { data: any }) => res.data);\n\n if (!(\"email\" in clientResponse)) {\n throw new Error(\n \"Failed to automatically obtain authenticated IAM principal's \" +\n \"email address using environment's ADC credentials!\"\n );\n }\n const { email } = clientResponse;\n return email.replace(\".gserviceaccount.com\", \"\");\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const customZip = (...arrays: any[]) => {\n const minLength = Math.min(...arrays.map((arr) => arr.length));\n const result = [];\n for (let i = 0; i < minLength; i += 1) {\n result.push(arrays.map((arr) => arr[i]));\n }\n return result;\n};\n\n/*\n Formatter functions\n*/\n\n// txt document formatter.\nexport function textFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\" \");\n}\n\n// CSV document formatter.\nexport function csvFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\", \");\n}\n\n// YAML document formatter\nexport function yamlFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => `${column}: ${String(row[column])}`)\n .join(\"\\n\");\n}\n\n// JSON document formatter\nexport function jsonFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n const dictionary: { [key: string]: string } = {};\n for (const column of content_columns) {\n if (column in row) {\n dictionary[column] = row[column];\n }\n }\n return JSON.stringify(dictionary);\n}\n"],"mappings":";;;;;;;;;AASA,MAAa,uBAAuB,OAClCA,SACoB;CACpB,MAAM,cAAc,MAAM,KAAK,gBAAgB;AAE/C,KAAI,kBAAkB,eAAe,YAAY,iBAAiB,OAChE,QAAO,YAAY,aAAa,QAAQ,wBAAwB,GAAG;CAGrE,MAAM,cAAc,MAAM,KAAK,gBAAgB;CAC/C,MAAM,SAAS,MAAM,KAAK,WAAW;CAErC,MAAM,MAAM,CAAC,qDAAqD,EAAE,aAAa;CACjF,MAAM,iBAAiB,MAAM,OAC1B,QAAQ,EAAE,IAAK,EAAC,CAEhB,KAAK,CAACC,QAAuB,IAAI,KAAK;AAEzC,KAAI,EAAE,WAAW,gBACf,OAAM,IAAI,MACR;CAIJ,MAAM,EAAE,OAAO,GAAG;AAClB,QAAO,MAAM,QAAQ,wBAAwB,GAAG;AACjD;AAGD,MAAa,YAAY,CAAC,GAAG,WAAkB;CAC7C,MAAM,YAAY,KAAK,IAAI,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;CAC9D,MAAM,SAAS,CAAE;AACjB,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK,GAClC,OAAO,KAAK,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAE1C,QAAO;AACR;AAOD,SAAgB,cACdC,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,IAAI;AACb;AAGD,SAAgB,aACdD,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,KAAK;AACd;AAGD,SAAgB,cACdD,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,EAAE,OAAO,IAAI,QAAQ,EAAE,CAAC,CACpD,KAAK,KAAK;AACd;AAGD,SAAgB,cACdD,KACAC,iBACQ;CACR,MAAMC,aAAwC,CAAE;AAChD,MAAK,MAAM,UAAU,gBACnB,KAAI,UAAU,KACZ,WAAW,UAAU,IAAI;AAG7B,QAAO,KAAK,UAAU,WAAW;AAClC"}
|