@langchain/google-cloud-sql-pg 0.0.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/LICENSE +6 -6
  3. package/README.md +1 -1
  4. package/dist/_virtual/rolldown_runtime.cjs +25 -0
  5. package/dist/chat_message_history.cjs +104 -137
  6. package/dist/chat_message_history.cjs.map +1 -0
  7. package/dist/chat_message_history.d.cts +57 -0
  8. package/dist/chat_message_history.d.cts.map +1 -0
  9. package/dist/chat_message_history.d.ts +53 -43
  10. package/dist/chat_message_history.d.ts.map +1 -0
  11. package/dist/chat_message_history.js +103 -133
  12. package/dist/chat_message_history.js.map +1 -0
  13. package/dist/engine.cjs +188 -242
  14. package/dist/engine.cjs.map +1 -0
  15. package/dist/engine.d.cts +138 -0
  16. package/dist/engine.d.cts.map +1 -0
  17. package/dist/engine.d.ts +102 -80
  18. package/dist/engine.d.ts.map +1 -0
  19. package/dist/engine.js +186 -234
  20. package/dist/engine.js.map +1 -0
  21. package/dist/index.cjs +21 -20
  22. package/dist/index.d.cts +6 -0
  23. package/dist/index.d.ts +6 -4
  24. package/dist/index.js +7 -4
  25. package/dist/indexes.cjs +96 -168
  26. package/dist/indexes.cjs.map +1 -0
  27. package/dist/indexes.d.cts +68 -0
  28. package/dist/indexes.d.cts.map +1 -0
  29. package/dist/indexes.d.ts +50 -47
  30. package/dist/indexes.d.ts.map +1 -0
  31. package/dist/indexes.js +90 -161
  32. package/dist/indexes.js.map +1 -0
  33. package/dist/loader.cjs +159 -242
  34. package/dist/loader.cjs.map +1 -0
  35. package/dist/loader.d.cts +101 -0
  36. package/dist/loader.d.cts.map +1 -0
  37. package/dist/loader.d.ts +40 -26
  38. package/dist/loader.d.ts.map +1 -0
  39. package/dist/loader.js +157 -237
  40. package/dist/loader.js.map +1 -0
  41. package/dist/utils/utils.cjs +36 -65
  42. package/dist/utils/utils.cjs.map +1 -0
  43. package/dist/utils/utils.js +36 -62
  44. package/dist/utils/utils.js.map +1 -0
  45. package/dist/vectorstore.cjs +438 -593
  46. package/dist/vectorstore.cjs.map +1 -0
  47. package/dist/vectorstore.d.cts +300 -0
  48. package/dist/vectorstore.d.cts.map +1 -0
  49. package/dist/vectorstore.d.ts +147 -130
  50. package/dist/vectorstore.d.ts.map +1 -0
  51. package/dist/vectorstore.js +436 -588
  52. package/dist/vectorstore.js.map +1 -0
  53. package/package.json +38 -47
  54. package/dist/utils/utils.d.ts +0 -22
  55. package/index.cjs +0 -1
  56. package/index.d.cts +0 -1
  57. package/index.d.ts +0 -1
  58. package/index.js +0 -1
package/dist/loader.js CHANGED
@@ -1,242 +1,162 @@
1
+ import { csvFormatter, jsonFormatter, textFormatter, yamlFormatter } from "./utils/utils.js";
1
2
  import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
2
- import { textFormatter, csvFormatter, yamlFormatter, jsonFormatter, } from "./utils/utils.js";
3
+
4
+ //#region src/loader.ts
3
5
  const DEFAULT_METADATA_COL = "langchain_metadata";
4
6
  function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter = textFormatter) {
5
- const pageContent = formatter(row, contentColumns);
6
- const metadata = {};
7
- if (metadataJsonColumn && row[metadataJsonColumn]) {
8
- Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {
9
- metadata[k] = v;
10
- });
11
- }
12
- metadataColumns.forEach((column) => {
13
- if (column in row && column !== metadataJsonColumn) {
14
- metadata[column] = row[column];
15
- }
16
- });
17
- return { pageContent, metadata };
7
+ const pageContent = formatter(row, contentColumns);
8
+ const metadata = {};
9
+ if (metadataJsonColumn && row[metadataJsonColumn]) Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {
10
+ metadata[k] = v;
11
+ });
12
+ metadataColumns.forEach((column) => {
13
+ if (column in row && column !== metadataJsonColumn) metadata[column] = row[column];
14
+ });
15
+ return {
16
+ pageContent,
17
+ metadata
18
+ };
18
19
  }
19
20
  /**
20
- * Google Cloud SQL for PostgreSQL vector store integration.
21
- *
22
- * Setup:
23
- * Install`@langchain/google-cloud-sql-pg`
24
- *
25
- * ```bash
26
- * npm install @langchain/google-cloud-sql-pg
27
- * ```
28
- *
29
- * <details open >
30
- * <summary><strong>Use with Table Name < /strong></summary >
31
- *
32
- * ```typescript
33
- * import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
34
- *
35
- * const documentLoaderArgs: PostgresLoaderOptions = {
36
- * tableName: "test_table_custom",
37
- * contentColumns: [ "fruit_name", "variety"],
38
- * metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
39
- * format: "text"
40
- * };
41
- *
42
- * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
43
- *
44
- * const documents = await documentLoaderInstance.load();
45
- * ```
46
- * </details>
47
- *
48
- * <br />
49
- *
50
- * <details open >
51
- * <summary><strong>Use with Query < /strong></summary >
52
- *
53
- * ```typescript
54
- * import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
55
- *
56
- * const documentLoaderArgs: PostgresLoaderOptions = {
57
- * query: "SELECT * FROM my_table WHERE organic = true;",
58
- * contentColumns: [ "fruit_name", "variety"],
59
- * metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
60
- * format: "text"
61
- * };
62
- *
63
- * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
64
- *
65
- * for await (const doc of documentLoaderInstance.lazyLoad()) {
66
- * console.log(doc);
67
- * break; // break based on required condition
68
- * }
69
- * ```
70
- * </details>
71
- *
72
- * <br />
73
- */
74
- export class PostgresLoader extends BaseDocumentLoader {
75
- constructor(engine, options) {
76
- super();
77
- Object.defineProperty(this, "engine", {
78
- enumerable: true,
79
- configurable: true,
80
- writable: true,
81
- value: void 0
82
- });
83
- Object.defineProperty(this, "tableName", {
84
- enumerable: true,
85
- configurable: true,
86
- writable: true,
87
- value: void 0
88
- });
89
- Object.defineProperty(this, "schemaName", {
90
- enumerable: true,
91
- configurable: true,
92
- writable: true,
93
- value: void 0
94
- });
95
- Object.defineProperty(this, "contentColumns", {
96
- enumerable: true,
97
- configurable: true,
98
- writable: true,
99
- value: void 0
100
- });
101
- Object.defineProperty(this, "metadataColumns", {
102
- enumerable: true,
103
- configurable: true,
104
- writable: true,
105
- value: void 0
106
- });
107
- Object.defineProperty(this, "format", {
108
- enumerable: true,
109
- configurable: true,
110
- writable: true,
111
- value: void 0
112
- });
113
- Object.defineProperty(this, "formatter", {
114
- enumerable: true,
115
- configurable: true,
116
- writable: true,
117
- value: void 0
118
- });
119
- Object.defineProperty(this, "query", {
120
- enumerable: true,
121
- configurable: true,
122
- writable: true,
123
- value: void 0
124
- });
125
- Object.defineProperty(this, "metadataJsonColumn", {
126
- enumerable: true,
127
- configurable: true,
128
- writable: true,
129
- value: void 0
130
- });
131
- this.engine = engine;
132
- this.contentColumns = options.contentColumns;
133
- this.metadataColumns = options.metadataColumns;
134
- this.query = options.query;
135
- this.formatter = options.formatter;
136
- this.metadataJsonColumn = options.metadataJsonColumn;
137
- }
138
- static async initialize(engine, { schemaName = "public", tableName, contentColumns, metadataColumns, format, query, formatter, metadataJsonColumn, }) {
139
- if (tableName && query) {
140
- throw new Error("Only one of 'table_name' or 'query' should be specified.");
141
- }
142
- if (!tableName && !query) {
143
- throw new Error("At least one of the parameters 'table_name' or 'query' needs to be provided");
144
- }
145
- if (format && formatter) {
146
- throw new Error("Only one of 'format' or 'formatter' should be specified.");
147
- }
148
- if (format && !["csv", "text", "json", "yaml"].includes(format)) {
149
- throw new Error("format must be type: 'csv', 'text', 'json', 'yaml'");
150
- }
151
- let formatFunc = formatter;
152
- if (formatFunc === undefined) {
153
- if (format === "csv") {
154
- formatFunc = csvFormatter;
155
- }
156
- else if (format === "yaml") {
157
- formatFunc = yamlFormatter;
158
- }
159
- else if (format === "json") {
160
- formatFunc = jsonFormatter;
161
- }
162
- else {
163
- formatFunc = textFormatter;
164
- }
165
- }
166
- let queryStmt = query;
167
- if (!queryStmt) {
168
- queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
169
- }
170
- let result;
171
- try {
172
- result = await engine.pool.raw(queryStmt);
173
- }
174
- catch (error) {
175
- if (typeof error === "string") {
176
- throw Error(error);
177
- }
178
- }
179
- const columnNames = result.fields.map((field) => field.name);
180
- const contentColumnNames = contentColumns || [columnNames[0]];
181
- const metadataColumnNames = metadataColumns ||
182
- columnNames.filter((col) => !contentColumnNames.includes(col));
183
- if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {
184
- throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
185
- }
186
- let jsonColumnName = metadataJsonColumn;
187
- if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {
188
- jsonColumnName = DEFAULT_METADATA_COL;
189
- }
190
- const allNames = [
191
- ...(contentColumnNames || []),
192
- ...(metadataColumnNames || []),
193
- ];
194
- allNames.forEach((name) => {
195
- if (!columnNames.includes(name)) {
196
- throw new Error(`Column ${name} not found in query result ${columnNames}.`);
197
- }
198
- });
199
- return new PostgresLoader(engine, {
200
- contentColumns: contentColumnNames,
201
- metadataColumns: metadataColumnNames,
202
- query: queryStmt,
203
- formatter: formatFunc,
204
- metadataJsonColumn: jsonColumnName,
205
- });
206
- }
207
- async load() {
208
- const documents = [];
209
- for await (const doc of this.lazyLoad()) {
210
- documents.push(doc);
211
- }
212
- return documents;
213
- }
214
- async *lazyLoad() {
215
- const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn, } = this;
216
- try {
217
- if (!query) {
218
- throw new Error("Query is undefined");
219
- }
220
- const result = await this.engine.pool.raw(query);
221
- for (const row of result.rows) {
222
- const rowData = {};
223
- const columnNames = [
224
- ...(contentColumns || []),
225
- ...(metadataColumns || []),
226
- ];
227
- if (metadataJsonColumn) {
228
- columnNames.push(metadataJsonColumn);
229
- }
230
- columnNames.forEach((column) => {
231
- rowData[column] = row[column];
232
- });
233
- yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
234
- }
235
- }
236
- catch (error) {
237
- if (typeof error === "string") {
238
- throw Error(error);
239
- }
240
- }
241
- }
242
- }
21
+ * Google Cloud SQL for PostgreSQL vector store integration.
22
+ *
23
+ * Setup:
24
+ * Install`@langchain/google-cloud-sql-pg`
25
+ *
26
+ * ```bash
27
+ * npm install @langchain/google-cloud-sql-pg
28
+ * ```
29
+ *
30
+ * <details open >
31
+ * <summary><strong>Use with Table Name < /strong></summary >
32
+ *
33
+ * ```typescript
34
+ * import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
35
+ *
36
+ * const documentLoaderArgs: PostgresLoaderOptions = {
37
+ * tableName: "test_table_custom",
38
+ * contentColumns: [ "fruit_name", "variety"],
39
+ * metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
40
+ * format: "text"
41
+ * };
42
+ *
43
+ * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
44
+ *
45
+ * const documents = await documentLoaderInstance.load();
46
+ * ```
47
+ * </details>
48
+ *
49
+ * <br />
50
+ *
51
+ * <details open >
52
+ * <summary><strong>Use with Query < /strong></summary >
53
+ *
54
+ * ```typescript
55
+ * import { PostgresEngine, PostgresLoader } from "@langchain/google-cloud-sql-pg";
56
+ *
57
+ * const documentLoaderArgs: PostgresLoaderOptions = {
58
+ * query: "SELECT * FROM my_table WHERE organic = true;",
59
+ * contentColumns: [ "fruit_name", "variety"],
60
+ * metadataColumns: ["fruit_id", "quantity_in_stock", "price_per_unit", "organic"],
61
+ * format: "text"
62
+ * };
63
+ *
64
+ * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);
65
+ *
66
+ * for await (const doc of documentLoaderInstance.lazyLoad()) {
67
+ * console.log(doc);
68
+ * break; // break based on required condition
69
+ * }
70
+ * ```
71
+ * </details>
72
+ *
73
+ * <br />
74
+ */
75
+ var PostgresLoader = class PostgresLoader extends BaseDocumentLoader {
76
+ engine;
77
+ tableName;
78
+ schemaName;
79
+ contentColumns;
80
+ metadataColumns;
81
+ format;
82
+ formatter;
83
+ query;
84
+ metadataJsonColumn;
85
+ constructor(engine, options) {
86
+ super();
87
+ this.engine = engine;
88
+ this.contentColumns = options.contentColumns;
89
+ this.metadataColumns = options.metadataColumns;
90
+ this.query = options.query;
91
+ this.formatter = options.formatter;
92
+ this.metadataJsonColumn = options.metadataJsonColumn;
93
+ }
94
+ static async initialize(engine, { schemaName = "public", tableName, contentColumns, metadataColumns, format, query, formatter, metadataJsonColumn }) {
95
+ if (tableName && query) throw new Error("Only one of 'table_name' or 'query' should be specified.");
96
+ if (!tableName && !query) throw new Error("At least one of the parameters 'table_name' or 'query' needs to be provided");
97
+ if (format && formatter) throw new Error("Only one of 'format' or 'formatter' should be specified.");
98
+ if (format && ![
99
+ "csv",
100
+ "text",
101
+ "json",
102
+ "yaml"
103
+ ].includes(format)) throw new Error("format must be type: 'csv', 'text', 'json', 'yaml'");
104
+ let formatFunc = formatter;
105
+ if (formatFunc === void 0) if (format === "csv") formatFunc = csvFormatter;
106
+ else if (format === "yaml") formatFunc = yamlFormatter;
107
+ else if (format === "json") formatFunc = jsonFormatter;
108
+ else formatFunc = textFormatter;
109
+ let queryStmt = query;
110
+ if (!queryStmt) queryStmt = `SELECT * FROM "${schemaName}"."${tableName}"`;
111
+ let result;
112
+ try {
113
+ result = await engine.pool.raw(queryStmt);
114
+ } catch (error) {
115
+ if (typeof error === "string") throw Error(error);
116
+ }
117
+ const columnNames = result.fields.map((field) => field.name);
118
+ const contentColumnNames = contentColumns || [columnNames[0]];
119
+ const metadataColumnNames = metadataColumns || columnNames.filter((col) => !contentColumnNames.includes(col));
120
+ if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
121
+ let jsonColumnName = metadataJsonColumn;
122
+ if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) jsonColumnName = DEFAULT_METADATA_COL;
123
+ const allNames = [...contentColumnNames || [], ...metadataColumnNames || []];
124
+ allNames.forEach((name) => {
125
+ if (!columnNames.includes(name)) throw new Error(`Column ${name} not found in query result ${columnNames}.`);
126
+ });
127
+ return new PostgresLoader(engine, {
128
+ contentColumns: contentColumnNames,
129
+ metadataColumns: metadataColumnNames,
130
+ query: queryStmt,
131
+ formatter: formatFunc,
132
+ metadataJsonColumn: jsonColumnName
133
+ });
134
+ }
135
+ async load() {
136
+ const documents = [];
137
+ for await (const doc of this.lazyLoad()) documents.push(doc);
138
+ return documents;
139
+ }
140
+ async *lazyLoad() {
141
+ const { query, contentColumns, metadataColumns, formatter, metadataJsonColumn } = this;
142
+ try {
143
+ if (!query) throw new Error("Query is undefined");
144
+ const result = await this.engine.pool.raw(query);
145
+ for (const row of result.rows) {
146
+ const rowData = {};
147
+ const columnNames = [...contentColumns || [], ...metadataColumns || []];
148
+ if (metadataJsonColumn) columnNames.push(metadataJsonColumn);
149
+ columnNames.forEach((column) => {
150
+ rowData[column] = row[column];
151
+ });
152
+ yield parseDocFromRow(contentColumns || [], metadataColumns || [], rowData, metadataJsonColumn, formatter);
153
+ }
154
+ } catch (error) {
155
+ if (typeof error === "string") throw Error(error);
156
+ }
157
+ }
158
+ };
159
+
160
+ //#endregion
161
+ export { PostgresLoader };
162
+ //# sourceMappingURL=loader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loader.js","names":["contentColumns: string[]","metadataColumns: string[]","row: Row","metadataJsonColumn: string | null","formatter: (row: Row, contentColumns: string[]) => string","metadata: { [key: string]: string }","engine: PostgresEngine","options: PostgresLoaderOptions","field: { name: string }","col: string","documents: Document[]","rowData: Row"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACPA,gBACAC,iBACAC,KACAC,qBAAoC,sBACpCC,YAA4D,eAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAMC,WAAsC,CAAE;AAE9C,KAAI,sBAAsB,IAAI,qBAC5B,OAAO,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,KAAK;EAC1D,SAAS,KAAK;CACf,EAAC;CAGJ,gBAAgB,QAAQ,CAAC,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,oBAC9B,SAAS,UAAU,IAAI;CAE1B,EAAC;AAEF,QAAO;EAAE;EAAa;CAAU;AACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDD,IAAa,iBAAb,MAAa,uBAAuB,mBAAmB;CACrD,AAAQ;CAER;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAYC,QAAwBC,SAAgC;EAClE,OAAO;EACP,KAAK,SAAS;EACd,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,kBAAkB,QAAQ;EAC/B,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;EACzB,KAAK,qBAAqB,QAAQ;CACnC;CAED,aAAa,WACXD,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,oBACsB,EACC;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR;AAGJ,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR;AAGJ,MAAI,UAAU,UACZ,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;EAAO,EAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM;EAGlB,IAAI,aAAa;AACjB,MAAI,eAAe,OACjB,KAAI,WAAW,OACb,aAAa;WACJ,WAAW,QACpB,aAAa;WACJ,WAAW,QACpB,aAAa;OAEb,aAAa;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,WACH,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,EAAE,UAAU,CAAC,CAAC;EAG5D,IAAI;AACJ,MAAI;GACF,SAAS,MAAM,OAAO,KAAK,IAAI,UAAU;EAC1C,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;EACD,MAAM,cAAc,OAAO,OAAO,IAChC,CAACE,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,EAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,OAAO,CAACC,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,mBAAmB,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG5E,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,EAC/D,iBAAiB;EAGnB,MAAM,WAAW,CACf,GAAI,sBAAsB,CAAE,GAC5B,GAAI,uBAAuB,CAAE,CAC9B;EACD,SAAS,QAAQ,CAAC,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,KAAK,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG/D,EAAC;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;EACrB;CACF;CAED,MAAM,OAA4B;EAChC,MAAMC,YAAwB,CAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,EACrC,UAAU,KAAK,IAAI;AAErB,SAAO;CACR;CAED,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,oBACD,GAAG;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM;GAElB,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAMC,UAAe,CAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,CAAE,GACxB,GAAI,mBAAmB,CAAE,CAC1B;AACD,QAAI,oBACF,YAAY,KAAK,mBAAmB;IAEtC,YAAY,QAAQ,CAAC,WAAW;KAC9B,QAAQ,UAAU,IAAI;IACvB,EAAC;IAEF,MAAM,gBACJ,kBAAkB,CAAE,GACpB,mBAAmB,CAAE,GACrB,SACA,oBACA,UACD;GACF;EACF,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;CACF;AACF"}
@@ -1,78 +1,49 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.customZip = exports.getIAMPrincipalEmail = void 0;
4
- exports.textFormatter = textFormatter;
5
- exports.csvFormatter = csvFormatter;
6
- exports.yamlFormatter = yamlFormatter;
7
- exports.jsonFormatter = jsonFormatter;
1
+
2
+ //#region src/utils/utils.ts
8
3
  /**
9
- * Get email address associated with current authenticated IAM principal.
10
- * Email will be used for automatic IAM database authentication to Cloud SQL.
11
- *
12
- * @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.
13
- * @returns {string} email - email address associated with the current authenticated IAM principal
14
- */
4
+ * Get email address associated with current authenticated IAM principal.
5
+ * Email will be used for automatic IAM database authentication to Cloud SQL.
6
+ *
7
+ * @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.
8
+ * @returns {string} email - email address associated with the current authenticated IAM principal
9
+ */
15
10
  const getIAMPrincipalEmail = async (auth) => {
16
- const credentials = await auth.getCredentials();
17
- if ("client_email" in credentials && credentials.client_email !== undefined) {
18
- return credentials.client_email.replace(".gserviceaccount.com", "");
19
- }
20
- const accessToken = await auth.getAccessToken();
21
- const client = await auth.getClient();
22
- const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;
23
- const clientResponse = await client
24
- .request({ url })
25
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
26
- .then((res) => res.data);
27
- if (!("email" in clientResponse)) {
28
- throw new Error("Failed to automatically obtain authenticated IAM principal's " +
29
- "email address using environment's ADC credentials!");
30
- }
31
- const { email } = clientResponse;
32
- return email.replace(".gserviceaccount.com", "");
11
+ const credentials = await auth.getCredentials();
12
+ if ("client_email" in credentials && credentials.client_email !== void 0) return credentials.client_email.replace(".gserviceaccount.com", "");
13
+ const accessToken = await auth.getAccessToken();
14
+ const client = await auth.getClient();
15
+ const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;
16
+ const clientResponse = await client.request({ url }).then((res) => res.data);
17
+ if (!("email" in clientResponse)) throw new Error("Failed to automatically obtain authenticated IAM principal's email address using environment's ADC credentials!");
18
+ const { email } = clientResponse;
19
+ return email.replace(".gserviceaccount.com", "");
33
20
  };
34
- exports.getIAMPrincipalEmail = getIAMPrincipalEmail;
35
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
36
21
  const customZip = (...arrays) => {
37
- const minLength = Math.min(...arrays.map((arr) => arr.length));
38
- const result = [];
39
- for (let i = 0; i < minLength; i += 1) {
40
- result.push(arrays.map((arr) => arr[i]));
41
- }
42
- return result;
22
+ const minLength = Math.min(...arrays.map((arr) => arr.length));
23
+ const result = [];
24
+ for (let i = 0; i < minLength; i += 1) result.push(arrays.map((arr) => arr[i]));
25
+ return result;
43
26
  };
44
- exports.customZip = customZip;
45
- /*
46
- Formatter functions
47
- */
48
- // txt document formatter.
49
27
  function textFormatter(row, content_columns) {
50
- return content_columns
51
- .filter((column) => column in row)
52
- .map((column) => String(row[column]))
53
- .join(" ");
28
+ return content_columns.filter((column) => column in row).map((column) => String(row[column])).join(" ");
54
29
  }
55
- // CSV document formatter.
56
30
  function csvFormatter(row, content_columns) {
57
- return content_columns
58
- .filter((column) => column in row)
59
- .map((column) => String(row[column]))
60
- .join(", ");
31
+ return content_columns.filter((column) => column in row).map((column) => String(row[column])).join(", ");
61
32
  }
62
- // YAML document formatter
63
33
  function yamlFormatter(row, content_columns) {
64
- return content_columns
65
- .filter((column) => column in row)
66
- .map((column) => `${column}: ${String(row[column])}`)
67
- .join("\n");
34
+ return content_columns.filter((column) => column in row).map((column) => `${column}: ${String(row[column])}`).join("\n");
68
35
  }
69
- // JSON document formatter
70
36
  function jsonFormatter(row, content_columns) {
71
- const dictionary = {};
72
- for (const column of content_columns) {
73
- if (column in row) {
74
- dictionary[column] = row[column];
75
- }
76
- }
77
- return JSON.stringify(dictionary);
37
+ const dictionary = {};
38
+ for (const column of content_columns) if (column in row) dictionary[column] = row[column];
39
+ return JSON.stringify(dictionary);
78
40
  }
41
+
42
+ //#endregion
43
+ exports.csvFormatter = csvFormatter;
44
+ exports.customZip = customZip;
45
+ exports.getIAMPrincipalEmail = getIAMPrincipalEmail;
46
+ exports.jsonFormatter = jsonFormatter;
47
+ exports.textFormatter = textFormatter;
48
+ exports.yamlFormatter = yamlFormatter;
49
+ //# sourceMappingURL=utils.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.cjs","names":["auth: GoogleAuth","res: { data: any }","row: { [key: string]: string }","content_columns: string[]","dictionary: { [key: string]: string }"],"sources":["../../src/utils/utils.ts"],"sourcesContent":["import { GoogleAuth } from \"google-auth-library\";\n\n/**\n * Get email address associated with current authenticated IAM principal.\n * Email will be used for automatic IAM database authentication to Cloud SQL.\n *\n * @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.\n * @returns {string} email - email address associated with the current authenticated IAM principal\n */\nexport const getIAMPrincipalEmail = async (\n auth: GoogleAuth\n): Promise<string> => {\n const credentials = await auth.getCredentials();\n\n if (\"client_email\" in credentials && credentials.client_email !== undefined) {\n return credentials.client_email.replace(\".gserviceaccount.com\", \"\");\n }\n\n const accessToken = await auth.getAccessToken();\n const client = await auth.getClient();\n\n const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;\n const clientResponse = await client\n .request({ url })\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n .then((res: { data: any }) => res.data);\n\n if (!(\"email\" in clientResponse)) {\n throw new Error(\n \"Failed to automatically obtain authenticated IAM principal's \" +\n \"email address using environment's ADC credentials!\"\n );\n }\n const { email } = clientResponse;\n return email.replace(\".gserviceaccount.com\", \"\");\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const customZip = (...arrays: any[]) => {\n const minLength = Math.min(...arrays.map((arr) => arr.length));\n const result = [];\n for (let i = 0; i < minLength; i += 1) {\n result.push(arrays.map((arr) => arr[i]));\n }\n return result;\n};\n\n/*\n Formatter functions\n*/\n\n// txt document formatter.\nexport function textFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\" \");\n}\n\n// CSV document formatter.\nexport function csvFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\", \");\n}\n\n// YAML document formatter\nexport function yamlFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => `${column}: ${String(row[column])}`)\n .join(\"\\n\");\n}\n\n// JSON document formatter\nexport function jsonFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n const dictionary: { [key: string]: string } = {};\n for (const column of content_columns) {\n if (column in row) {\n dictionary[column] = row[column];\n }\n }\n return JSON.stringify(dictionary);\n}\n"],"mappings":";;;;;;;;;AASA,MAAa,uBAAuB,OAClCA,SACoB;CACpB,MAAM,cAAc,MAAM,KAAK,gBAAgB;AAE/C,KAAI,kBAAkB,eAAe,YAAY,iBAAiB,OAChE,QAAO,YAAY,aAAa,QAAQ,wBAAwB,GAAG;CAGrE,MAAM,cAAc,MAAM,KAAK,gBAAgB;CAC/C,MAAM,SAAS,MAAM,KAAK,WAAW;CAErC,MAAM,MAAM,CAAC,qDAAqD,EAAE,aAAa;CACjF,MAAM,iBAAiB,MAAM,OAC1B,QAAQ,EAAE,IAAK,EAAC,CAEhB,KAAK,CAACC,QAAuB,IAAI,KAAK;AAEzC,KAAI,EAAE,WAAW,gBACf,OAAM,IAAI,MACR;CAIJ,MAAM,EAAE,OAAO,GAAG;AAClB,QAAO,MAAM,QAAQ,wBAAwB,GAAG;AACjD;AAGD,MAAa,YAAY,CAAC,GAAG,WAAkB;CAC7C,MAAM,YAAY,KAAK,IAAI,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;CAC9D,MAAM,SAAS,CAAE;AACjB,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK,GAClC,OAAO,KAAK,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAE1C,QAAO;AACR;AAOD,SAAgB,cACdC,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,IAAI;AACb;AAGD,SAAgB,aACdD,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,KAAK;AACd;AAGD,SAAgB,cACdD,KACAC,iBACQ;AACR,QAAO,gBACJ,OAAO,CAAC,WAAW,UAAU,IAAI,CACjC,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,EAAE,OAAO,IAAI,QAAQ,EAAE,CAAC,CACpD,KAAK,KAAK;AACd;AAGD,SAAgB,cACdD,KACAC,iBACQ;CACR,MAAMC,aAAwC,CAAE;AAChD,MAAK,MAAM,UAAU,gBACnB,KAAI,UAAU,KACZ,WAAW,UAAU,IAAI;AAG7B,QAAO,KAAK,UAAU,WAAW;AAClC"}