@langchain/google-cloud-sql-pg 1.0.22 → 1.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +37 -23
- package/dist/_virtual/{rolldown_runtime.cjs → _rolldown/runtime.cjs} +2 -4
- package/dist/chat_message_history.cjs +9 -11
- package/dist/chat_message_history.cjs.map +1 -1
- package/dist/chat_message_history.d.cts.map +1 -1
- package/dist/chat_message_history.d.ts.map +1 -1
- package/dist/chat_message_history.js +2 -4
- package/dist/chat_message_history.js.map +1 -1
- package/dist/engine.cjs +23 -32
- package/dist/engine.cjs.map +1 -1
- package/dist/engine.d.cts.map +1 -1
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +15 -25
- package/dist/engine.js.map +1 -1
- package/dist/index.cjs +9 -9
- package/dist/index.js +3 -4
- package/dist/indexes.cjs +1 -2
- package/dist/indexes.cjs.map +1 -1
- package/dist/indexes.d.cts.map +1 -1
- package/dist/indexes.d.ts.map +1 -1
- package/dist/indexes.js +1 -1
- package/dist/indexes.js.map +1 -1
- package/dist/loader.cjs +6 -8
- package/dist/loader.cjs.map +1 -1
- package/dist/loader.d.cts.map +1 -1
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +2 -4
- package/dist/loader.js.map +1 -1
- package/dist/utils/utils.cjs +1 -2
- package/dist/utils/utils.cjs.map +1 -1
- package/dist/utils/utils.js +1 -1
- package/dist/utils/utils.js.map +1 -1
- package/dist/vectorstore.cjs +24 -32
- package/dist/vectorstore.cjs.map +1 -1
- package/dist/vectorstore.d.cts.map +1 -1
- package/dist/vectorstore.d.ts.map +1 -1
- package/dist/vectorstore.js +13 -21
- package/dist/vectorstore.js.map +1 -1
- package/package.json +15 -25
package/dist/engine.js
CHANGED
|
@@ -2,7 +2,6 @@ import { getIAMPrincipalEmail } from "./utils/utils.js";
|
|
|
2
2
|
import { AuthTypes, Connector, IpAddressTypes } from "@google-cloud/cloud-sql-connector";
|
|
3
3
|
import { GoogleAuth } from "google-auth-library";
|
|
4
4
|
import knex from "knex";
|
|
5
|
-
|
|
6
5
|
//#region src/engine.ts
|
|
7
6
|
var Column = class {
|
|
8
7
|
name;
|
|
@@ -79,27 +78,22 @@ var PostgresEngine = class PostgresEngine {
|
|
|
79
78
|
} else {
|
|
80
79
|
enableIAMAuth = true;
|
|
81
80
|
if (iamAccountEmail !== void 0) dbUser = iamAccountEmail;
|
|
82
|
-
else {
|
|
83
|
-
const auth = new GoogleAuth({ scopes: "https://www.googleapis.com/auth/cloud-platform" });
|
|
84
|
-
dbUser = await getIAMPrincipalEmail(auth);
|
|
85
|
-
}
|
|
81
|
+
else dbUser = await getIAMPrincipalEmail(new GoogleAuth({ scopes: "https://www.googleapis.com/auth/cloud-platform" }));
|
|
86
82
|
}
|
|
87
83
|
PostgresEngine.connector = new Connector({ userAgent: USER_AGENT });
|
|
88
|
-
const
|
|
89
|
-
instanceConnectionName: `${projectId}:${region}:${instance}`,
|
|
90
|
-
ipType,
|
|
91
|
-
authType: enableIAMAuth ? AuthTypes.IAM : AuthTypes.PASSWORD
|
|
92
|
-
});
|
|
93
|
-
const dbConfig = {
|
|
84
|
+
const engine = knex({
|
|
94
85
|
client: "pg",
|
|
95
86
|
connection: {
|
|
96
|
-
...
|
|
87
|
+
...await PostgresEngine.connector.getOptions({
|
|
88
|
+
instanceConnectionName: `${projectId}:${region}:${instance}`,
|
|
89
|
+
ipType,
|
|
90
|
+
authType: enableIAMAuth ? AuthTypes.IAM : AuthTypes.PASSWORD
|
|
91
|
+
}),
|
|
97
92
|
...password ? { password } : {},
|
|
98
93
|
user: dbUser,
|
|
99
94
|
database
|
|
100
95
|
}
|
|
101
|
-
};
|
|
102
|
-
const engine = knex(dbConfig);
|
|
96
|
+
});
|
|
103
97
|
return new PostgresEngine(PostgresEngine._createKey, engine);
|
|
104
98
|
}
|
|
105
99
|
/**
|
|
@@ -119,9 +113,8 @@ var PostgresEngine = class PostgresEngine {
|
|
|
119
113
|
* @returns PostgresEngine instance
|
|
120
114
|
*/
|
|
121
115
|
static async fromConnectionString(url, poolConfig) {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
const dbConfig = {
|
|
116
|
+
if (typeof url === "string" && !url.startsWith("postgresql+asyncpg")) throw Error("Driver must be type 'postgresql+asyncpg'");
|
|
117
|
+
const engine = knex({
|
|
125
118
|
client: "pg",
|
|
126
119
|
connection: url,
|
|
127
120
|
acquireConnectionTimeout: 1e6,
|
|
@@ -129,8 +122,7 @@ var PostgresEngine = class PostgresEngine {
|
|
|
129
122
|
...poolConfig,
|
|
130
123
|
acquireTimeoutMillis: 6e5
|
|
131
124
|
}
|
|
132
|
-
};
|
|
133
|
-
const engine = knex(dbConfig);
|
|
125
|
+
});
|
|
134
126
|
return new PostgresEngine(PostgresEngine._createKey, engine);
|
|
135
127
|
}
|
|
136
128
|
/**
|
|
@@ -152,9 +144,8 @@ var PostgresEngine = class PostgresEngine {
|
|
|
152
144
|
await this.pool.raw("CREATE EXTENSION IF NOT EXISTS vector");
|
|
153
145
|
if (overwriteExisting) await this.pool.schema.withSchema(schemaName).dropTableIfExists(tableName);
|
|
154
146
|
const idDataType = typeof idColumn === "string" ? "UUID" : idColumn.dataType;
|
|
155
|
-
const idColumnName = typeof idColumn === "string" ? idColumn : idColumn.name;
|
|
156
147
|
let query = `CREATE TABLE ${schemaName}.${tableName}(
|
|
157
|
-
${
|
|
148
|
+
${typeof idColumn === "string" ? idColumn : idColumn.name} ${idDataType} PRIMARY KEY,
|
|
158
149
|
${contentColumn} TEXT NOT NULL,
|
|
159
150
|
${embeddingColumn} ${embeddingColumnType}(${vectorSize}) NOT NULL`;
|
|
160
151
|
for (const column of metadataColumns) {
|
|
@@ -186,11 +177,10 @@ var PostgresEngine = class PostgresEngine {
|
|
|
186
177
|
if (PostgresEngine.connector !== void 0) PostgresEngine.connector.close();
|
|
187
178
|
}
|
|
188
179
|
testConnection() {
|
|
189
|
-
|
|
190
|
-
return now;
|
|
180
|
+
return this.pool.raw("SELECT NOW() as currentTimestamp");
|
|
191
181
|
}
|
|
192
182
|
};
|
|
193
|
-
|
|
194
183
|
//#endregion
|
|
195
|
-
export { Column, PostgresEngine };
|
|
184
|
+
export { Column, PostgresEngine as default };
|
|
185
|
+
|
|
196
186
|
//# sourceMappingURL=engine.js.map
|
package/dist/engine.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine.js","names":["name: string","dataType: string","nullable: boolean","key: symbol","pool: knex.Knex","projectId: string","region: string","instance: string","database: string","dbUser: string","enableIAMAuth: boolean","dbConfig: knex.Knex.Config","engine: knex.Knex","url: string | knex.Knex.StaticConnectionConfig","poolConfig?: knex.Knex.PoolConfig","tableName: string","vectorSize: number","schemaName: string"],"sources":["../src/engine.ts"],"sourcesContent":["import {\n AuthTypes,\n Connector,\n IpAddressTypes,\n} from \"@google-cloud/cloud-sql-connector\";\nimport { GoogleAuth } from \"google-auth-library\";\nimport knex from \"knex\";\nimport { getIAMPrincipalEmail } from \"./utils/utils.js\";\n\nexport interface PostgresEngineArgs {\n ipType?: IpAddressTypes;\n user?: string;\n password?: string;\n iamAccountEmail?: string;\n}\n\nexport interface VectorStoreTableArgs {\n schemaName?: string;\n contentColumn?: string;\n embeddingColumn?: string;\n embeddingColumnType?: \"vector\" | \"halfvec\" | \"bit\" | \"sparsevec\";\n metadataColumns?: Column[];\n metadataJsonColumn?: string;\n idColumn?: string | Column;\n overwriteExisting?: boolean;\n storeMetadata?: boolean;\n}\n\nexport class Column {\n name: string;\n\n dataType: string;\n\n nullable: boolean;\n\n constructor(name: string, dataType: string, nullable: boolean = true) {\n this.name = name;\n this.dataType = dataType;\n this.nullable = nullable;\n\n this.postInitilization();\n }\n\n private postInitilization() {\n if (typeof this.name !== \"string\") {\n throw Error(\"Column name must be type string\");\n }\n\n if (typeof this.dataType !== \"string\") {\n throw Error(\"Column data_type must be type string\");\n }\n }\n}\n\nconst USER_AGENT = \"langchain-google-cloud-sql-pg-js\";\n\n/**\n * Cloud SQL shared connection pool\n *\n * Setup:\n * Install `@langchain/google-cloud-sql-pg`\n *\n * <details open>\n * <summary><strong>Instantiate</strong></summary>\n *\n * ```typescript\n * import { Column, PostgresEngine, PostgresEngineArgs } from \"@langchain/google-cloud-sql-pg\";\n *\n * const pgArgs: PostgresEngineArgs = {\n * user: \"db-user\",\n * password: \"password\"\n *}\n *\n * const engine: PostgresEngine = await PostgresEngine.fromInstance(\n * \"project-id\",\n * \"region\",\n * \"instance-name\",\n * \"database-name\",\n * pgArgs\n * );\n * ```\n * </details>\n *\n * <br />\n *\n */\nexport class PostgresEngine {\n private static _createKey = Symbol(\"key\");\n\n pool: knex.Knex;\n\n static connector: Connector;\n\n constructor(key: symbol, pool: knex.Knex) {\n if (key !== PostgresEngine._createKey) {\n throw Error(\"Only create class through 'create' method!\");\n }\n this.pool = pool;\n }\n\n /**\n * @param projectId Required - GCP Project ID\n * @param region Required - Postgres Instance Region\n * @param instance Required - Postgres Instance name\n * @param database Required - Database name\n * @param ipType Optional - IP address type. Defaults to IPAddressType.PUBLIC\n * @param user Optional - Postgres user name. Defaults to undefined\n * @param password Optional - Postgres user password. Defaults to undefined\n * @param iamAccountEmail Optional - IAM service account email. Defaults to undefined\n * @returns PostgresEngine instance\n */\n\n static async fromInstance(\n projectId: string,\n region: string,\n instance: string,\n database: string,\n {\n ipType = IpAddressTypes.PUBLIC,\n user,\n password,\n iamAccountEmail,\n }: PostgresEngineArgs = {}\n ): Promise<PostgresEngine> {\n let dbUser: string;\n let enableIAMAuth: boolean;\n\n if ((!user && password) || (user && !password)) {\n // XOR for strings\n throw Error(\n \"Only one of 'user' or 'password' were specified. Either \" +\n \"both should be specified to use basic user/password \" +\n \"authentication or neither for IAM DB authentication.\"\n );\n }\n\n // User and password are given so we use the basic auth\n if (user !== undefined && password !== undefined) {\n enableIAMAuth = false;\n dbUser = user!;\n } else {\n enableIAMAuth = true;\n if (iamAccountEmail !== undefined) {\n dbUser = iamAccountEmail;\n } else {\n // Get application default credentials\n const auth = new GoogleAuth({\n scopes: \"https://www.googleapis.com/auth/cloud-platform\",\n });\n // dbUser should be the iam principal email by passing the credentials obtained\n dbUser = await getIAMPrincipalEmail(auth);\n }\n }\n\n PostgresEngine.connector = new Connector({ userAgent: USER_AGENT });\n const clientOpts = await PostgresEngine.connector.getOptions({\n instanceConnectionName: `${projectId}:${region}:${instance}`,\n ipType,\n authType: enableIAMAuth ? AuthTypes.IAM : AuthTypes.PASSWORD,\n });\n\n const dbConfig: knex.Knex.Config = {\n client: \"pg\",\n connection: {\n ...clientOpts,\n ...(password ? { password } : {}),\n user: dbUser,\n database,\n },\n };\n\n const engine = knex(dbConfig);\n\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a PostgresEngine instance from an Knex instance.\n *\n * @param engine knex instance\n * @returns PostgresEngine instance from a knex instance\n */\n static async fromPool(engine: knex.Knex) {\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a PostgresEngine instance from arguments.\n *\n * @param url URL use to connect to a database\n * @param poolConfig Optional - Configuration pool to use in the Knex configuration\n * @returns PostgresEngine instance\n */\n static async fromConnectionString(\n url: string | knex.Knex.StaticConnectionConfig,\n poolConfig?: knex.Knex.PoolConfig\n ) {\n const driver = \"postgresql+asyncpg\";\n\n if (typeof url === \"string\" && !url.startsWith(driver)) {\n throw Error(\"Driver must be type 'postgresql+asyncpg'\");\n }\n\n const dbConfig: knex.Knex.Config = {\n client: \"pg\",\n connection: url,\n acquireConnectionTimeout: 1000000,\n pool: {\n ...poolConfig,\n acquireTimeoutMillis: 600000,\n },\n };\n\n const engine = knex(dbConfig);\n\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a table for saving of vectors to be used with PostgresVectorStore.\n *\n * @param tableName Postgres database table name. Parameter is not escaped. Do not use with end user input.\n * @param vectorSize Vector size for the embedding model to be used.\n * @param schemaName The schema name to store Postgres database table. Default: \"public\". Parameter is not escaped. Do not use with end user input.\n * @param contentColumn Name of the column to store document content. Default: \"content\".\n * @param embeddingColumn Name of the column to store vector embeddings. Default: \"embedding\".\n * @param embeddingColumnType Type of the embedding column (\"vector\" | \"halfvec\" | \"bit\" | \"sparsevec\"). Default: \"vector\". More info on HNSW-supported types: https://github.com/pgvector/pgvector#hnsw\n * @param metadataColumns Optional - A list of Columns to create for custom metadata. Default: [].\n * @param metadataJsonColumn Optional - The column to store extra metadata in JSON format. Default: \"langchain_metadata\".\n * @param idColumn Optional - Column to store ids. Default: \"langchain_id\" column name with data type UUID.\n * @param overwriteExisting Whether to drop existing table. Default: False.\n * @param storeMetadata Whether to store metadata in the table. Default: True.\n */\n async initVectorstoreTable(\n tableName: string,\n vectorSize: number,\n {\n schemaName = \"public\",\n contentColumn = \"content\",\n embeddingColumn = \"embedding\",\n embeddingColumnType = \"vector\",\n metadataColumns = [],\n metadataJsonColumn = \"langchain_metadata\",\n idColumn = \"langchain_id\",\n overwriteExisting = false,\n storeMetadata = true,\n }: VectorStoreTableArgs = {}\n ): Promise<void> {\n await this.pool.raw(\"CREATE EXTENSION IF NOT EXISTS vector\");\n\n if (overwriteExisting) {\n await this.pool.schema\n .withSchema(schemaName)\n .dropTableIfExists(tableName);\n }\n\n const idDataType =\n typeof idColumn === \"string\" ? \"UUID\" : idColumn.dataType;\n const idColumnName =\n typeof idColumn === \"string\" ? idColumn : idColumn.name;\n\n let query = `CREATE TABLE ${schemaName}.${tableName}(\n ${idColumnName} ${idDataType} PRIMARY KEY,\n ${contentColumn} TEXT NOT NULL,\n ${embeddingColumn} ${embeddingColumnType}(${vectorSize}) NOT NULL`;\n\n for (const column of metadataColumns) {\n const nullable = !column.nullable ? \"NOT NULL\" : \"\";\n query += `,\\n ${column.name} ${column.dataType} ${nullable}`;\n }\n\n if (storeMetadata) {\n query += `,\\n${metadataJsonColumn} JSON`;\n }\n\n query += `\\n);`;\n\n await this.pool.raw(query);\n }\n\n /**\n * Create a Cloud SQL table to store chat history.\n *\n * @param tableName Table name to store chat history\n * @param schemaName Schema name to store chat history table\n */\n\n async initChatHistoryTable(\n tableName: string,\n schemaName: string = \"public\"\n ): Promise<void> {\n await this.pool.raw(\n `CREATE TABLE IF NOT EXISTS ${schemaName}.${tableName}(\n id SERIAL PRIMARY KEY,\n session_id TEXT NOT NULL,\n data JSONB NOT NULL,\n type TEXT NOT NULL);`\n );\n }\n\n /**\n * Dispose of connection pool\n */\n async closeConnection(): Promise<void> {\n await this.pool.destroy();\n if (PostgresEngine.connector !== undefined) {\n PostgresEngine.connector.close();\n }\n }\n\n // Just to test the connection to the database\n testConnection() {\n const now = this.pool.raw(\"SELECT NOW() as currentTimestamp\");\n return now;\n }\n}\n\nexport default PostgresEngine;\n"],"mappings":";;;;;;AA4BA,IAAa,SAAb,MAAoB;CAClB;CAEA;CAEA;CAEA,YAAYA,MAAcC,UAAkBC,WAAoB,MAAM;EACpE,KAAK,OAAO;EACZ,KAAK,WAAW;EAChB,KAAK,WAAW;EAEhB,KAAK,mBAAmB;CACzB;CAED,AAAQ,oBAAoB;AAC1B,MAAI,OAAO,KAAK,SAAS,SACvB,OAAM,MAAM,kCAAkC;AAGhD,MAAI,OAAO,KAAK,aAAa,SAC3B,OAAM,MAAM,uCAAuC;CAEtD;AACF;AAED,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCnB,IAAa,iBAAb,MAAa,eAAe;CAC1B,OAAe,aAAa,OAAO,MAAM;CAEzC;CAEA,OAAO;CAEP,YAAYC,KAAaC,MAAiB;AACxC,MAAI,QAAQ,eAAe,WACzB,OAAM,MAAM,6CAA6C;EAE3D,KAAK,OAAO;CACb;;;;;;;;;;;;CAcD,aAAa,aACXC,WACAC,QACAC,UACAC,UACA,EACE,SAAS,eAAe,QACxB,MACA,UACA,iBACmB,GAAG,CAAE,GACD;EACzB,IAAIC;EACJ,IAAIC;AAEJ,MAAK,CAAC,QAAQ,YAAc,QAAQ,CAAC,SAEnC,OAAM,MACJ,mKAGD;AAIH,MAAI,SAAS,UAAa,aAAa,QAAW;GAChD,gBAAgB;GAChB,SAAS;EACV,OAAM;GACL,gBAAgB;AAChB,OAAI,oBAAoB,QACtB,SAAS;QACJ;IAEL,MAAM,OAAO,IAAI,WAAW,EAC1B,QAAQ,iDACT;IAED,SAAS,MAAM,qBAAqB,KAAK;GAC1C;EACF;EAED,eAAe,YAAY,IAAI,UAAU,EAAE,WAAW,WAAY;EAClE,MAAM,aAAa,MAAM,eAAe,UAAU,WAAW;GAC3D,wBAAwB,GAAG,UAAU,CAAC,EAAE,OAAO,CAAC,EAAE,UAAU;GAC5D;GACA,UAAU,gBAAgB,UAAU,MAAM,UAAU;EACrD,EAAC;EAEF,MAAMC,WAA6B;GACjC,QAAQ;GACR,YAAY;IACV,GAAG;IACH,GAAI,WAAW,EAAE,SAAU,IAAG,CAAE;IAChC,MAAM;IACN;GACD;EACF;EAED,MAAM,SAAS,KAAK,SAAS;AAE7B,SAAO,IAAI,eAAe,eAAe,YAAY;CACtD;;;;;;;CAQD,aAAa,SAASC,QAAmB;AACvC,SAAO,IAAI,eAAe,eAAe,YAAY;CACtD;;;;;;;;CASD,aAAa,qBACXC,KACAC,YACA;EACA,MAAM,SAAS;AAEf,MAAI,OAAO,QAAQ,YAAY,CAAC,IAAI,WAAW,OAAO,CACpD,OAAM,MAAM,2CAA2C;EAGzD,MAAMH,WAA6B;GACjC,QAAQ;GACR,YAAY;GACZ,0BAA0B;GAC1B,MAAM;IACJ,GAAG;IACH,sBAAsB;GACvB;EACF;EAED,MAAM,SAAS,KAAK,SAAS;AAE7B,SAAO,IAAI,eAAe,eAAe,YAAY;CACtD;;;;;;;;;;;;;;;;CAiBD,MAAM,qBACJI,WACAC,YACA,EACE,aAAa,UACb,gBAAgB,WAChB,kBAAkB,aAClB,sBAAsB,UACtB,kBAAkB,CAAE,GACpB,qBAAqB,sBACrB,WAAW,gBACX,oBAAoB,OACpB,gBAAgB,MACK,GAAG,CAAE,GACb;EACf,MAAM,KAAK,KAAK,IAAI,wCAAwC;AAE5D,MAAI,mBACF,MAAM,KAAK,KAAK,OACb,WAAW,WAAW,CACtB,kBAAkB,UAAU;EAGjC,MAAM,aACJ,OAAO,aAAa,WAAW,SAAS,SAAS;EACnD,MAAM,eACJ,OAAO,aAAa,WAAW,WAAW,SAAS;EAErD,IAAI,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,UAAU;MAClD,EAAE,aAAa,CAAC,EAAE,WAAW;MAC7B,EAAE,cAAc;MAChB,EAAE,gBAAgB,CAAC,EAAE,oBAAoB,CAAC,EAAE,WAAW,UAAU,CAAC;AAEpE,OAAK,MAAM,UAAU,iBAAiB;GACpC,MAAM,WAAW,CAAC,OAAO,WAAW,aAAa;GACjD,SAAS,CAAC,IAAI,EAAE,OAAO,KAAK,CAAC,EAAE,OAAO,SAAS,CAAC,EAAE,UAAU;EAC7D;AAED,MAAI,eACF,SAAS,CAAC,GAAG,EAAE,mBAAmB,KAAK,CAAC;EAG1C,SAAS,CAAC,IAAI,CAAC;EAEf,MAAM,KAAK,KAAK,IAAI,MAAM;CAC3B;;;;;;;CASD,MAAM,qBACJD,WACAE,aAAqB,UACN;EACf,MAAM,KAAK,KAAK,IACd,CAAC,2BAA2B,EAAE,WAAW,CAAC,EAAE,UAAU;;;;0BAIlC,CAAC,CACtB;CACF;;;;CAKD,MAAM,kBAAiC;EACrC,MAAM,KAAK,KAAK,SAAS;AACzB,MAAI,eAAe,cAAc,QAC/B,eAAe,UAAU,OAAO;CAEnC;CAGD,iBAAiB;EACf,MAAM,MAAM,KAAK,KAAK,IAAI,mCAAmC;AAC7D,SAAO;CACR;AACF"}
|
|
1
|
+
{"version":3,"file":"engine.js","names":[],"sources":["../src/engine.ts"],"sourcesContent":["import {\n AuthTypes,\n Connector,\n IpAddressTypes,\n} from \"@google-cloud/cloud-sql-connector\";\nimport { GoogleAuth } from \"google-auth-library\";\nimport knex from \"knex\";\nimport { getIAMPrincipalEmail } from \"./utils/utils.js\";\n\nexport interface PostgresEngineArgs {\n ipType?: IpAddressTypes;\n user?: string;\n password?: string;\n iamAccountEmail?: string;\n}\n\nexport interface VectorStoreTableArgs {\n schemaName?: string;\n contentColumn?: string;\n embeddingColumn?: string;\n embeddingColumnType?: \"vector\" | \"halfvec\" | \"bit\" | \"sparsevec\";\n metadataColumns?: Column[];\n metadataJsonColumn?: string;\n idColumn?: string | Column;\n overwriteExisting?: boolean;\n storeMetadata?: boolean;\n}\n\nexport class Column {\n name: string;\n\n dataType: string;\n\n nullable: boolean;\n\n constructor(name: string, dataType: string, nullable: boolean = true) {\n this.name = name;\n this.dataType = dataType;\n this.nullable = nullable;\n\n this.postInitilization();\n }\n\n private postInitilization() {\n if (typeof this.name !== \"string\") {\n throw Error(\"Column name must be type string\");\n }\n\n if (typeof this.dataType !== \"string\") {\n throw Error(\"Column data_type must be type string\");\n }\n }\n}\n\nconst USER_AGENT = \"langchain-google-cloud-sql-pg-js\";\n\n/**\n * Cloud SQL shared connection pool\n *\n * Setup:\n * Install `@langchain/google-cloud-sql-pg`\n *\n * <details open>\n * <summary><strong>Instantiate</strong></summary>\n *\n * ```typescript\n * import { Column, PostgresEngine, PostgresEngineArgs } from \"@langchain/google-cloud-sql-pg\";\n *\n * const pgArgs: PostgresEngineArgs = {\n * user: \"db-user\",\n * password: \"password\"\n *}\n *\n * const engine: PostgresEngine = await PostgresEngine.fromInstance(\n * \"project-id\",\n * \"region\",\n * \"instance-name\",\n * \"database-name\",\n * pgArgs\n * );\n * ```\n * </details>\n *\n * <br />\n *\n */\nexport class PostgresEngine {\n private static _createKey = Symbol(\"key\");\n\n pool: knex.Knex;\n\n static connector: Connector;\n\n constructor(key: symbol, pool: knex.Knex) {\n if (key !== PostgresEngine._createKey) {\n throw Error(\"Only create class through 'create' method!\");\n }\n this.pool = pool;\n }\n\n /**\n * @param projectId Required - GCP Project ID\n * @param region Required - Postgres Instance Region\n * @param instance Required - Postgres Instance name\n * @param database Required - Database name\n * @param ipType Optional - IP address type. Defaults to IPAddressType.PUBLIC\n * @param user Optional - Postgres user name. Defaults to undefined\n * @param password Optional - Postgres user password. Defaults to undefined\n * @param iamAccountEmail Optional - IAM service account email. Defaults to undefined\n * @returns PostgresEngine instance\n */\n\n static async fromInstance(\n projectId: string,\n region: string,\n instance: string,\n database: string,\n {\n ipType = IpAddressTypes.PUBLIC,\n user,\n password,\n iamAccountEmail,\n }: PostgresEngineArgs = {}\n ): Promise<PostgresEngine> {\n let dbUser: string;\n let enableIAMAuth: boolean;\n\n if ((!user && password) || (user && !password)) {\n // XOR for strings\n throw Error(\n \"Only one of 'user' or 'password' were specified. Either \" +\n \"both should be specified to use basic user/password \" +\n \"authentication or neither for IAM DB authentication.\"\n );\n }\n\n // User and password are given so we use the basic auth\n if (user !== undefined && password !== undefined) {\n enableIAMAuth = false;\n dbUser = user!;\n } else {\n enableIAMAuth = true;\n if (iamAccountEmail !== undefined) {\n dbUser = iamAccountEmail;\n } else {\n // Get application default credentials\n const auth = new GoogleAuth({\n scopes: \"https://www.googleapis.com/auth/cloud-platform\",\n });\n // dbUser should be the iam principal email by passing the credentials obtained\n dbUser = await getIAMPrincipalEmail(auth);\n }\n }\n\n PostgresEngine.connector = new Connector({ userAgent: USER_AGENT });\n const clientOpts = await PostgresEngine.connector.getOptions({\n instanceConnectionName: `${projectId}:${region}:${instance}`,\n ipType,\n authType: enableIAMAuth ? AuthTypes.IAM : AuthTypes.PASSWORD,\n });\n\n const dbConfig: knex.Knex.Config = {\n client: \"pg\",\n connection: {\n ...clientOpts,\n ...(password ? { password } : {}),\n user: dbUser,\n database,\n },\n };\n\n const engine = knex(dbConfig);\n\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a PostgresEngine instance from an Knex instance.\n *\n * @param engine knex instance\n * @returns PostgresEngine instance from a knex instance\n */\n static async fromPool(engine: knex.Knex) {\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a PostgresEngine instance from arguments.\n *\n * @param url URL use to connect to a database\n * @param poolConfig Optional - Configuration pool to use in the Knex configuration\n * @returns PostgresEngine instance\n */\n static async fromConnectionString(\n url: string | knex.Knex.StaticConnectionConfig,\n poolConfig?: knex.Knex.PoolConfig\n ) {\n const driver = \"postgresql+asyncpg\";\n\n if (typeof url === \"string\" && !url.startsWith(driver)) {\n throw Error(\"Driver must be type 'postgresql+asyncpg'\");\n }\n\n const dbConfig: knex.Knex.Config = {\n client: \"pg\",\n connection: url,\n acquireConnectionTimeout: 1000000,\n pool: {\n ...poolConfig,\n acquireTimeoutMillis: 600000,\n },\n };\n\n const engine = knex(dbConfig);\n\n return new PostgresEngine(PostgresEngine._createKey, engine);\n }\n\n /**\n * Create a table for saving of vectors to be used with PostgresVectorStore.\n *\n * @param tableName Postgres database table name. Parameter is not escaped. Do not use with end user input.\n * @param vectorSize Vector size for the embedding model to be used.\n * @param schemaName The schema name to store Postgres database table. Default: \"public\". Parameter is not escaped. Do not use with end user input.\n * @param contentColumn Name of the column to store document content. Default: \"content\".\n * @param embeddingColumn Name of the column to store vector embeddings. Default: \"embedding\".\n * @param embeddingColumnType Type of the embedding column (\"vector\" | \"halfvec\" | \"bit\" | \"sparsevec\"). Default: \"vector\". More info on HNSW-supported types: https://github.com/pgvector/pgvector#hnsw\n * @param metadataColumns Optional - A list of Columns to create for custom metadata. Default: [].\n * @param metadataJsonColumn Optional - The column to store extra metadata in JSON format. Default: \"langchain_metadata\".\n * @param idColumn Optional - Column to store ids. Default: \"langchain_id\" column name with data type UUID.\n * @param overwriteExisting Whether to drop existing table. Default: False.\n * @param storeMetadata Whether to store metadata in the table. Default: True.\n */\n async initVectorstoreTable(\n tableName: string,\n vectorSize: number,\n {\n schemaName = \"public\",\n contentColumn = \"content\",\n embeddingColumn = \"embedding\",\n embeddingColumnType = \"vector\",\n metadataColumns = [],\n metadataJsonColumn = \"langchain_metadata\",\n idColumn = \"langchain_id\",\n overwriteExisting = false,\n storeMetadata = true,\n }: VectorStoreTableArgs = {}\n ): Promise<void> {\n await this.pool.raw(\"CREATE EXTENSION IF NOT EXISTS vector\");\n\n if (overwriteExisting) {\n await this.pool.schema\n .withSchema(schemaName)\n .dropTableIfExists(tableName);\n }\n\n const idDataType =\n typeof idColumn === \"string\" ? \"UUID\" : idColumn.dataType;\n const idColumnName =\n typeof idColumn === \"string\" ? idColumn : idColumn.name;\n\n let query = `CREATE TABLE ${schemaName}.${tableName}(\n ${idColumnName} ${idDataType} PRIMARY KEY,\n ${contentColumn} TEXT NOT NULL,\n ${embeddingColumn} ${embeddingColumnType}(${vectorSize}) NOT NULL`;\n\n for (const column of metadataColumns) {\n const nullable = !column.nullable ? \"NOT NULL\" : \"\";\n query += `,\\n ${column.name} ${column.dataType} ${nullable}`;\n }\n\n if (storeMetadata) {\n query += `,\\n${metadataJsonColumn} JSON`;\n }\n\n query += `\\n);`;\n\n await this.pool.raw(query);\n }\n\n /**\n * Create a Cloud SQL table to store chat history.\n *\n * @param tableName Table name to store chat history\n * @param schemaName Schema name to store chat history table\n */\n\n async initChatHistoryTable(\n tableName: string,\n schemaName: string = \"public\"\n ): Promise<void> {\n await this.pool.raw(\n `CREATE TABLE IF NOT EXISTS ${schemaName}.${tableName}(\n id SERIAL PRIMARY KEY,\n session_id TEXT NOT NULL,\n data JSONB NOT NULL,\n type TEXT NOT NULL);`\n );\n }\n\n /**\n * Dispose of connection pool\n */\n async closeConnection(): Promise<void> {\n await this.pool.destroy();\n if (PostgresEngine.connector !== undefined) {\n PostgresEngine.connector.close();\n }\n }\n\n // Just to test the connection to the database\n testConnection() {\n const now = this.pool.raw(\"SELECT NOW() as currentTimestamp\");\n return now;\n }\n}\n\nexport default PostgresEngine;\n"],"mappings":";;;;;AA4BA,IAAa,SAAb,MAAoB;CAClB;CAEA;CAEA;CAEA,YAAY,MAAc,UAAkB,WAAoB,MAAM;AACpE,OAAK,OAAO;AACZ,OAAK,WAAW;AAChB,OAAK,WAAW;AAEhB,OAAK,mBAAmB;;CAG1B,oBAA4B;AAC1B,MAAI,OAAO,KAAK,SAAS,SACvB,OAAM,MAAM,kCAAkC;AAGhD,MAAI,OAAO,KAAK,aAAa,SAC3B,OAAM,MAAM,uCAAuC;;;AAKzD,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCnB,IAAa,iBAAb,MAAa,eAAe;CAC1B,OAAe,aAAa,OAAO,MAAM;CAEzC;CAEA,OAAO;CAEP,YAAY,KAAa,MAAiB;AACxC,MAAI,QAAQ,eAAe,WACzB,OAAM,MAAM,6CAA6C;AAE3D,OAAK,OAAO;;;;;;;;;;;;;CAed,aAAa,aACX,WACA,QACA,UACA,UACA,EACE,SAAS,eAAe,QACxB,MACA,UACA,oBACsB,EAAE,EACD;EACzB,IAAI;EACJ,IAAI;AAEJ,MAAK,CAAC,QAAQ,YAAc,QAAQ,CAAC,SAEnC,OAAM,MACJ,mKAGD;AAIH,MAAI,SAAS,KAAA,KAAa,aAAa,KAAA,GAAW;AAChD,mBAAgB;AAChB,YAAS;SACJ;AACL,mBAAgB;AAChB,OAAI,oBAAoB,KAAA,EACtB,UAAS;OAOT,UAAS,MAAM,qBAJF,IAAI,WAAW,EAC1B,QAAQ,kDACT,CAAC,CAEuC;;AAI7C,iBAAe,YAAY,IAAI,UAAU,EAAE,WAAW,YAAY,CAAC;EAiBnE,MAAM,SAAS,KAVoB;GACjC,QAAQ;GACR,YAAY;IACV,GATe,MAAM,eAAe,UAAU,WAAW;KAC3D,wBAAwB,GAAG,UAAU,GAAG,OAAO,GAAG;KAClD;KACA,UAAU,gBAAgB,UAAU,MAAM,UAAU;KACrD,CAAC;IAME,GAAI,WAAW,EAAE,UAAU,GAAG,EAAE;IAChC,MAAM;IACN;IACD;GACF,CAE4B;AAE7B,SAAO,IAAI,eAAe,eAAe,YAAY,OAAO;;;;;;;;CAS9D,aAAa,SAAS,QAAmB;AACvC,SAAO,IAAI,eAAe,eAAe,YAAY,OAAO;;;;;;;;;CAU9D,aAAa,qBACX,KACA,YACA;AAGA,MAAI,OAAO,QAAQ,YAAY,CAAC,IAAI,WAFrB,qBAEuC,CACpD,OAAM,MAAM,2CAA2C;EAazD,MAAM,SAAS,KAVoB;GACjC,QAAQ;GACR,YAAY;GACZ,0BAA0B;GAC1B,MAAM;IACJ,GAAG;IACH,sBAAsB;IACvB;GACF,CAE4B;AAE7B,SAAO,IAAI,eAAe,eAAe,YAAY,OAAO;;;;;;;;;;;;;;;;;CAkB9D,MAAM,qBACJ,WACA,YACA,EACE,aAAa,UACb,gBAAgB,WAChB,kBAAkB,aAClB,sBAAsB,UACtB,kBAAkB,EAAE,EACpB,qBAAqB,sBACrB,WAAW,gBACX,oBAAoB,OACpB,gBAAgB,SACQ,EAAE,EACb;AACf,QAAM,KAAK,KAAK,IAAI,wCAAwC;AAE5D,MAAI,kBACF,OAAM,KAAK,KAAK,OACb,WAAW,WAAW,CACtB,kBAAkB,UAAU;EAGjC,MAAM,aACJ,OAAO,aAAa,WAAW,SAAS,SAAS;EAInD,IAAI,QAAQ,gBAAgB,WAAW,GAAG,UAAU;QAFlD,OAAO,aAAa,WAAW,WAAW,SAAS,KAGpC,GAAG,WAAW;QAC3B,cAAc;QACd,gBAAgB,GAAG,oBAAoB,GAAG,WAAW;AAEzD,OAAK,MAAM,UAAU,iBAAiB;GACpC,MAAM,WAAW,CAAC,OAAO,WAAW,aAAa;AACjD,YAAS,OAAO,OAAO,KAAK,GAAG,OAAO,SAAS,GAAG;;AAGpD,MAAI,cACF,UAAS,MAAM,mBAAmB;AAGpC,WAAS;AAET,QAAM,KAAK,KAAK,IAAI,MAAM;;;;;;;;CAU5B,MAAM,qBACJ,WACA,aAAqB,UACN;AACf,QAAM,KAAK,KAAK,IACd,8BAA8B,WAAW,GAAG,UAAU;;;;4BAKvD;;;;;CAMH,MAAM,kBAAiC;AACrC,QAAM,KAAK,KAAK,SAAS;AACzB,MAAI,eAAe,cAAc,KAAA,EAC/B,gBAAe,UAAU,OAAO;;CAKpC,iBAAiB;AAEf,SADY,KAAK,KAAK,IAAI,mCAAmC"}
|
package/dist/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
const
|
|
3
|
-
const
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_engine = require("./engine.cjs");
|
|
3
|
+
const require_chat_message_history = require("./chat_message_history.cjs");
|
|
4
|
+
const require_indexes = require("./indexes.cjs");
|
|
5
|
+
const require_vectorstore = require("./vectorstore.cjs");
|
|
6
|
+
const require_loader = require("./loader.cjs");
|
|
7
7
|
exports.BaseIndex = require_indexes.BaseIndex;
|
|
8
8
|
exports.Column = require_engine.Column;
|
|
9
9
|
exports.DEFAULT_DISTANCE_STRATEGY = require_indexes.DEFAULT_DISTANCE_STRATEGY;
|
|
@@ -15,7 +15,7 @@ exports.HNSWQueryOptions = require_indexes.HNSWQueryOptions;
|
|
|
15
15
|
exports.IVFFlatIndex = require_indexes.IVFFlatIndex;
|
|
16
16
|
exports.IVFFlatQueryOptions = require_indexes.IVFFlatQueryOptions;
|
|
17
17
|
exports.PostgresChatMessageHistory = require_chat_message_history.PostgresChatMessageHistory;
|
|
18
|
-
exports.PostgresEngine = require_engine
|
|
18
|
+
exports.PostgresEngine = require_engine;
|
|
19
19
|
exports.PostgresLoader = require_loader.PostgresLoader;
|
|
20
|
-
exports.PostgresVectorStore = require_vectorstore
|
|
21
|
-
exports.QueryOptions = require_indexes.QueryOptions;
|
|
20
|
+
exports.PostgresVectorStore = require_vectorstore;
|
|
21
|
+
exports.QueryOptions = require_indexes.QueryOptions;
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Column
|
|
1
|
+
import PostgresEngine, { Column } from "./engine.js";
|
|
2
2
|
import { PostgresChatMessageHistory } from "./chat_message_history.js";
|
|
3
3
|
import { BaseIndex, DEFAULT_DISTANCE_STRATEGY, DEFAULT_INDEX_NAME_SUFFIX, DistanceStrategy, ExactNearestNeighbor, HNSWIndex, HNSWQueryOptions, IVFFlatIndex, IVFFlatQueryOptions, QueryOptions } from "./indexes.js";
|
|
4
|
-
import
|
|
4
|
+
import PostgresVectorStore from "./vectorstore.js";
|
|
5
5
|
import { PostgresLoader } from "./loader.js";
|
|
6
|
-
|
|
7
|
-
export { BaseIndex, Column, DEFAULT_DISTANCE_STRATEGY, DEFAULT_INDEX_NAME_SUFFIX, DistanceStrategy, ExactNearestNeighbor, HNSWIndex, HNSWQueryOptions, IVFFlatIndex, IVFFlatQueryOptions, PostgresChatMessageHistory, PostgresEngine, PostgresLoader, PostgresVectorStore, QueryOptions };
|
|
6
|
+
export { BaseIndex, Column, DEFAULT_DISTANCE_STRATEGY, DEFAULT_INDEX_NAME_SUFFIX, DistanceStrategy, ExactNearestNeighbor, HNSWIndex, HNSWQueryOptions, IVFFlatIndex, IVFFlatQueryOptions, PostgresChatMessageHistory, PostgresEngine, PostgresLoader, PostgresVectorStore, QueryOptions };
|
package/dist/indexes.cjs
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
//#region src/indexes.ts
|
|
3
2
|
var StrategyMixin = class {
|
|
4
3
|
operator;
|
|
@@ -87,7 +86,6 @@ var IVFFlatQueryOptions = class extends QueryOptions {
|
|
|
87
86
|
return `ivflfat.probes = ${this.probes}`;
|
|
88
87
|
}
|
|
89
88
|
};
|
|
90
|
-
|
|
91
89
|
//#endregion
|
|
92
90
|
exports.BaseIndex = BaseIndex;
|
|
93
91
|
exports.DEFAULT_DISTANCE_STRATEGY = DEFAULT_DISTANCE_STRATEGY;
|
|
@@ -99,4 +97,5 @@ exports.HNSWQueryOptions = HNSWQueryOptions;
|
|
|
99
97
|
exports.IVFFlatIndex = IVFFlatIndex;
|
|
100
98
|
exports.IVFFlatQueryOptions = IVFFlatQueryOptions;
|
|
101
99
|
exports.QueryOptions = QueryOptions;
|
|
100
|
+
|
|
102
101
|
//# sourceMappingURL=indexes.cjs.map
|
package/dist/indexes.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexes.cjs","names":[
|
|
1
|
+
{"version":3,"file":"indexes.cjs","names":[],"sources":["../src/indexes.ts"],"sourcesContent":["class StrategyMixin {\n operator: string;\n\n searchFunction: string;\n\n indexFunction: string;\n\n constructor(operator: string, searchFunction: string, indexFunction: string) {\n this.operator = operator;\n this.searchFunction = searchFunction;\n this.indexFunction = indexFunction;\n }\n}\n\n/**\n * Enumerator of the Distance strategies.\n */\nexport class DistanceStrategy extends StrategyMixin {\n public static EUCLIDEAN = new StrategyMixin(\n \"<->\",\n \"l2_distance\",\n \"vector_l2_ops\"\n );\n\n public static COSINE_DISTANCE = new StrategyMixin(\n \"<=>\",\n \"cosine_distance\",\n \"vector_cosine_ops\"\n );\n\n public static INNER_PRODUCT = new StrategyMixin(\n \"<#>\",\n \"inner_product\",\n \"vector_ip_ops\"\n );\n}\n\nexport const DEFAULT_DISTANCE_STRATEGY = DistanceStrategy.COSINE_DISTANCE;\nexport const DEFAULT_INDEX_NAME_SUFFIX: string = \"langchainvectorindex\";\n\nexport interface BaseIndexArgs {\n name?: string;\n distanceStrategy?: DistanceStrategy;\n partialIndexes?: string[];\n}\n\nexport abstract class BaseIndex {\n name?: string;\n\n indexType: string;\n\n distanceStrategy: DistanceStrategy;\n\n partialIndexes?: string[];\n\n constructor(\n name?: string,\n indexType: string = \"base\",\n distanceStrategy: DistanceStrategy = DistanceStrategy.COSINE_DISTANCE,\n partialIndexes: string[] = []\n ) {\n this.name = name;\n this.indexType = indexType;\n this.distanceStrategy = distanceStrategy;\n this.partialIndexes = partialIndexes;\n }\n\n /**\n * Set index query options for vector store initialization.\n */\n abstract indexOptions(): string;\n}\n\nexport class ExactNearestNeighbor extends BaseIndex {\n constructor(baseArgs?: BaseIndexArgs) {\n super(\n baseArgs?.name,\n \"exactnearestneighbor\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n }\n\n indexOptions(): string {\n throw new Error(\"indexOptions method must be implemented by subclass\");\n }\n}\n\nexport class HNSWIndex extends BaseIndex {\n m: number;\n\n efConstruction: number;\n\n constructor(baseArgs?: BaseIndexArgs, m?: number, efConstruction?: number) {\n super(\n baseArgs?.name,\n \"hnsw\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n this.m = m ?? 16;\n this.efConstruction = efConstruction ?? 64;\n }\n\n indexOptions(): string {\n return `(m = ${this.m}, ef_construction = ${this.efConstruction})`;\n }\n}\n\nexport class IVFFlatIndex extends BaseIndex {\n lists: number;\n\n constructor(baseArgs: BaseIndexArgs, lists?: number) {\n super(\n baseArgs?.name,\n \"ivfflat\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n this.lists = lists ?? 100;\n }\n\n indexOptions(): string {\n return `(lists = ${this.lists})`;\n }\n}\n\n/**\n * Convert index attributes to string.\n * Must be implemented by subclasses.\n */\nexport abstract class QueryOptions {\n abstract to_string(): string;\n}\n\nexport class HNSWQueryOptions extends QueryOptions {\n efSearch: number;\n\n constructor(efSearch?: number) {\n super();\n this.efSearch = efSearch ?? 40;\n }\n\n to_string(): string {\n return `hnsw.ef_search = ${this.efSearch}`;\n }\n}\n\nexport class IVFFlatQueryOptions extends QueryOptions {\n readonly probes: number;\n\n constructor(probes?: number) {\n super();\n this.probes = probes ?? 1;\n }\n\n to_string(): string {\n return `ivflfat.probes = ${this.probes}`;\n }\n}\n"],"mappings":";AAAA,IAAM,gBAAN,MAAoB;CAClB;CAEA;CAEA;CAEA,YAAY,UAAkB,gBAAwB,eAAuB;AAC3E,OAAK,WAAW;AAChB,OAAK,iBAAiB;AACtB,OAAK,gBAAgB;;;;;;AAOzB,IAAa,mBAAb,cAAsC,cAAc;CAClD,OAAc,YAAY,IAAI,cAC5B,OACA,eACA,gBACD;CAED,OAAc,kBAAkB,IAAI,cAClC,OACA,mBACA,oBACD;CAED,OAAc,gBAAgB,IAAI,cAChC,OACA,iBACA,gBACD;;AAGH,MAAa,4BAA4B,iBAAiB;AAC1D,MAAa,4BAAoC;AAQjD,IAAsB,YAAtB,MAAgC;CAC9B;CAEA;CAEA;CAEA;CAEA,YACE,MACA,YAAoB,QACpB,mBAAqC,iBAAiB,iBACtD,iBAA2B,EAAE,EAC7B;AACA,OAAK,OAAO;AACZ,OAAK,YAAY;AACjB,OAAK,mBAAmB;AACxB,OAAK,iBAAiB;;;AAS1B,IAAa,uBAAb,cAA0C,UAAU;CAClD,YAAY,UAA0B;AACpC,QACE,UAAU,MACV,wBACA,UAAU,kBACV,UAAU,eACX;;CAGH,eAAuB;AACrB,QAAM,IAAI,MAAM,sDAAsD;;;AAI1E,IAAa,YAAb,cAA+B,UAAU;CACvC;CAEA;CAEA,YAAY,UAA0B,GAAY,gBAAyB;AACzE,QACE,UAAU,MACV,QACA,UAAU,kBACV,UAAU,eACX;AACD,OAAK,IAAI,KAAK;AACd,OAAK,iBAAiB,kBAAkB;;CAG1C,eAAuB;AACrB,SAAO,QAAQ,KAAK,EAAE,sBAAsB,KAAK,eAAe;;;AAIpE,IAAa,eAAb,cAAkC,UAAU;CAC1C;CAEA,YAAY,UAAyB,OAAgB;AACnD,QACE,UAAU,MACV,WACA,UAAU,kBACV,UAAU,eACX;AACD,OAAK,QAAQ,SAAS;;CAGxB,eAAuB;AACrB,SAAO,YAAY,KAAK,MAAM;;;;;;;AAQlC,IAAsB,eAAtB,MAAmC;AAInC,IAAa,mBAAb,cAAsC,aAAa;CACjD;CAEA,YAAY,UAAmB;AAC7B,SAAO;AACP,OAAK,WAAW,YAAY;;CAG9B,YAAoB;AAClB,SAAO,oBAAoB,KAAK;;;AAIpC,IAAa,sBAAb,cAAyC,aAAa;CACpD;CAEA,YAAY,QAAiB;AAC3B,SAAO;AACP,OAAK,SAAS,UAAU;;CAG1B,YAAoB;AAClB,SAAO,oBAAoB,KAAK"}
|
package/dist/indexes.d.cts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexes.d.cts","names":[
|
|
1
|
+
{"version":3,"file":"indexes.d.cts","names":[],"sources":["../src/indexes.ts"],"mappings":";cAAM,aAAA;EACJ,QAAA;EAEA,cAAA;EAEA,aAAA;EAEA,WAAA,CAAY,QAAA,UAAkB,cAAA,UAAwB,aAAA;AAAA;;;;cAU3C,gBAAA,SAAyB,aAAA;EAAA,OACtB,SAAA,EAAS,aAAA;EAAA,OAMT,eAAA,EAAe,aAAA;EAAA,OAMf,aAAA,EAAa,aAAA;AAAA;AAAA,cAOhB,yBAAA,EAAyB,aAAA;AAAA,cACzB,yBAAA;AAAA,UAEI,aAAA;EACf,IAAA;EACA,gBAAA,GAAmB,gBAAA;EACnB,cAAA;AAAA;AAAA,uBAGoB,SAAA;EACpB,IAAA;EAEA,SAAA;EAEA,gBAAA,EAAkB,gBAAA;EAElB,cAAA;EAEA,WAAA,CACE,IAAA,WACA,SAAA,WACA,gBAAA,GAAkB,gBAAA,EAClB,cAAA;EAnC2B;;;EAAA,SA8CpB,YAAA,CAAA;AAAA;AAAA,cAGE,oBAAA,SAA6B,SAAA;EACxC,WAAA,CAAY,QAAA,GAAW,aAAA;EASvB,YAAA,CAAA;AAAA;AAAA,cAKW,SAAA,SAAkB,SAAA;EAC7B,CAAA;EAEA,cAAA;EAEA,WAAA,CAAY,QAAA,GAAW,aAAA,EAAe,CAAA,WAAY,cAAA;EAWlD,YAAA,CAAA;AAAA;AAAA,cAKW,YAAA,SAAqB,SAAA;EAChC,KAAA;EAEA,WAAA,CAAY,QAAA,EAAU,aAAA,EAAe,KAAA;EAUrC,YAAA,CAAA;AAAA;;;;;uBASoB,YAAA;EAAA,SACX,SAAA,CAAA;AAAA;AAAA,cAGE,gBAAA,SAAyB,YAAA;EACpC,QAAA;EAEA,WAAA,CAAY,QAAA;EAKZ,SAAA,CAAA;AAAA;AAAA,cAKW,mBAAA,SAA4B,YAAA;EAAA,SAC9B,MAAA;EAET,WAAA,CAAY,MAAA;EAKZ,SAAA,CAAA;AAAA"}
|
package/dist/indexes.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexes.d.ts","names":[
|
|
1
|
+
{"version":3,"file":"indexes.d.ts","names":[],"sources":["../src/indexes.ts"],"mappings":";cAAM,aAAA;EACJ,QAAA;EAEA,cAAA;EAEA,aAAA;EAEA,WAAA,CAAY,QAAA,UAAkB,cAAA,UAAwB,aAAA;AAAA;;;;cAU3C,gBAAA,SAAyB,aAAA;EAAA,OACtB,SAAA,EAAS,aAAA;EAAA,OAMT,eAAA,EAAe,aAAA;EAAA,OAMf,aAAA,EAAa,aAAA;AAAA;AAAA,cAOhB,yBAAA,EAAyB,aAAA;AAAA,cACzB,yBAAA;AAAA,UAEI,aAAA;EACf,IAAA;EACA,gBAAA,GAAmB,gBAAA;EACnB,cAAA;AAAA;AAAA,uBAGoB,SAAA;EACpB,IAAA;EAEA,SAAA;EAEA,gBAAA,EAAkB,gBAAA;EAElB,cAAA;EAEA,WAAA,CACE,IAAA,WACA,SAAA,WACA,gBAAA,GAAkB,gBAAA,EAClB,cAAA;EAnC2B;;;EAAA,SA8CpB,YAAA,CAAA;AAAA;AAAA,cAGE,oBAAA,SAA6B,SAAA;EACxC,WAAA,CAAY,QAAA,GAAW,aAAA;EASvB,YAAA,CAAA;AAAA;AAAA,cAKW,SAAA,SAAkB,SAAA;EAC7B,CAAA;EAEA,cAAA;EAEA,WAAA,CAAY,QAAA,GAAW,aAAA,EAAe,CAAA,WAAY,cAAA;EAWlD,YAAA,CAAA;AAAA;AAAA,cAKW,YAAA,SAAqB,SAAA;EAChC,KAAA;EAEA,WAAA,CAAY,QAAA,EAAU,aAAA,EAAe,KAAA;EAUrC,YAAA,CAAA;AAAA;;;;;uBASoB,YAAA;EAAA,SACX,SAAA,CAAA;AAAA;AAAA,cAGE,gBAAA,SAAyB,YAAA;EACpC,QAAA;EAEA,WAAA,CAAY,QAAA;EAKZ,SAAA,CAAA;AAAA;AAAA,cAKW,mBAAA,SAA4B,YAAA;EAAA,SAC9B,MAAA;EAET,WAAA,CAAY,MAAA;EAKZ,SAAA,CAAA;AAAA"}
|
package/dist/indexes.js
CHANGED
|
@@ -86,7 +86,7 @@ var IVFFlatQueryOptions = class extends QueryOptions {
|
|
|
86
86
|
return `ivflfat.probes = ${this.probes}`;
|
|
87
87
|
}
|
|
88
88
|
};
|
|
89
|
-
|
|
90
89
|
//#endregion
|
|
91
90
|
export { BaseIndex, DEFAULT_DISTANCE_STRATEGY, DEFAULT_INDEX_NAME_SUFFIX, DistanceStrategy, ExactNearestNeighbor, HNSWIndex, HNSWQueryOptions, IVFFlatIndex, IVFFlatQueryOptions, QueryOptions };
|
|
91
|
+
|
|
92
92
|
//# sourceMappingURL=indexes.js.map
|
package/dist/indexes.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexes.js","names":[
|
|
1
|
+
{"version":3,"file":"indexes.js","names":[],"sources":["../src/indexes.ts"],"sourcesContent":["class StrategyMixin {\n operator: string;\n\n searchFunction: string;\n\n indexFunction: string;\n\n constructor(operator: string, searchFunction: string, indexFunction: string) {\n this.operator = operator;\n this.searchFunction = searchFunction;\n this.indexFunction = indexFunction;\n }\n}\n\n/**\n * Enumerator of the Distance strategies.\n */\nexport class DistanceStrategy extends StrategyMixin {\n public static EUCLIDEAN = new StrategyMixin(\n \"<->\",\n \"l2_distance\",\n \"vector_l2_ops\"\n );\n\n public static COSINE_DISTANCE = new StrategyMixin(\n \"<=>\",\n \"cosine_distance\",\n \"vector_cosine_ops\"\n );\n\n public static INNER_PRODUCT = new StrategyMixin(\n \"<#>\",\n \"inner_product\",\n \"vector_ip_ops\"\n );\n}\n\nexport const DEFAULT_DISTANCE_STRATEGY = DistanceStrategy.COSINE_DISTANCE;\nexport const DEFAULT_INDEX_NAME_SUFFIX: string = \"langchainvectorindex\";\n\nexport interface BaseIndexArgs {\n name?: string;\n distanceStrategy?: DistanceStrategy;\n partialIndexes?: string[];\n}\n\nexport abstract class BaseIndex {\n name?: string;\n\n indexType: string;\n\n distanceStrategy: DistanceStrategy;\n\n partialIndexes?: string[];\n\n constructor(\n name?: string,\n indexType: string = \"base\",\n distanceStrategy: DistanceStrategy = DistanceStrategy.COSINE_DISTANCE,\n partialIndexes: string[] = []\n ) {\n this.name = name;\n this.indexType = indexType;\n this.distanceStrategy = distanceStrategy;\n this.partialIndexes = partialIndexes;\n }\n\n /**\n * Set index query options for vector store initialization.\n */\n abstract indexOptions(): string;\n}\n\nexport class ExactNearestNeighbor extends BaseIndex {\n constructor(baseArgs?: BaseIndexArgs) {\n super(\n baseArgs?.name,\n \"exactnearestneighbor\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n }\n\n indexOptions(): string {\n throw new Error(\"indexOptions method must be implemented by subclass\");\n }\n}\n\nexport class HNSWIndex extends BaseIndex {\n m: number;\n\n efConstruction: number;\n\n constructor(baseArgs?: BaseIndexArgs, m?: number, efConstruction?: number) {\n super(\n baseArgs?.name,\n \"hnsw\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n this.m = m ?? 16;\n this.efConstruction = efConstruction ?? 64;\n }\n\n indexOptions(): string {\n return `(m = ${this.m}, ef_construction = ${this.efConstruction})`;\n }\n}\n\nexport class IVFFlatIndex extends BaseIndex {\n lists: number;\n\n constructor(baseArgs: BaseIndexArgs, lists?: number) {\n super(\n baseArgs?.name,\n \"ivfflat\",\n baseArgs?.distanceStrategy,\n baseArgs?.partialIndexes\n );\n this.lists = lists ?? 100;\n }\n\n indexOptions(): string {\n return `(lists = ${this.lists})`;\n }\n}\n\n/**\n * Convert index attributes to string.\n * Must be implemented by subclasses.\n */\nexport abstract class QueryOptions {\n abstract to_string(): string;\n}\n\nexport class HNSWQueryOptions extends QueryOptions {\n efSearch: number;\n\n constructor(efSearch?: number) {\n super();\n this.efSearch = efSearch ?? 40;\n }\n\n to_string(): string {\n return `hnsw.ef_search = ${this.efSearch}`;\n }\n}\n\nexport class IVFFlatQueryOptions extends QueryOptions {\n readonly probes: number;\n\n constructor(probes?: number) {\n super();\n this.probes = probes ?? 1;\n }\n\n to_string(): string {\n return `ivflfat.probes = ${this.probes}`;\n }\n}\n"],"mappings":";AAAA,IAAM,gBAAN,MAAoB;CAClB;CAEA;CAEA;CAEA,YAAY,UAAkB,gBAAwB,eAAuB;AAC3E,OAAK,WAAW;AAChB,OAAK,iBAAiB;AACtB,OAAK,gBAAgB;;;;;;AAOzB,IAAa,mBAAb,cAAsC,cAAc;CAClD,OAAc,YAAY,IAAI,cAC5B,OACA,eACA,gBACD;CAED,OAAc,kBAAkB,IAAI,cAClC,OACA,mBACA,oBACD;CAED,OAAc,gBAAgB,IAAI,cAChC,OACA,iBACA,gBACD;;AAGH,MAAa,4BAA4B,iBAAiB;AAC1D,MAAa,4BAAoC;AAQjD,IAAsB,YAAtB,MAAgC;CAC9B;CAEA;CAEA;CAEA;CAEA,YACE,MACA,YAAoB,QACpB,mBAAqC,iBAAiB,iBACtD,iBAA2B,EAAE,EAC7B;AACA,OAAK,OAAO;AACZ,OAAK,YAAY;AACjB,OAAK,mBAAmB;AACxB,OAAK,iBAAiB;;;AAS1B,IAAa,uBAAb,cAA0C,UAAU;CAClD,YAAY,UAA0B;AACpC,QACE,UAAU,MACV,wBACA,UAAU,kBACV,UAAU,eACX;;CAGH,eAAuB;AACrB,QAAM,IAAI,MAAM,sDAAsD;;;AAI1E,IAAa,YAAb,cAA+B,UAAU;CACvC;CAEA;CAEA,YAAY,UAA0B,GAAY,gBAAyB;AACzE,QACE,UAAU,MACV,QACA,UAAU,kBACV,UAAU,eACX;AACD,OAAK,IAAI,KAAK;AACd,OAAK,iBAAiB,kBAAkB;;CAG1C,eAAuB;AACrB,SAAO,QAAQ,KAAK,EAAE,sBAAsB,KAAK,eAAe;;;AAIpE,IAAa,eAAb,cAAkC,UAAU;CAC1C;CAEA,YAAY,UAAyB,OAAgB;AACnD,QACE,UAAU,MACV,WACA,UAAU,kBACV,UAAU,eACX;AACD,OAAK,QAAQ,SAAS;;CAGxB,eAAuB;AACrB,SAAO,YAAY,KAAK,MAAM;;;;;;;AAQlC,IAAsB,eAAtB,MAAmC;AAInC,IAAa,mBAAb,cAAsC,aAAa;CACjD;CAEA,YAAY,UAAmB;AAC7B,SAAO;AACP,OAAK,WAAW,YAAY;;CAG9B,YAAoB;AAClB,SAAO,oBAAoB,KAAK;;;AAIpC,IAAa,sBAAb,cAAyC,aAAa;CACpD;CAEA,YAAY,QAAiB;AAC3B,SAAO;AACP,OAAK,SAAS,UAAU;;CAG1B,YAAoB;AAClB,SAAO,oBAAoB,KAAK"}
|
package/dist/loader.cjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
const require_utils = require(
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
require("./_virtual/_rolldown/runtime.cjs");
|
|
2
|
+
const require_utils = require("./utils/utils.cjs");
|
|
3
|
+
let _langchain_core_document_loaders_base = require("@langchain/core/document_loaders/base");
|
|
5
4
|
//#region src/loader.ts
|
|
6
5
|
const DEFAULT_METADATA_COL = "langchain_metadata";
|
|
7
6
|
function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter = require_utils.textFormatter) {
|
|
@@ -73,7 +72,7 @@ function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColum
|
|
|
73
72
|
*
|
|
74
73
|
* <br />
|
|
75
74
|
*/
|
|
76
|
-
var PostgresLoader = class PostgresLoader extends
|
|
75
|
+
var PostgresLoader = class PostgresLoader extends _langchain_core_document_loaders_base.BaseDocumentLoader {
|
|
77
76
|
engine;
|
|
78
77
|
tableName;
|
|
79
78
|
schemaName;
|
|
@@ -121,8 +120,7 @@ var PostgresLoader = class PostgresLoader extends __langchain_core_document_load
|
|
|
121
120
|
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
122
121
|
let jsonColumnName = metadataJsonColumn;
|
|
123
122
|
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) jsonColumnName = DEFAULT_METADATA_COL;
|
|
124
|
-
|
|
125
|
-
allNames.forEach((name) => {
|
|
123
|
+
[...contentColumnNames || [], ...metadataColumnNames || []].forEach((name) => {
|
|
126
124
|
if (!columnNames.includes(name)) throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
127
125
|
});
|
|
128
126
|
return new PostgresLoader(engine, {
|
|
@@ -157,7 +155,7 @@ var PostgresLoader = class PostgresLoader extends __langchain_core_document_load
|
|
|
157
155
|
}
|
|
158
156
|
}
|
|
159
157
|
};
|
|
160
|
-
|
|
161
158
|
//#endregion
|
|
162
159
|
exports.PostgresLoader = PostgresLoader;
|
|
160
|
+
|
|
163
161
|
//# sourceMappingURL=loader.cjs.map
|
package/dist/loader.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.cjs","names":["contentColumns: string[]","metadataColumns: string[]","row: Row","metadataJsonColumn: string | null","formatter: (row: Row, contentColumns: string[]) => string","textFormatter","metadata: { [key: string]: string }","BaseDocumentLoader","engine: PostgresEngine","options: PostgresLoaderOptions","csvFormatter","yamlFormatter","jsonFormatter","field: { name: string }","col: string","documents: Document[]","rowData: Row"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACPA,gBACAC,iBACAC,KACAC,qBAAoC,sBACpCC,YAA4DC,6BAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAMC,WAAsC,CAAE;AAE9C,KAAI,sBAAsB,IAAI,qBAC5B,OAAO,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,KAAK;EAC1D,SAAS,KAAK;CACf,EAAC;CAGJ,gBAAgB,QAAQ,CAAC,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,oBAC9B,SAAS,UAAU,IAAI;CAE1B,EAAC;AAEF,QAAO;EAAE;EAAa;CAAU;AACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDD,IAAa,iBAAb,MAAa,uBAAuBC,0DAAmB;CACrD,AAAQ;CAER;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAYC,QAAwBC,SAAgC;EAClE,OAAO;EACP,KAAK,SAAS;EACd,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,kBAAkB,QAAQ;EAC/B,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;EACzB,KAAK,qBAAqB,QAAQ;CACnC;CAED,aAAa,WACXD,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,oBACsB,EACC;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR;AAGJ,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR;AAGJ,MAAI,UAAU,UACZ,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;EAAO,EAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM;EAGlB,IAAI,aAAa;AACjB,MAAI,eAAe,OACjB,KAAI,WAAW,OACb,aAAaE;WACJ,WAAW,QACpB,aAAaC;WACJ,WAAW,QACpB,aAAaC;OAEb,aAAaP;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,WACH,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,EAAE,UAAU,CAAC,CAAC;EAG5D,IAAI;AACJ,MAAI;GACF,SAAS,MAAM,OAAO,KAAK,IAAI,UAAU;EAC1C,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;EACD,MAAM,cAAc,OAAO,OAAO,IAChC,CAACQ,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,EAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,OAAO,CAACC,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,mBAAmB,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG5E,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,EAC/D,iBAAiB;EAGnB,MAAM,WAAW,CACf,GAAI,sBAAsB,CAAE,GAC5B,GAAI,uBAAuB,CAAE,CAC9B;EACD,SAAS,QAAQ,CAAC,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,KAAK,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG/D,EAAC;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;EACrB;CACF;CAED,MAAM,OAA4B;EAChC,MAAMC,YAAwB,CAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,EACrC,UAAU,KAAK,IAAI;AAErB,SAAO;CACR;CAED,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,oBACD,GAAG;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM;GAElB,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAMC,UAAe,CAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,CAAE,GACxB,GAAI,mBAAmB,CAAE,CAC1B;AACD,QAAI,oBACF,YAAY,KAAK,mBAAmB;IAEtC,YAAY,QAAQ,CAAC,WAAW;KAC9B,QAAQ,UAAU,IAAI;IACvB,EAAC;IAEF,MAAM,gBACJ,kBAAkB,CAAE,GACpB,mBAAmB,CAAE,GACrB,SACA,oBACA,UACD;GACF;EACF,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"loader.cjs","names":["textFormatter","BaseDocumentLoader","csvFormatter","yamlFormatter","jsonFormatter"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACP,gBACA,iBACA,KACA,qBAAoC,sBACpC,YAA4DA,cAAAA,eAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAM,WAAsC,EAAE;AAE9C,KAAI,sBAAsB,IAAI,oBAC5B,QAAO,QAAQ,IAAI,oBAAoB,CAAC,SAAS,CAAC,GAAG,OAAO;AAC1D,WAAS,KAAK;GACd;AAGJ,iBAAgB,SAAS,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,mBAC9B,UAAS,UAAU,IAAI;GAEzB;AAEF,QAAO;EAAE;EAAa;EAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0DlC,IAAa,iBAAb,MAAa,uBAAuBC,sCAAAA,mBAAmB;CACrD;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,QAAwB,SAAgC;AAClE,SAAO;AACP,OAAK,SAAS;AACd,OAAK,iBAAiB,QAAQ;AAC9B,OAAK,kBAAkB,QAAQ;AAC/B,OAAK,QAAQ,QAAQ;AACrB,OAAK,YAAY,QAAQ;AACzB,OAAK,qBAAqB,QAAQ;;CAGpC,aAAa,WACX,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,sBAEuB;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR,2DACD;AAEH,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR,8EACD;AAEH,MAAI,UAAU,UACZ,OAAM,IAAI,MACR,2DACD;AAGH,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;GAAO,CAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM,qDAAqD;EAGvE,IAAI,aAAa;AACjB,MAAI,eAAe,KAAA,EACjB,KAAI,WAAW,MACb,cAAaC,cAAAA;WACJ,WAAW,OACpB,cAAaC,cAAAA;WACJ,WAAW,OACpB,cAAaC,cAAAA;MAEb,cAAaJ,cAAAA;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,UACH,aAAY,kBAAkB,WAAW,KAAK,UAAU;EAG1D,IAAI;AACJ,MAAI;AACF,YAAS,MAAM,OAAO,KAAK,IAAI,UAAU;WAClC,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;;EAGtB,MAAM,cAAc,OAAO,OAAO,KAC/B,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,GAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,QAAQ,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,UAAU,mBAAmB,6BAA6B,YAAY,GACvE;EAEH,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,CAC/D,kBAAiB;AAGF,GACf,GAAI,sBAAsB,EAAE,EAC5B,GAAI,uBAAuB,EAAE,CAC9B,CACQ,SAAS,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,UAAU,KAAK,6BAA6B,YAAY,GACzD;IAEH;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;GACrB,CAAC;;CAGJ,MAAM,OAA4B;EAChC,MAAM,YAAwB,EAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,CACrC,WAAU,KAAK,IAAI;AAErB,SAAO;;CAGT,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,uBACE;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM,qBAAqB;GAEvC,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAM,UAAe,EAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,EAAE,EACxB,GAAI,mBAAmB,EAAE,CAC1B;AACD,QAAI,mBACF,aAAY,KAAK,mBAAmB;AAEtC,gBAAY,SAAS,WAAW;AAC9B,aAAQ,UAAU,IAAI;MACtB;AAEF,UAAM,gBACJ,kBAAkB,EAAE,EACpB,mBAAmB,EAAE,EACrB,SACA,oBACA,UACD;;WAEI,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM"}
|
package/dist/loader.d.cts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.d.cts","names":[
|
|
1
|
+
{"version":3,"file":"loader.d.cts","names":[],"sources":["../src/loader.ts"],"mappings":";;;;;KAWK,GAAA;EAAA,CAAS,GAAA;AAAA;AAAA,UAGG,qBAAA;EACf,SAAA;EACA,UAAA;EACA,cAAA;EACA,eAAA;EACA,MAAA;EACA,SAAA,IAAa,GAAA,EAAK,GAAA,EAAK,cAAA;EACvB,KAAA;EACA,kBAAA;AAAA;;;;;;;;;;;;AAmFF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAAa,cAAA,SAAuB,kBAAA;EAAA,QAC1B,MAAA;EAER,SAAA;EAEA,UAAA;EAEA,cAAA;EAEA,eAAA;EAEA,MAAA;EAEA,SAAA,IAAa,GAAA,EAAK,GAAA,EAAK,cAAA;EAEvB,KAAA;EAEA,kBAAA;EAEA,WAAA,CAAY,MAAA,EAAQ,cAAA,EAAgB,OAAA,EAAS,qBAAA;EAAA,OAUhC,UAAA,CACX,MAAA,EAAQ,cAAA;IAEN,UAAA;IACA,SAAA;IACA,cAAA;IACA,eAAA;IACA,MAAA;IACA,KAAA;IACA,SAAA;IACA;EAAA,GACC,qBAAA,GACF,OAAA,CAAQ,cAAA;EAuFL,IAAA,CAAA,GAAQ,OAAA,CAAQ,QAAA;EAQf,QAAA,CAAA,GAAY,cAAA,CAAe,QAAA;AAAA"}
|
package/dist/loader.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.d.ts","names":[
|
|
1
|
+
{"version":3,"file":"loader.d.ts","names":[],"sources":["../src/loader.ts"],"mappings":";;;;;KAWK,GAAA;EAAA,CAAS,GAAA;AAAA;AAAA,UAGG,qBAAA;EACf,SAAA;EACA,UAAA;EACA,cAAA;EACA,eAAA;EACA,MAAA;EACA,SAAA,IAAa,GAAA,EAAK,GAAA,EAAK,cAAA;EACvB,KAAA;EACA,kBAAA;AAAA;;;;;;;;;;;;AAmFF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAAa,cAAA,SAAuB,kBAAA;EAAA,QAC1B,MAAA;EAER,SAAA;EAEA,UAAA;EAEA,cAAA;EAEA,eAAA;EAEA,MAAA;EAEA,SAAA,IAAa,GAAA,EAAK,GAAA,EAAK,cAAA;EAEvB,KAAA;EAEA,kBAAA;EAEA,WAAA,CAAY,MAAA,EAAQ,cAAA,EAAgB,OAAA,EAAS,qBAAA;EAAA,OAUhC,UAAA,CACX,MAAA,EAAQ,cAAA;IAEN,UAAA;IACA,SAAA;IACA,cAAA;IACA,eAAA;IACA,MAAA;IACA,KAAA;IACA,SAAA;IACA;EAAA,GACC,qBAAA,GACF,OAAA,CAAQ,cAAA;EAuFL,IAAA,CAAA,GAAQ,OAAA,CAAQ,QAAA;EAQf,QAAA,CAAA,GAAY,cAAA,CAAe,QAAA;AAAA"}
|
package/dist/loader.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { csvFormatter, jsonFormatter, textFormatter, yamlFormatter } from "./utils/utils.js";
|
|
2
2
|
import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
|
|
3
|
-
|
|
4
3
|
//#region src/loader.ts
|
|
5
4
|
const DEFAULT_METADATA_COL = "langchain_metadata";
|
|
6
5
|
function parseDocFromRow(contentColumns, metadataColumns, row, metadataJsonColumn = DEFAULT_METADATA_COL, formatter = textFormatter) {
|
|
@@ -120,8 +119,7 @@ var PostgresLoader = class PostgresLoader extends BaseDocumentLoader {
|
|
|
120
119
|
if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) throw new Error(`Column ${metadataJsonColumn} not found in query result ${columnNames}.`);
|
|
121
120
|
let jsonColumnName = metadataJsonColumn;
|
|
122
121
|
if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) jsonColumnName = DEFAULT_METADATA_COL;
|
|
123
|
-
|
|
124
|
-
allNames.forEach((name) => {
|
|
122
|
+
[...contentColumnNames || [], ...metadataColumnNames || []].forEach((name) => {
|
|
125
123
|
if (!columnNames.includes(name)) throw new Error(`Column ${name} not found in query result ${columnNames}.`);
|
|
126
124
|
});
|
|
127
125
|
return new PostgresLoader(engine, {
|
|
@@ -156,7 +154,7 @@ var PostgresLoader = class PostgresLoader extends BaseDocumentLoader {
|
|
|
156
154
|
}
|
|
157
155
|
}
|
|
158
156
|
};
|
|
159
|
-
|
|
160
157
|
//#endregion
|
|
161
158
|
export { PostgresLoader };
|
|
159
|
+
|
|
162
160
|
//# sourceMappingURL=loader.js.map
|
package/dist/loader.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.js","names":["contentColumns: string[]","metadataColumns: string[]","row: Row","metadataJsonColumn: string | null","formatter: (row: Row, contentColumns: string[]) => string","metadata: { [key: string]: string }","engine: PostgresEngine","options: PostgresLoaderOptions","field: { name: string }","col: string","documents: Document[]","rowData: Row"],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACPA,gBACAC,iBACAC,KACAC,qBAAoC,sBACpCC,YAA4D,eAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAMC,WAAsC,CAAE;AAE9C,KAAI,sBAAsB,IAAI,qBAC5B,OAAO,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,KAAK;EAC1D,SAAS,KAAK;CACf,EAAC;CAGJ,gBAAgB,QAAQ,CAAC,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,oBAC9B,SAAS,UAAU,IAAI;CAE1B,EAAC;AAEF,QAAO;EAAE;EAAa;CAAU;AACjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDD,IAAa,iBAAb,MAAa,uBAAuB,mBAAmB;CACrD,AAAQ;CAER;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAYC,QAAwBC,SAAgC;EAClE,OAAO;EACP,KAAK,SAAS;EACd,KAAK,iBAAiB,QAAQ;EAC9B,KAAK,kBAAkB,QAAQ;EAC/B,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;EACzB,KAAK,qBAAqB,QAAQ;CACnC;CAED,aAAa,WACXD,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,oBACsB,EACC;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR;AAGJ,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR;AAGJ,MAAI,UAAU,UACZ,OAAM,IAAI,MACR;AAIJ,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;EAAO,EAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM;EAGlB,IAAI,aAAa;AACjB,MAAI,eAAe,OACjB,KAAI,WAAW,OACb,aAAa;WACJ,WAAW,QACpB,aAAa;WACJ,WAAW,QACpB,aAAa;OAEb,aAAa;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,WACH,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,EAAE,UAAU,CAAC,CAAC;EAG5D,IAAI;AACJ,MAAI;GACF,SAAS,MAAM,OAAO,KAAK,IAAI,UAAU;EAC1C,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;EACD,MAAM,cAAc,OAAO,OAAO,IAChC,CAACE,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,EAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,OAAO,CAACC,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,mBAAmB,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG5E,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,EAC/D,iBAAiB;EAGnB,MAAM,WAAW,CACf,GAAI,sBAAsB,CAAE,GAC5B,GAAI,uBAAuB,CAAE,CAC9B;EACD,SAAS,QAAQ,CAAC,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,CAAC,OAAO,EAAE,KAAK,2BAA2B,EAAE,YAAY,CAAC,CAAC;EAG/D,EAAC;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;EACrB;CACF;CAED,MAAM,OAA4B;EAChC,MAAMC,YAAwB,CAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,EACrC,UAAU,KAAK,IAAI;AAErB,SAAO;CACR;CAED,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,oBACD,GAAG;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM;GAElB,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAMC,UAAe,CAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,CAAE,GACxB,GAAI,mBAAmB,CAAE,CAC1B;AACD,QAAI,oBACF,YAAY,KAAK,mBAAmB;IAEtC,YAAY,QAAQ,CAAC,WAAW;KAC9B,QAAQ,UAAU,IAAI;IACvB,EAAC;IAEF,MAAM,gBACJ,kBAAkB,CAAE,GACpB,mBAAmB,CAAE,GACrB,SACA,oBACA,UACD;GACF;EACF,SAAQ,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;EAErB;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"loader.js","names":[],"sources":["../src/loader.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport PostgresEngine from \"./engine.js\";\nimport {\n textFormatter,\n csvFormatter,\n yamlFormatter,\n jsonFormatter,\n} from \"./utils/utils.js\";\n\nconst DEFAULT_METADATA_COL = \"langchain_metadata\";\ntype Row = { [key: string]: string };\n\n// Options for PostgresLoader\nexport interface PostgresLoaderOptions {\n tableName?: string;\n schemaName?: string;\n contentColumns?: string[];\n metadataColumns?: string[];\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n formatter?: (row: Row, contentColumns: string[]) => string;\n query?: string;\n metadataJsonColumn?: string | null;\n}\n\nfunction parseDocFromRow(\n contentColumns: string[],\n metadataColumns: string[],\n row: Row,\n metadataJsonColumn: string | null = DEFAULT_METADATA_COL,\n formatter: (row: Row, contentColumns: string[]) => string = textFormatter\n): Document {\n const pageContent = formatter(row, contentColumns);\n const metadata: { [key: string]: string } = {};\n\n if (metadataJsonColumn && row[metadataJsonColumn]) {\n Object.entries(row[metadataJsonColumn]).forEach(([k, v]) => {\n metadata[k] = v;\n });\n }\n\n metadataColumns.forEach((column) => {\n if (column in row && column !== metadataJsonColumn) {\n metadata[column] = row[column];\n }\n });\n\n return { pageContent, metadata };\n}\n\n/**\n * Google Cloud SQL for PostgreSQL vector store integration.\n *\n * Setup:\n * Install`@langchain/google-cloud-sql-pg`\n *\n * ```bash\n * npm install @langchain/google-cloud-sql-pg\n * ```\n *\n * <details open >\n * <summary><strong>Use with Table Name < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * tableName: \"test_table_custom\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * const documents = await documentLoaderInstance.load();\n * ```\n * </details>\n *\n * <br />\n *\n * <details open >\n * <summary><strong>Use with Query < /strong></summary >\n *\n * ```typescript\n * import { PostgresEngine, PostgresLoader } from \"@langchain/google-cloud-sql-pg\";\n *\n * const documentLoaderArgs: PostgresLoaderOptions = {\n * query: \"SELECT * FROM my_table WHERE organic = true;\",\n * contentColumns: [ \"fruit_name\", \"variety\"],\n * metadataColumns: [\"fruit_id\", \"quantity_in_stock\", \"price_per_unit\", \"organic\"],\n * format: \"text\"\n * };\n *\n * const documentLoaderInstance = await PostgresLoader.initialize(PEInstance, documentLoaderArgs);\n *\n * for await (const doc of documentLoaderInstance.lazyLoad()) {\n * console.log(doc);\n * break; // break based on required condition\n * }\n * ```\n * </details>\n *\n * <br />\n */\nexport class PostgresLoader extends BaseDocumentLoader {\n private engine: PostgresEngine;\n\n tableName?: string;\n\n schemaName?: string;\n\n contentColumns?: string[];\n\n metadataColumns?: string[];\n\n format?: \"text\" | \"json\" | \"yaml\" | \"csv\";\n\n formatter?: (row: Row, contentColumns: string[]) => string;\n\n query?: string;\n\n metadataJsonColumn?: string | null;\n\n constructor(engine: PostgresEngine, options: PostgresLoaderOptions) {\n super();\n this.engine = engine;\n this.contentColumns = options.contentColumns;\n this.metadataColumns = options.metadataColumns;\n this.query = options.query;\n this.formatter = options.formatter;\n this.metadataJsonColumn = options.metadataJsonColumn;\n }\n\n static async initialize(\n engine: PostgresEngine,\n {\n schemaName = \"public\",\n tableName,\n contentColumns,\n metadataColumns,\n format,\n query,\n formatter,\n metadataJsonColumn,\n }: PostgresLoaderOptions\n ): Promise<PostgresLoader> {\n if (tableName && query) {\n throw new Error(\n \"Only one of 'table_name' or 'query' should be specified.\"\n );\n }\n if (!tableName && !query) {\n throw new Error(\n \"At least one of the parameters 'table_name' or 'query' needs to be provided\"\n );\n }\n if (format && formatter) {\n throw new Error(\n \"Only one of 'format' or 'formatter' should be specified.\"\n );\n }\n\n if (format && ![\"csv\", \"text\", \"json\", \"yaml\"].includes(format)) {\n throw new Error(\"format must be type: 'csv', 'text', 'json', 'yaml'\");\n }\n\n let formatFunc = formatter;\n if (formatFunc === undefined) {\n if (format === \"csv\") {\n formatFunc = csvFormatter;\n } else if (format === \"yaml\") {\n formatFunc = yamlFormatter;\n } else if (format === \"json\") {\n formatFunc = jsonFormatter;\n } else {\n formatFunc = textFormatter;\n }\n }\n\n let queryStmt = query;\n if (!queryStmt) {\n queryStmt = `SELECT * FROM \"${schemaName}\".\"${tableName}\"`;\n }\n\n let result;\n try {\n result = await engine.pool.raw(queryStmt);\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n const columnNames = result.fields.map(\n (field: { name: string }) => field.name\n );\n\n const contentColumnNames = contentColumns || [columnNames[0]];\n const metadataColumnNames =\n metadataColumns ||\n columnNames.filter((col: string) => !contentColumnNames.includes(col));\n\n if (metadataJsonColumn && !columnNames.includes(metadataJsonColumn)) {\n throw new Error(\n `Column ${metadataJsonColumn} not found in query result ${columnNames}.`\n );\n }\n let jsonColumnName = metadataJsonColumn;\n if (!jsonColumnName && columnNames.includes(DEFAULT_METADATA_COL)) {\n jsonColumnName = DEFAULT_METADATA_COL;\n }\n\n const allNames = [\n ...(contentColumnNames || []),\n ...(metadataColumnNames || []),\n ];\n allNames.forEach((name) => {\n if (!columnNames.includes(name)) {\n throw new Error(\n `Column ${name} not found in query result ${columnNames}.`\n );\n }\n });\n\n return new PostgresLoader(engine, {\n contentColumns: contentColumnNames,\n metadataColumns: metadataColumnNames,\n query: queryStmt,\n formatter: formatFunc,\n metadataJsonColumn: jsonColumnName,\n });\n }\n\n async load(): Promise<Document[]> {\n const documents: Document[] = [];\n for await (const doc of this.lazyLoad()) {\n documents.push(doc);\n }\n return documents;\n }\n\n async *lazyLoad(): AsyncGenerator<Document> {\n const {\n query,\n contentColumns,\n metadataColumns,\n formatter,\n metadataJsonColumn,\n } = this;\n try {\n if (!query) {\n throw new Error(\"Query is undefined\");\n }\n const result = await this.engine.pool.raw(query);\n\n for (const row of result.rows) {\n const rowData: Row = {};\n const columnNames = [\n ...(contentColumns || []),\n ...(metadataColumns || []),\n ];\n if (metadataJsonColumn) {\n columnNames.push(metadataJsonColumn);\n }\n columnNames.forEach((column) => {\n rowData[column] = row[column];\n });\n\n yield parseDocFromRow(\n contentColumns || [],\n metadataColumns || [],\n rowData,\n metadataJsonColumn,\n formatter\n );\n }\n } catch (error) {\n if (typeof error === \"string\") {\n throw Error(error);\n }\n }\n }\n}\n"],"mappings":";;;AAUA,MAAM,uBAAuB;AAe7B,SAAS,gBACP,gBACA,iBACA,KACA,qBAAoC,sBACpC,YAA4D,eAClD;CACV,MAAM,cAAc,UAAU,KAAK,eAAe;CAClD,MAAM,WAAsC,EAAE;AAE9C,KAAI,sBAAsB,IAAI,oBAC5B,QAAO,QAAQ,IAAI,oBAAoB,CAAC,SAAS,CAAC,GAAG,OAAO;AAC1D,WAAS,KAAK;GACd;AAGJ,iBAAgB,SAAS,WAAW;AAClC,MAAI,UAAU,OAAO,WAAW,mBAC9B,UAAS,UAAU,IAAI;GAEzB;AAEF,QAAO;EAAE;EAAa;EAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0DlC,IAAa,iBAAb,MAAa,uBAAuB,mBAAmB;CACrD;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,QAAwB,SAAgC;AAClE,SAAO;AACP,OAAK,SAAS;AACd,OAAK,iBAAiB,QAAQ;AAC9B,OAAK,kBAAkB,QAAQ;AAC/B,OAAK,QAAQ,QAAQ;AACrB,OAAK,YAAY,QAAQ;AACzB,OAAK,qBAAqB,QAAQ;;CAGpC,aAAa,WACX,QACA,EACE,aAAa,UACb,WACA,gBACA,iBACA,QACA,OACA,WACA,sBAEuB;AACzB,MAAI,aAAa,MACf,OAAM,IAAI,MACR,2DACD;AAEH,MAAI,CAAC,aAAa,CAAC,MACjB,OAAM,IAAI,MACR,8EACD;AAEH,MAAI,UAAU,UACZ,OAAM,IAAI,MACR,2DACD;AAGH,MAAI,UAAU,CAAC;GAAC;GAAO;GAAQ;GAAQ;GAAO,CAAC,SAAS,OAAO,CAC7D,OAAM,IAAI,MAAM,qDAAqD;EAGvE,IAAI,aAAa;AACjB,MAAI,eAAe,KAAA,EACjB,KAAI,WAAW,MACb,cAAa;WACJ,WAAW,OACpB,cAAa;WACJ,WAAW,OACpB,cAAa;MAEb,cAAa;EAIjB,IAAI,YAAY;AAChB,MAAI,CAAC,UACH,aAAY,kBAAkB,WAAW,KAAK,UAAU;EAG1D,IAAI;AACJ,MAAI;AACF,YAAS,MAAM,OAAO,KAAK,IAAI,UAAU;WAClC,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM;;EAGtB,MAAM,cAAc,OAAO,OAAO,KAC/B,UAA4B,MAAM,KACpC;EAED,MAAM,qBAAqB,kBAAkB,CAAC,YAAY,GAAG;EAC7D,MAAM,sBACJ,mBACA,YAAY,QAAQ,QAAgB,CAAC,mBAAmB,SAAS,IAAI,CAAC;AAExE,MAAI,sBAAsB,CAAC,YAAY,SAAS,mBAAmB,CACjE,OAAM,IAAI,MACR,UAAU,mBAAmB,6BAA6B,YAAY,GACvE;EAEH,IAAI,iBAAiB;AACrB,MAAI,CAAC,kBAAkB,YAAY,SAAS,qBAAqB,CAC/D,kBAAiB;AAGF,GACf,GAAI,sBAAsB,EAAE,EAC5B,GAAI,uBAAuB,EAAE,CAC9B,CACQ,SAAS,SAAS;AACzB,OAAI,CAAC,YAAY,SAAS,KAAK,CAC7B,OAAM,IAAI,MACR,UAAU,KAAK,6BAA6B,YAAY,GACzD;IAEH;AAEF,SAAO,IAAI,eAAe,QAAQ;GAChC,gBAAgB;GAChB,iBAAiB;GACjB,OAAO;GACP,WAAW;GACX,oBAAoB;GACrB,CAAC;;CAGJ,MAAM,OAA4B;EAChC,MAAM,YAAwB,EAAE;AAChC,aAAW,MAAM,OAAO,KAAK,UAAU,CACrC,WAAU,KAAK,IAAI;AAErB,SAAO;;CAGT,OAAO,WAAqC;EAC1C,MAAM,EACJ,OACA,gBACA,iBACA,WACA,uBACE;AACJ,MAAI;AACF,OAAI,CAAC,MACH,OAAM,IAAI,MAAM,qBAAqB;GAEvC,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,IAAI,MAAM;AAEhD,QAAK,MAAM,OAAO,OAAO,MAAM;IAC7B,MAAM,UAAe,EAAE;IACvB,MAAM,cAAc,CAClB,GAAI,kBAAkB,EAAE,EACxB,GAAI,mBAAmB,EAAE,CAC1B;AACD,QAAI,mBACF,aAAY,KAAK,mBAAmB;AAEtC,gBAAY,SAAS,WAAW;AAC9B,aAAQ,UAAU,IAAI;MACtB;AAEF,UAAM,gBACJ,kBAAkB,EAAE,EACpB,mBAAmB,EAAE,EACrB,SACA,oBACA,UACD;;WAEI,OAAO;AACd,OAAI,OAAO,UAAU,SACnB,OAAM,MAAM,MAAM"}
|
package/dist/utils/utils.cjs
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
|
|
2
1
|
//#region src/utils/utils.ts
|
|
3
2
|
/**
|
|
4
3
|
* Get email address associated with current authenticated IAM principal.
|
|
@@ -38,7 +37,6 @@ function jsonFormatter(row, content_columns) {
|
|
|
38
37
|
for (const column of content_columns) if (column in row) dictionary[column] = row[column];
|
|
39
38
|
return JSON.stringify(dictionary);
|
|
40
39
|
}
|
|
41
|
-
|
|
42
40
|
//#endregion
|
|
43
41
|
exports.csvFormatter = csvFormatter;
|
|
44
42
|
exports.customZip = customZip;
|
|
@@ -46,4 +44,5 @@ exports.getIAMPrincipalEmail = getIAMPrincipalEmail;
|
|
|
46
44
|
exports.jsonFormatter = jsonFormatter;
|
|
47
45
|
exports.textFormatter = textFormatter;
|
|
48
46
|
exports.yamlFormatter = yamlFormatter;
|
|
47
|
+
|
|
49
48
|
//# sourceMappingURL=utils.cjs.map
|
package/dist/utils/utils.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.cjs","names":[
|
|
1
|
+
{"version":3,"file":"utils.cjs","names":[],"sources":["../../src/utils/utils.ts"],"sourcesContent":["import { GoogleAuth } from \"google-auth-library\";\n\n/**\n * Get email address associated with current authenticated IAM principal.\n * Email will be used for automatic IAM database authentication to Cloud SQL.\n *\n * @param {GoogleAuth} auth - object to use in finding the associated IAM principal email address.\n * @returns {string} email - email address associated with the current authenticated IAM principal\n */\nexport const getIAMPrincipalEmail = async (\n auth: GoogleAuth\n): Promise<string> => {\n const credentials = await auth.getCredentials();\n\n if (\"client_email\" in credentials && credentials.client_email !== undefined) {\n return credentials.client_email.replace(\".gserviceaccount.com\", \"\");\n }\n\n const accessToken = await auth.getAccessToken();\n const client = await auth.getClient();\n\n const url = `https://oauth2.googleapis.com/tokeninfo?access_token=${accessToken}`;\n const clientResponse = await client\n .request({ url })\n // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n .then((res: { data: any }) => res.data);\n\n if (!(\"email\" in clientResponse)) {\n throw new Error(\n \"Failed to automatically obtain authenticated IAM principal's \" +\n \"email address using environment's ADC credentials!\"\n );\n }\n const { email } = clientResponse;\n return email.replace(\".gserviceaccount.com\", \"\");\n};\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const customZip = (...arrays: any[]) => {\n const minLength = Math.min(...arrays.map((arr) => arr.length));\n const result = [];\n for (let i = 0; i < minLength; i += 1) {\n result.push(arrays.map((arr) => arr[i]));\n }\n return result;\n};\n\n/*\n Formatter functions\n*/\n\n// txt document formatter.\nexport function textFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\" \");\n}\n\n// CSV document formatter.\nexport function csvFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => String(row[column]))\n .join(\", \");\n}\n\n// YAML document formatter\nexport function yamlFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n return content_columns\n .filter((column) => column in row)\n .map((column) => `${column}: ${String(row[column])}`)\n .join(\"\\n\");\n}\n\n// JSON document formatter\nexport function jsonFormatter(\n row: { [key: string]: string },\n content_columns: string[]\n): string {\n const dictionary: { [key: string]: string } = {};\n for (const column of content_columns) {\n if (column in row) {\n dictionary[column] = row[column];\n }\n }\n return JSON.stringify(dictionary);\n}\n"],"mappings":";;;;;;;;AASA,MAAa,uBAAuB,OAClC,SACoB;CACpB,MAAM,cAAc,MAAM,KAAK,gBAAgB;AAE/C,KAAI,kBAAkB,eAAe,YAAY,iBAAiB,KAAA,EAChE,QAAO,YAAY,aAAa,QAAQ,wBAAwB,GAAG;CAGrE,MAAM,cAAc,MAAM,KAAK,gBAAgB;CAC/C,MAAM,SAAS,MAAM,KAAK,WAAW;CAErC,MAAM,MAAM,wDAAwD;CACpE,MAAM,iBAAiB,MAAM,OAC1B,QAAQ,EAAE,KAAK,CAAC,CAEhB,MAAM,QAAuB,IAAI,KAAK;AAEzC,KAAI,EAAE,WAAW,gBACf,OAAM,IAAI,MACR,kHAED;CAEH,MAAM,EAAE,UAAU;AAClB,QAAO,MAAM,QAAQ,wBAAwB,GAAG;;AAIlD,MAAa,aAAa,GAAG,WAAkB;CAC7C,MAAM,YAAY,KAAK,IAAI,GAAG,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC;CAC9D,MAAM,SAAS,EAAE;AACjB,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK,EAClC,QAAO,KAAK,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC;AAE1C,QAAO;;AAQT,SAAgB,cACd,KACA,iBACQ;AACR,QAAO,gBACJ,QAAQ,WAAW,UAAU,IAAI,CACjC,KAAK,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,IAAI;;AAId,SAAgB,aACd,KACA,iBACQ;AACR,QAAO,gBACJ,QAAQ,WAAW,UAAU,IAAI,CACjC,KAAK,WAAW,OAAO,IAAI,QAAQ,CAAC,CACpC,KAAK,KAAK;;AAIf,SAAgB,cACd,KACA,iBACQ;AACR,QAAO,gBACJ,QAAQ,WAAW,UAAU,IAAI,CACjC,KAAK,WAAW,GAAG,OAAO,IAAI,OAAO,IAAI,QAAQ,GAAG,CACpD,KAAK,KAAK;;AAIf,SAAgB,cACd,KACA,iBACQ;CACR,MAAM,aAAwC,EAAE;AAChD,MAAK,MAAM,UAAU,gBACnB,KAAI,UAAU,IACZ,YAAW,UAAU,IAAI;AAG7B,QAAO,KAAK,UAAU,WAAW"}
|
package/dist/utils/utils.js
CHANGED
|
@@ -37,7 +37,7 @@ function jsonFormatter(row, content_columns) {
|
|
|
37
37
|
for (const column of content_columns) if (column in row) dictionary[column] = row[column];
|
|
38
38
|
return JSON.stringify(dictionary);
|
|
39
39
|
}
|
|
40
|
-
|
|
41
40
|
//#endregion
|
|
42
41
|
export { csvFormatter, customZip, getIAMPrincipalEmail, jsonFormatter, textFormatter, yamlFormatter };
|
|
42
|
+
|
|
43
43
|
//# sourceMappingURL=utils.js.map
|