@owox/connectors 0.14.0-next-20251127124948 → 0.14.0-next-20251128101118

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@
3
3
  const OWOX = require("@owox/connectors");
4
4
  const AdmZip = require("adm-zip");
5
5
  const { BigQuery } = require("@google-cloud/bigquery");
6
+ const snowflake = require("snowflake-sdk");
6
7
  const {
7
8
  AthenaClient,
8
9
  StartQueryExecutionCommand,
@@ -20,6 +21,7 @@ const { Upload } = require("@aws-sdk/lib-storage");
20
21
  global.OWOX = OWOX;
21
22
  global.AdmZip = AdmZip;
22
23
  global.BigQuery = BigQuery;
24
+ global.snowflake = snowflake;
23
25
  global.AthenaClient = AthenaClient;
24
26
  global.StartQueryExecutionCommand = StartQueryExecutionCommand;
25
27
  global.GetQueryExecutionCommand = GetQueryExecutionCommand;
@@ -2,6 +2,7 @@
2
2
  const OWOX = require("@owox/connectors");
3
3
  const AdmZip = require("adm-zip");
4
4
  const { BigQuery } = require("@google-cloud/bigquery");
5
+ const snowflake = require("snowflake-sdk");
5
6
  const {
6
7
  AthenaClient,
7
8
  StartQueryExecutionCommand,
@@ -19,6 +20,7 @@ const { Upload } = require("@aws-sdk/lib-storage");
19
20
  global.OWOX = OWOX;
20
21
  global.AdmZip = AdmZip;
21
22
  global.BigQuery = BigQuery;
23
+ global.snowflake = snowflake;
22
24
  global.AthenaClient = AthenaClient;
23
25
  global.StartQueryExecutionCommand = StartQueryExecutionCommand;
24
26
  global.GetQueryExecutionCommand = GetQueryExecutionCommand;
package/dist/index.cjs CHANGED
@@ -1677,6 +1677,8 @@ class StorageConfigDto {
1677
1677
  return "GoogleBigQuery";
1678
1678
  } else if (name === "AWS_ATHENA") {
1679
1679
  return "AwsAthena";
1680
+ } else if (name === "SNOWFLAKE") {
1681
+ return "Snowflake";
1680
1682
  }
1681
1683
  return name;
1682
1684
  }
@@ -1989,6 +1991,554 @@ const Core = {
1989
1991
  OAUTH_CONSTANTS,
1990
1992
  OAUTH_SOURCE_CREDENTIALS_KEY
1991
1993
  };
1994
+ const Snowflake = (function() {
1995
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
1996
+ function quoteIdentifier(identifier) {
1997
+ if (!identifier) return identifier;
1998
+ if (identifier.startsWith('"') && identifier.endsWith('"')) {
1999
+ return identifier;
2000
+ }
2001
+ return `"${identifier}"`;
2002
+ }
2003
+ var SnowflakeStorage = class SnowflakeStorage extends AbstractStorage2 {
2004
+ //---- constructor -------------------------------------------------
2005
+ /**
2006
+ * Snowflake storage operations class
2007
+ *
2008
+ * @param config (object) instance of AbstractConfig
2009
+ * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2010
+ * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2011
+ * @param description (string) string with storage description }
2012
+ */
2013
+ constructor(config, uniqueKeyColumns, schema = null, description = null) {
2014
+ super(
2015
+ config.mergeParameters({
2016
+ SnowflakeAccount: {
2017
+ isRequired: true,
2018
+ requiredType: "string"
2019
+ },
2020
+ SnowflakeWarehouse: {
2021
+ isRequired: true,
2022
+ requiredType: "string"
2023
+ },
2024
+ SnowflakeDatabase: {
2025
+ isRequired: true,
2026
+ requiredType: "string"
2027
+ },
2028
+ SnowflakeSchema: {
2029
+ isRequired: true,
2030
+ requiredType: "string"
2031
+ },
2032
+ SnowflakeRole: {
2033
+ isRequired: false,
2034
+ requiredType: "string",
2035
+ default: null
2036
+ },
2037
+ SnowflakeUsername: {
2038
+ isRequired: true,
2039
+ requiredType: "string"
2040
+ },
2041
+ SnowflakePassword: {
2042
+ isRequired: true,
2043
+ requiredType: "string"
2044
+ },
2045
+ SnowflakeAuthenticator: {
2046
+ isRequired: false,
2047
+ requiredType: "string",
2048
+ default: "SNOWFLAKE"
2049
+ },
2050
+ SnowflakePrivateKey: {
2051
+ isRequired: false,
2052
+ requiredType: "string",
2053
+ default: null
2054
+ },
2055
+ SnowflakePrivateKeyPassphrase: {
2056
+ isRequired: false,
2057
+ requiredType: "string",
2058
+ default: null
2059
+ },
2060
+ DestinationTableName: {
2061
+ isRequired: true,
2062
+ requiredType: "string",
2063
+ default: "Data"
2064
+ },
2065
+ MaxBufferSize: {
2066
+ isRequired: true,
2067
+ default: 250
2068
+ }
2069
+ }),
2070
+ uniqueKeyColumns,
2071
+ schema,
2072
+ description
2073
+ );
2074
+ this.updatedRecordsBuffer = {};
2075
+ this.totalRecordsProcessed = 0;
2076
+ this.connection = null;
2077
+ }
2078
+ //---- init --------------------------------------------------------
2079
+ /**
2080
+ * Initializing storage - establishes connection and creates table if needed
2081
+ */
2082
+ async init() {
2083
+ this.checkIfSnowflakeIsConnected();
2084
+ await this.createConnection();
2085
+ await this.testConnection();
2086
+ await this.loadTableSchema();
2087
+ }
2088
+ //----------------------------------------------------------------
2089
+ //---- checkIfSnowflakeIsConnected ---------------------------------
2090
+ checkIfSnowflakeIsConnected() {
2091
+ if (typeof snowflake == "undefined") {
2092
+ throw new Error(`Snowflake SDK is not available. Ensure snowflake-sdk is installed.`);
2093
+ }
2094
+ }
2095
+ //----------------------------------------------------------------
2096
+ //---- createConnection --------------------------------------------
2097
+ /**
2098
+ * Creates and connects to Snowflake
2099
+ */
2100
+ async createConnection() {
2101
+ snowflake.configure({
2102
+ logLevel: "OFF"
2103
+ });
2104
+ const connectionConfig = {
2105
+ account: this.config.SnowflakeAccount.value,
2106
+ username: this.config.SnowflakeUsername.value,
2107
+ password: this.config.SnowflakePassword.value,
2108
+ warehouse: this.config.SnowflakeWarehouse.value,
2109
+ database: this.config.SnowflakeDatabase.value,
2110
+ schema: this.config.SnowflakeSchema.value,
2111
+ authenticator: this.config.SnowflakeAuthenticator.value || "SNOWFLAKE"
2112
+ };
2113
+ if (this.config.SnowflakeRole && this.config.SnowflakeRole.value) {
2114
+ connectionConfig.role = this.config.SnowflakeRole.value;
2115
+ }
2116
+ if (this.config.SnowflakePrivateKey && this.config.SnowflakePrivateKey.value) {
2117
+ connectionConfig.privateKey = this.config.SnowflakePrivateKey.value;
2118
+ if (this.config.SnowflakePrivateKeyPassphrase && this.config.SnowflakePrivateKeyPassphrase.value) {
2119
+ connectionConfig.privateKeyPassphrase = this.config.SnowflakePrivateKeyPassphrase.value;
2120
+ }
2121
+ }
2122
+ this.connection = snowflake.createConnection(connectionConfig);
2123
+ return new Promise((resolve, reject) => {
2124
+ this.connection.connect((err, conn) => {
2125
+ if (err) {
2126
+ reject(new Error(`Failed to connect to Snowflake: ${err.message}`));
2127
+ } else {
2128
+ this.config.logMessage(`Connected to Snowflake (account: ${this.config.SnowflakeAccount.value})`);
2129
+ resolve(conn);
2130
+ }
2131
+ });
2132
+ });
2133
+ }
2134
+ //----------------------------------------------------------------
2135
+ //---- testConnection ----------------------------------------------
2136
+ /**
2137
+ * Tests the connection with a simple query
2138
+ */
2139
+ async testConnection() {
2140
+ try {
2141
+ await this.executeQuery("SELECT 1 as test");
2142
+ this.config.logMessage("Snowflake connection established successfully");
2143
+ } catch (error) {
2144
+ throw new Error(`Snowflake connection test failed: ${error.message}`);
2145
+ }
2146
+ }
2147
+ //----------------------------------------------------------------
2148
+ //---- loadTableSchema ---------------------------------------------
2149
+ /**
2150
+ * Loads existing table schema from Snowflake
2151
+ */
2152
+ async loadTableSchema() {
2153
+ this.existingColumns = await this.getAListOfExistingColumns() || {};
2154
+ if (Object.keys(this.existingColumns).length == 0) {
2155
+ await this.createDatabaseAndSchemaIfNotExist();
2156
+ this.existingColumns = await this.createTableIfItDoesntExist();
2157
+ } else {
2158
+ let selectedFields = this.getSelectedFields();
2159
+ let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
2160
+ if (newFields.length > 0) {
2161
+ await this.addNewColumns(newFields);
2162
+ }
2163
+ }
2164
+ }
2165
+ //----------------------------------------------------------------
2166
+ //---- getAListOfExistingColumns -----------------------------------
2167
+ /**
2168
+ * Reads columns list of the table and returns it as object
2169
+ *
2170
+ * @return columns (object)
2171
+ */
2172
+ async getAListOfExistingColumns() {
2173
+ let query = `SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE
2174
+ FROM ${this.config.SnowflakeDatabase.value}.INFORMATION_SCHEMA.COLUMNS
2175
+ WHERE TABLE_CATALOG = '${this.config.SnowflakeDatabase.value}'
2176
+ AND TABLE_SCHEMA = UPPER('${this.config.SnowflakeSchema.value}')
2177
+ AND TABLE_NAME = UPPER('${this.config.DestinationTableName.value}')
2178
+ ORDER BY ORDINAL_POSITION`;
2179
+ let queryResults = [];
2180
+ try {
2181
+ queryResults = await this.executeQuery(query);
2182
+ } catch (error) {
2183
+ if (error.message && error.message.includes("does not exist")) {
2184
+ return {};
2185
+ }
2186
+ throw error;
2187
+ }
2188
+ let columns = {};
2189
+ if (Array.isArray(queryResults)) {
2190
+ queryResults.forEach((row) => {
2191
+ const columnName = row.COLUMN_NAME;
2192
+ const dataType = row.DATA_TYPE;
2193
+ columns[columnName] = { "name": columnName, "type": dataType };
2194
+ });
2195
+ }
2196
+ return columns;
2197
+ }
2198
+ //----------------------------------------------------------------
2199
+ //---- createDatabaseAndSchemaIfNotExist ---------------------------
2200
+ async createDatabaseAndSchemaIfNotExist() {
2201
+ let createDbQuery = `CREATE DATABASE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}`;
2202
+ await this.executeQuery(createDbQuery);
2203
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2204
+ let createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}`;
2205
+ await this.executeQuery(createSchemaQuery);
2206
+ this.config.logMessage(`Database and schema ensured: ${this.config.SnowflakeDatabase.value}.${quotedSchema}`);
2207
+ }
2208
+ //----------------------------------------------------------------
2209
+ //---- createTableIfItDoesntExist ----------------------------------
2210
+ async createTableIfItDoesntExist() {
2211
+ let columns = [];
2212
+ let existingColumns = {};
2213
+ let selectedFields = this.getSelectedFields();
2214
+ let tableColumns = selectedFields.length > 0 ? selectedFields : this.uniqueKeyColumns;
2215
+ for (let i in tableColumns) {
2216
+ let columnName = tableColumns[i];
2217
+ let columnDescription = "";
2218
+ if (!(columnName in this.schema)) {
2219
+ throw new Error(`Required field ${columnName} not found in schema`);
2220
+ }
2221
+ let columnType = this.getColumnType(columnName);
2222
+ if ("description" in this.schema[columnName]) {
2223
+ columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
2224
+ }
2225
+ columns.push(`"${columnName}" ${columnType}${columnDescription}`);
2226
+ existingColumns[columnName] = { "name": columnName, "type": columnType };
2227
+ }
2228
+ columns.push(`PRIMARY KEY (${this.uniqueKeyColumns.map((col) => `"${col}"`).join(",")}) NOT ENFORCED`);
2229
+ let columnsStr = columns.join(",\n ");
2230
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2231
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2232
+ let query = `CREATE TABLE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} (
2233
+ ${columnsStr}
2234
+ )`;
2235
+ if (this.description) {
2236
+ query += `
2237
+ COMMENT = '${this.description}'`;
2238
+ }
2239
+ await this.executeQuery(query);
2240
+ this.config.logMessage(`Table ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} was created`);
2241
+ return existingColumns;
2242
+ }
2243
+ //----------------------------------------------------------------
2244
+ //---- addNewColumns -----------------------------------------------
2245
+ /**
2246
+ * ALTER table by adding missed columns
2247
+ *
2248
+ * @param {newColumns} array with a list of new columns
2249
+ */
2250
+ async addNewColumns(newColumns) {
2251
+ let columns = [];
2252
+ for (var i in newColumns) {
2253
+ let columnName = newColumns[i];
2254
+ if (columnName in this.schema) {
2255
+ let columnDescription = "";
2256
+ let columnType = this.getColumnType(columnName);
2257
+ if ("description" in this.schema[columnName]) {
2258
+ columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
2259
+ }
2260
+ columns.push(`ADD COLUMN IF NOT EXISTS "${columnName}" ${columnType}${columnDescription}`);
2261
+ this.existingColumns[columnName] = { "name": columnName, "type": columnType };
2262
+ }
2263
+ }
2264
+ if (columns.length > 0) {
2265
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2266
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2267
+ let query = `ALTER TABLE ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}
2268
+ `;
2269
+ query += columns.join(",\n");
2270
+ await this.executeQuery(query);
2271
+ this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`);
2272
+ }
2273
+ }
2274
+ //----------------------------------------------------------------
2275
+ //---- saveData ----------------------------------------------------
2276
+ /**
2277
+ * Saving data to storage
2278
+ * @param {data} array of assoc objects with records to save
2279
+ */
2280
+ async saveData(data) {
2281
+ for (const row of data) {
2282
+ let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
2283
+ if (newFields.length > 0) {
2284
+ await this.addNewColumns(newFields);
2285
+ }
2286
+ this.addRecordToBuffer(row);
2287
+ await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2288
+ }
2289
+ await this.saveRecordsAddedToBuffer();
2290
+ }
2291
+ //----------------------------------------------------------------
2292
+ //---- addRecordToBuffer -------------------------------------------
2293
+ /**
2294
+ * Adds record to buffer with deduplication
2295
+ * @param {record} object
2296
+ */
2297
+ addRecordToBuffer(record) {
2298
+ let uniqueKey = this.getUniqueKeyByRecordFields(record);
2299
+ this.updatedRecordsBuffer[uniqueKey] = record;
2300
+ }
2301
+ //----------------------------------------------------------------
2302
+ //---- saveRecordsAddedToBuffer ------------------------------------
2303
+ /**
2304
+ * Add records from buffer to storage
2305
+ * @param (integer) {maxBufferSize} records will be added only if buffer size is larger than this parameter
2306
+ */
2307
+ async saveRecordsAddedToBuffer(maxBufferSize = 0) {
2308
+ let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
2309
+ if (bufferSize && bufferSize >= maxBufferSize) {
2310
+ await this.executeQueryWithSizeLimit();
2311
+ }
2312
+ }
2313
+ //----------------------------------------------------------------
2314
+ //---- executeQueryWithSizeLimit -----------------------------------
2315
+ /**
2316
+ * Executes the MERGE query with automatic batching for large datasets
2317
+ */
2318
+ async executeQueryWithSizeLimit() {
2319
+ const bufferKeys = Object.keys(this.updatedRecordsBuffer);
2320
+ const totalRecords = bufferKeys.length;
2321
+ if (totalRecords === 0) {
2322
+ return;
2323
+ }
2324
+ await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
2325
+ this.updatedRecordsBuffer = {};
2326
+ }
2327
+ //----------------------------------------------------------------
2328
+ //---- executeMergeQueryRecursively --------------------------------
2329
+ /**
2330
+ * Recursively attempts to execute MERGE queries, reducing batch size if needed
2331
+ * @param {Array} recordKeys - Array of record keys to process
2332
+ * @param {number} batchSize - Current batch size to attempt
2333
+ */
2334
+ async executeMergeQueryRecursively(recordKeys, batchSize) {
2335
+ if (recordKeys.length === 0) {
2336
+ return;
2337
+ }
2338
+ if (batchSize < 1) {
2339
+ throw new Error("Cannot process records: batch size reduced below 1");
2340
+ }
2341
+ const currentBatch = recordKeys.slice(0, batchSize);
2342
+ const remainingRecords = recordKeys.slice(batchSize);
2343
+ const query = this.buildMergeQuery(currentBatch);
2344
+ try {
2345
+ await this.executeQuery(query);
2346
+ this.totalRecordsProcessed += currentBatch.length;
2347
+ if (remainingRecords.length > 0) {
2348
+ await this.executeMergeQueryRecursively(remainingRecords, batchSize);
2349
+ }
2350
+ } catch (error) {
2351
+ if (batchSize > 1) {
2352
+ await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
2353
+ } else {
2354
+ throw error;
2355
+ }
2356
+ }
2357
+ }
2358
+ //----------------------------------------------------------------
2359
+ //---- buildMergeQuery ---------------------------------------------
2360
+ /**
2361
+ * Builds a MERGE query for the specified record keys
2362
+ * @param {Array} recordKeys - Array of record keys to include in the query
2363
+ * @return {string} - The constructed MERGE query
2364
+ */
2365
+ buildMergeQuery(recordKeys) {
2366
+ let rows = [];
2367
+ for (let i = 0; i < recordKeys.length; i++) {
2368
+ const key = recordKeys[i];
2369
+ let record = this.stringifyNeastedFields(this.updatedRecordsBuffer[key]);
2370
+ let fields = [];
2371
+ for (var j in this.existingColumns) {
2372
+ let columnName = this.existingColumns[j]["name"];
2373
+ let columnType = this.existingColumns[j]["type"];
2374
+ let columnValue = null;
2375
+ if (record[columnName] === void 0 || record[columnName] === null) {
2376
+ columnValue = null;
2377
+ } else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
2378
+ columnValue = DateUtils3.formatDate(record[columnName]);
2379
+ } else if ((columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") && record[columnName] instanceof Date) {
2380
+ const isoString = record[columnName].toISOString();
2381
+ columnValue = isoString.replace("T", " ").substring(0, 19);
2382
+ } else {
2383
+ columnValue = this.obfuscateSpecialCharacters(record[columnName]);
2384
+ }
2385
+ if (columnValue === null) {
2386
+ fields.push(`CAST(NULL AS ${columnType}) AS "${columnName}"`);
2387
+ } else if (columnType.toUpperCase() == "DATE") {
2388
+ fields.push(`TO_DATE('${columnValue}', 'YYYY-MM-DD') AS "${columnName}"`);
2389
+ } else if (columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") {
2390
+ fields.push(`TO_TIMESTAMP('${columnValue}', 'YYYY-MM-DD HH24:MI:SS') AS "${columnName}"`);
2391
+ } else {
2392
+ fields.push(`CAST('${columnValue}' AS ${columnType}) AS "${columnName}"`);
2393
+ }
2394
+ }
2395
+ rows.push(`SELECT ${fields.join(",\n ")}`);
2396
+ }
2397
+ let existingColumnsNames = Object.keys(this.existingColumns);
2398
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2399
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2400
+ let fullTableName = `${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`;
2401
+ let query = `MERGE INTO ${fullTableName} AS target
2402
+ USING (
2403
+ ${rows.join("\n UNION ALL\n ")}
2404
+ ) AS source
2405
+
2406
+ ON ${this.uniqueKeyColumns.map((item) => `target."${item}" = source."${item}"`).join("\n AND ")}
2407
+
2408
+ WHEN MATCHED THEN
2409
+ UPDATE SET
2410
+ ${existingColumnsNames.map((item) => `target."${item}" = source."${item}"`).join(",\n ")}
2411
+ WHEN NOT MATCHED THEN
2412
+ INSERT (
2413
+ ${existingColumnsNames.map((item) => `"${item}"`).join(", ")}
2414
+ )
2415
+ VALUES (
2416
+ ${existingColumnsNames.map((item) => `source."${item}"`).join(", ")}
2417
+ )`;
2418
+ return query;
2419
+ }
2420
+ //----------------------------------------------------------------
2421
+ //---- executeQuery ------------------------------------------------
2422
+ /**
2423
+ * Executes Snowflake Query and returns a result
2424
+ *
2425
+ * @param {query} string
2426
+ * @return Promise<Array>
2427
+ */
2428
+ async executeQuery(query) {
2429
+ if (!this.connection) {
2430
+ throw new Error("Snowflake connection not initialized");
2431
+ }
2432
+ return new Promise((resolve, reject) => {
2433
+ this.connection.execute({
2434
+ sqlText: query,
2435
+ complete: (err, stmt, rows) => {
2436
+ if (err) {
2437
+ reject(new Error(`Snowflake query failed: ${err.message}`));
2438
+ } else {
2439
+ resolve(rows || []);
2440
+ }
2441
+ }
2442
+ });
2443
+ });
2444
+ }
2445
+ //----------------------------------------------------------------
2446
+ //---- obfuscateSpecialCharacters ----------------------------------
2447
+ /**
2448
+ * Escape special characters for SQL string literals
2449
+ * @param {string} inputString - String to escape
2450
+ * @return {string} - Escaped string
2451
+ */
2452
+ obfuscateSpecialCharacters(inputString) {
2453
+ return String(inputString).replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/"/g, '\\"').replace(/[\x00-\x1F]/g, " ");
2454
+ }
2455
+ //----------------------------------------------------------------
2456
+ //---- getColumnType -----------------------------------------------
2457
+ /**
2458
+ * Get column type for Snowflake from schema
2459
+ * @param {string} columnName - Name of the column
2460
+ * @returns {string} Snowflake column type
2461
+ */
2462
+ getColumnType(columnName) {
2463
+ var _a;
2464
+ return this.schema[columnName]["SnowflakeFieldType"] || this._convertTypeToStorageType((_a = this.schema[columnName]["type"]) == null ? void 0 : _a.toLowerCase());
2465
+ }
2466
+ //----------------------------------------------------------------
2467
+ //---- _convertTypeToStorageType -----------------------------------
2468
+ /**
2469
+ * Converts generic type to Snowflake-specific type
2470
+ * @param {string} genericType - Generic type from schema
2471
+ * @returns {string} Snowflake column type
2472
+ */
2473
+ _convertTypeToStorageType(genericType) {
2474
+ if (!genericType) return "VARCHAR";
2475
+ switch (genericType.toLowerCase()) {
2476
+ // Integer types
2477
+ case "integer":
2478
+ case "int32":
2479
+ return "INTEGER";
2480
+ case "int64":
2481
+ case "long":
2482
+ return "BIGINT";
2483
+ // Float types
2484
+ case "float":
2485
+ case "number":
2486
+ case "double":
2487
+ return "FLOAT";
2488
+ case "decimal":
2489
+ return "NUMERIC";
2490
+ // Boolean types
2491
+ case "bool":
2492
+ case "boolean":
2493
+ return "BOOLEAN";
2494
+ // Date/time types
2495
+ case "date":
2496
+ return "DATE";
2497
+ case "datetime":
2498
+ return "TIMESTAMP_NTZ";
2499
+ // Timezone-naive
2500
+ case "timestamp":
2501
+ return "TIMESTAMP_TZ";
2502
+ // Timezone-aware
2503
+ // JSON/Object types
2504
+ case "json":
2505
+ case "object":
2506
+ case "array":
2507
+ return "VARIANT";
2508
+ // Default to VARCHAR for unknown types
2509
+ default:
2510
+ return "VARCHAR";
2511
+ }
2512
+ }
2513
+ //----------------------------------------------------------------
2514
+ };
2515
+ const manifest = {
2516
+ "name": "SnowflakeStorage",
2517
+ "description": "Storage for Snowflake Data Warehouse",
2518
+ "title": "Snowflake",
2519
+ "version": "0.0.0",
2520
+ "author": "OWOX, Inc.",
2521
+ "license": "MIT",
2522
+ "environment": {
2523
+ "node": {
2524
+ "enabled": true,
2525
+ "dependencies": [
2526
+ {
2527
+ "name": "snowflake-sdk",
2528
+ "version": "^1.6.20",
2529
+ "global": [
2530
+ "snowflake"
2531
+ ]
2532
+ }
2533
+ ]
2534
+ }
2535
+ }
2536
+ };
2537
+ return {
2538
+ SnowflakeStorage,
2539
+ manifest
2540
+ };
2541
+ })();
1992
2542
  const GoogleBigQuery = (function() {
1993
2543
  const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
1994
2544
  var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
@@ -3101,6 +3651,7 @@ const AwsAthena = (function() {
3101
3651
  };
3102
3652
  })();
3103
3653
  const Storages = {
3654
+ Snowflake,
3104
3655
  GoogleBigQuery,
3105
3656
  AwsAthena
3106
3657
  };
@@ -22040,6 +22591,7 @@ const AvailableConnectors = [
22040
22591
  "BankOfCanada"
22041
22592
  ];
22042
22593
  const AvailableStorages = [
22594
+ "Snowflake",
22043
22595
  "GoogleBigQuery",
22044
22596
  "AwsAthena"
22045
22597
  ];
@@ -22064,6 +22616,7 @@ const OWOX = {
22064
22616
  CriteoAds,
22065
22617
  BankOfCanada,
22066
22618
  // Individual storages
22619
+ Snowflake,
22067
22620
  GoogleBigQuery,
22068
22621
  AwsAthena
22069
22622
  };
package/dist/index.js CHANGED
@@ -1675,6 +1675,8 @@ class StorageConfigDto {
1675
1675
  return "GoogleBigQuery";
1676
1676
  } else if (name === "AWS_ATHENA") {
1677
1677
  return "AwsAthena";
1678
+ } else if (name === "SNOWFLAKE") {
1679
+ return "Snowflake";
1678
1680
  }
1679
1681
  return name;
1680
1682
  }
@@ -1987,6 +1989,554 @@ const Core = {
1987
1989
  OAUTH_CONSTANTS,
1988
1990
  OAUTH_SOURCE_CREDENTIALS_KEY
1989
1991
  };
1992
+ const Snowflake = (function() {
1993
+ const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
1994
+ function quoteIdentifier(identifier) {
1995
+ if (!identifier) return identifier;
1996
+ if (identifier.startsWith('"') && identifier.endsWith('"')) {
1997
+ return identifier;
1998
+ }
1999
+ return `"${identifier}"`;
2000
+ }
2001
+ var SnowflakeStorage = class SnowflakeStorage extends AbstractStorage2 {
2002
+ //---- constructor -------------------------------------------------
2003
+ /**
2004
+ * Snowflake storage operations class
2005
+ *
2006
+ * @param config (object) instance of AbstractConfig
2007
+ * @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
2008
+ * @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
2009
+ * @param description (string) string with storage description }
2010
+ */
2011
+ constructor(config, uniqueKeyColumns, schema = null, description = null) {
2012
+ super(
2013
+ config.mergeParameters({
2014
+ SnowflakeAccount: {
2015
+ isRequired: true,
2016
+ requiredType: "string"
2017
+ },
2018
+ SnowflakeWarehouse: {
2019
+ isRequired: true,
2020
+ requiredType: "string"
2021
+ },
2022
+ SnowflakeDatabase: {
2023
+ isRequired: true,
2024
+ requiredType: "string"
2025
+ },
2026
+ SnowflakeSchema: {
2027
+ isRequired: true,
2028
+ requiredType: "string"
2029
+ },
2030
+ SnowflakeRole: {
2031
+ isRequired: false,
2032
+ requiredType: "string",
2033
+ default: null
2034
+ },
2035
+ SnowflakeUsername: {
2036
+ isRequired: true,
2037
+ requiredType: "string"
2038
+ },
2039
+ SnowflakePassword: {
2040
+ isRequired: true,
2041
+ requiredType: "string"
2042
+ },
2043
+ SnowflakeAuthenticator: {
2044
+ isRequired: false,
2045
+ requiredType: "string",
2046
+ default: "SNOWFLAKE"
2047
+ },
2048
+ SnowflakePrivateKey: {
2049
+ isRequired: false,
2050
+ requiredType: "string",
2051
+ default: null
2052
+ },
2053
+ SnowflakePrivateKeyPassphrase: {
2054
+ isRequired: false,
2055
+ requiredType: "string",
2056
+ default: null
2057
+ },
2058
+ DestinationTableName: {
2059
+ isRequired: true,
2060
+ requiredType: "string",
2061
+ default: "Data"
2062
+ },
2063
+ MaxBufferSize: {
2064
+ isRequired: true,
2065
+ default: 250
2066
+ }
2067
+ }),
2068
+ uniqueKeyColumns,
2069
+ schema,
2070
+ description
2071
+ );
2072
+ this.updatedRecordsBuffer = {};
2073
+ this.totalRecordsProcessed = 0;
2074
+ this.connection = null;
2075
+ }
2076
+ //---- init --------------------------------------------------------
2077
+ /**
2078
+ * Initializing storage - establishes connection and creates table if needed
2079
+ */
2080
+ async init() {
2081
+ this.checkIfSnowflakeIsConnected();
2082
+ await this.createConnection();
2083
+ await this.testConnection();
2084
+ await this.loadTableSchema();
2085
+ }
2086
+ //----------------------------------------------------------------
2087
+ //---- checkIfSnowflakeIsConnected ---------------------------------
2088
+ checkIfSnowflakeIsConnected() {
2089
+ if (typeof snowflake == "undefined") {
2090
+ throw new Error(`Snowflake SDK is not available. Ensure snowflake-sdk is installed.`);
2091
+ }
2092
+ }
2093
+ //----------------------------------------------------------------
2094
+ //---- createConnection --------------------------------------------
2095
+ /**
2096
+ * Creates and connects to Snowflake
2097
+ */
2098
+ async createConnection() {
2099
+ snowflake.configure({
2100
+ logLevel: "OFF"
2101
+ });
2102
+ const connectionConfig = {
2103
+ account: this.config.SnowflakeAccount.value,
2104
+ username: this.config.SnowflakeUsername.value,
2105
+ password: this.config.SnowflakePassword.value,
2106
+ warehouse: this.config.SnowflakeWarehouse.value,
2107
+ database: this.config.SnowflakeDatabase.value,
2108
+ schema: this.config.SnowflakeSchema.value,
2109
+ authenticator: this.config.SnowflakeAuthenticator.value || "SNOWFLAKE"
2110
+ };
2111
+ if (this.config.SnowflakeRole && this.config.SnowflakeRole.value) {
2112
+ connectionConfig.role = this.config.SnowflakeRole.value;
2113
+ }
2114
+ if (this.config.SnowflakePrivateKey && this.config.SnowflakePrivateKey.value) {
2115
+ connectionConfig.privateKey = this.config.SnowflakePrivateKey.value;
2116
+ if (this.config.SnowflakePrivateKeyPassphrase && this.config.SnowflakePrivateKeyPassphrase.value) {
2117
+ connectionConfig.privateKeyPassphrase = this.config.SnowflakePrivateKeyPassphrase.value;
2118
+ }
2119
+ }
2120
+ this.connection = snowflake.createConnection(connectionConfig);
2121
+ return new Promise((resolve, reject) => {
2122
+ this.connection.connect((err, conn) => {
2123
+ if (err) {
2124
+ reject(new Error(`Failed to connect to Snowflake: ${err.message}`));
2125
+ } else {
2126
+ this.config.logMessage(`Connected to Snowflake (account: ${this.config.SnowflakeAccount.value})`);
2127
+ resolve(conn);
2128
+ }
2129
+ });
2130
+ });
2131
+ }
2132
+ //----------------------------------------------------------------
2133
+ //---- testConnection ----------------------------------------------
2134
+ /**
2135
+ * Tests the connection with a simple query
2136
+ */
2137
+ async testConnection() {
2138
+ try {
2139
+ await this.executeQuery("SELECT 1 as test");
2140
+ this.config.logMessage("Snowflake connection established successfully");
2141
+ } catch (error) {
2142
+ throw new Error(`Snowflake connection test failed: ${error.message}`);
2143
+ }
2144
+ }
2145
+ //----------------------------------------------------------------
2146
+ //---- loadTableSchema ---------------------------------------------
2147
+ /**
2148
+ * Loads existing table schema from Snowflake
2149
+ */
2150
+ async loadTableSchema() {
2151
+ this.existingColumns = await this.getAListOfExistingColumns() || {};
2152
+ if (Object.keys(this.existingColumns).length == 0) {
2153
+ await this.createDatabaseAndSchemaIfNotExist();
2154
+ this.existingColumns = await this.createTableIfItDoesntExist();
2155
+ } else {
2156
+ let selectedFields = this.getSelectedFields();
2157
+ let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
2158
+ if (newFields.length > 0) {
2159
+ await this.addNewColumns(newFields);
2160
+ }
2161
+ }
2162
+ }
2163
+ //----------------------------------------------------------------
2164
+ //---- getAListOfExistingColumns -----------------------------------
2165
+ /**
2166
+ * Reads columns list of the table and returns it as object
2167
+ *
2168
+ * @return columns (object)
2169
+ */
2170
+ async getAListOfExistingColumns() {
2171
+ let query = `SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE
2172
+ FROM ${this.config.SnowflakeDatabase.value}.INFORMATION_SCHEMA.COLUMNS
2173
+ WHERE TABLE_CATALOG = '${this.config.SnowflakeDatabase.value}'
2174
+ AND TABLE_SCHEMA = UPPER('${this.config.SnowflakeSchema.value}')
2175
+ AND TABLE_NAME = UPPER('${this.config.DestinationTableName.value}')
2176
+ ORDER BY ORDINAL_POSITION`;
2177
+ let queryResults = [];
2178
+ try {
2179
+ queryResults = await this.executeQuery(query);
2180
+ } catch (error) {
2181
+ if (error.message && error.message.includes("does not exist")) {
2182
+ return {};
2183
+ }
2184
+ throw error;
2185
+ }
2186
+ let columns = {};
2187
+ if (Array.isArray(queryResults)) {
2188
+ queryResults.forEach((row) => {
2189
+ const columnName = row.COLUMN_NAME;
2190
+ const dataType = row.DATA_TYPE;
2191
+ columns[columnName] = { "name": columnName, "type": dataType };
2192
+ });
2193
+ }
2194
+ return columns;
2195
+ }
2196
+ //----------------------------------------------------------------
2197
+ //---- createDatabaseAndSchemaIfNotExist ---------------------------
2198
+ async createDatabaseAndSchemaIfNotExist() {
2199
+ let createDbQuery = `CREATE DATABASE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}`;
2200
+ await this.executeQuery(createDbQuery);
2201
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2202
+ let createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}`;
2203
+ await this.executeQuery(createSchemaQuery);
2204
+ this.config.logMessage(`Database and schema ensured: ${this.config.SnowflakeDatabase.value}.${quotedSchema}`);
2205
+ }
2206
+ //----------------------------------------------------------------
2207
+ //---- createTableIfItDoesntExist ----------------------------------
2208
+ async createTableIfItDoesntExist() {
2209
+ let columns = [];
2210
+ let existingColumns = {};
2211
+ let selectedFields = this.getSelectedFields();
2212
+ let tableColumns = selectedFields.length > 0 ? selectedFields : this.uniqueKeyColumns;
2213
+ for (let i in tableColumns) {
2214
+ let columnName = tableColumns[i];
2215
+ let columnDescription = "";
2216
+ if (!(columnName in this.schema)) {
2217
+ throw new Error(`Required field ${columnName} not found in schema`);
2218
+ }
2219
+ let columnType = this.getColumnType(columnName);
2220
+ if ("description" in this.schema[columnName]) {
2221
+ columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
2222
+ }
2223
+ columns.push(`"${columnName}" ${columnType}${columnDescription}`);
2224
+ existingColumns[columnName] = { "name": columnName, "type": columnType };
2225
+ }
2226
+ columns.push(`PRIMARY KEY (${this.uniqueKeyColumns.map((col) => `"${col}"`).join(",")}) NOT ENFORCED`);
2227
+ let columnsStr = columns.join(",\n ");
2228
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2229
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2230
+ let query = `CREATE TABLE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} (
2231
+ ${columnsStr}
2232
+ )`;
2233
+ if (this.description) {
2234
+ query += `
2235
+ COMMENT = '${this.description}'`;
2236
+ }
2237
+ await this.executeQuery(query);
2238
+ this.config.logMessage(`Table ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} was created`);
2239
+ return existingColumns;
2240
+ }
2241
+ //----------------------------------------------------------------
2242
+ //---- addNewColumns -----------------------------------------------
2243
+ /**
2244
+ * ALTER table by adding missed columns
2245
+ *
2246
+ * @param {newColumns} array with a list of new columns
2247
+ */
2248
+ async addNewColumns(newColumns) {
2249
+ let columns = [];
2250
+ for (var i in newColumns) {
2251
+ let columnName = newColumns[i];
2252
+ if (columnName in this.schema) {
2253
+ let columnDescription = "";
2254
+ let columnType = this.getColumnType(columnName);
2255
+ if ("description" in this.schema[columnName]) {
2256
+ columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
2257
+ }
2258
+ columns.push(`ADD COLUMN IF NOT EXISTS "${columnName}" ${columnType}${columnDescription}`);
2259
+ this.existingColumns[columnName] = { "name": columnName, "type": columnType };
2260
+ }
2261
+ }
2262
+ if (columns.length > 0) {
2263
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2264
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2265
+ let query = `ALTER TABLE ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}
2266
+ `;
2267
+ query += columns.join(",\n");
2268
+ await this.executeQuery(query);
2269
+ this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`);
2270
+ }
2271
+ }
2272
+ //----------------------------------------------------------------
2273
+ //---- saveData ----------------------------------------------------
2274
+ /**
2275
+ * Saving data to storage
2276
+ * @param {data} array of assoc objects with records to save
2277
+ */
2278
+ async saveData(data) {
2279
+ for (const row of data) {
2280
+ let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
2281
+ if (newFields.length > 0) {
2282
+ await this.addNewColumns(newFields);
2283
+ }
2284
+ this.addRecordToBuffer(row);
2285
+ await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
2286
+ }
2287
+ await this.saveRecordsAddedToBuffer();
2288
+ }
2289
+ //----------------------------------------------------------------
2290
+ //---- addRecordToBuffer -------------------------------------------
2291
+ /**
2292
+ * Adds record to buffer with deduplication
2293
+ * @param {record} object
2294
+ */
2295
+ addRecordToBuffer(record) {
2296
+ let uniqueKey = this.getUniqueKeyByRecordFields(record);
2297
+ this.updatedRecordsBuffer[uniqueKey] = record;
2298
+ }
2299
+ //----------------------------------------------------------------
2300
+ //---- saveRecordsAddedToBuffer ------------------------------------
2301
+ /**
2302
+ * Add records from buffer to storage
2303
+ * @param (integer) {maxBufferSize} records will be added only if buffer size is larger than this parameter
2304
+ */
2305
+ async saveRecordsAddedToBuffer(maxBufferSize = 0) {
2306
+ let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
2307
+ if (bufferSize && bufferSize >= maxBufferSize) {
2308
+ await this.executeQueryWithSizeLimit();
2309
+ }
2310
+ }
2311
+ //----------------------------------------------------------------
2312
+ //---- executeQueryWithSizeLimit -----------------------------------
2313
+ /**
2314
+ * Executes the MERGE query with automatic batching for large datasets
2315
+ */
2316
+ async executeQueryWithSizeLimit() {
2317
+ const bufferKeys = Object.keys(this.updatedRecordsBuffer);
2318
+ const totalRecords = bufferKeys.length;
2319
+ if (totalRecords === 0) {
2320
+ return;
2321
+ }
2322
+ await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
2323
+ this.updatedRecordsBuffer = {};
2324
+ }
2325
+ //----------------------------------------------------------------
2326
+ //---- executeMergeQueryRecursively --------------------------------
2327
+ /**
2328
+ * Recursively attempts to execute MERGE queries, reducing batch size if needed
2329
+ * @param {Array} recordKeys - Array of record keys to process
2330
+ * @param {number} batchSize - Current batch size to attempt
2331
+ */
2332
+ async executeMergeQueryRecursively(recordKeys, batchSize) {
2333
+ if (recordKeys.length === 0) {
2334
+ return;
2335
+ }
2336
+ if (batchSize < 1) {
2337
+ throw new Error("Cannot process records: batch size reduced below 1");
2338
+ }
2339
+ const currentBatch = recordKeys.slice(0, batchSize);
2340
+ const remainingRecords = recordKeys.slice(batchSize);
2341
+ const query = this.buildMergeQuery(currentBatch);
2342
+ try {
2343
+ await this.executeQuery(query);
2344
+ this.totalRecordsProcessed += currentBatch.length;
2345
+ if (remainingRecords.length > 0) {
2346
+ await this.executeMergeQueryRecursively(remainingRecords, batchSize);
2347
+ }
2348
+ } catch (error) {
2349
+ if (batchSize > 1) {
2350
+ await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
2351
+ } else {
2352
+ throw error;
2353
+ }
2354
+ }
2355
+ }
2356
+ //----------------------------------------------------------------
2357
+ //---- buildMergeQuery ---------------------------------------------
2358
+ /**
2359
+ * Builds a MERGE query for the specified record keys
2360
+ * @param {Array} recordKeys - Array of record keys to include in the query
2361
+ * @return {string} - The constructed MERGE query
2362
+ */
2363
+ buildMergeQuery(recordKeys) {
2364
+ let rows = [];
2365
+ for (let i = 0; i < recordKeys.length; i++) {
2366
+ const key = recordKeys[i];
2367
+ let record = this.stringifyNeastedFields(this.updatedRecordsBuffer[key]);
2368
+ let fields = [];
2369
+ for (var j in this.existingColumns) {
2370
+ let columnName = this.existingColumns[j]["name"];
2371
+ let columnType = this.existingColumns[j]["type"];
2372
+ let columnValue = null;
2373
+ if (record[columnName] === void 0 || record[columnName] === null) {
2374
+ columnValue = null;
2375
+ } else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
2376
+ columnValue = DateUtils3.formatDate(record[columnName]);
2377
+ } else if ((columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") && record[columnName] instanceof Date) {
2378
+ const isoString = record[columnName].toISOString();
2379
+ columnValue = isoString.replace("T", " ").substring(0, 19);
2380
+ } else {
2381
+ columnValue = this.obfuscateSpecialCharacters(record[columnName]);
2382
+ }
2383
+ if (columnValue === null) {
2384
+ fields.push(`CAST(NULL AS ${columnType}) AS "${columnName}"`);
2385
+ } else if (columnType.toUpperCase() == "DATE") {
2386
+ fields.push(`TO_DATE('${columnValue}', 'YYYY-MM-DD') AS "${columnName}"`);
2387
+ } else if (columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") {
2388
+ fields.push(`TO_TIMESTAMP('${columnValue}', 'YYYY-MM-DD HH24:MI:SS') AS "${columnName}"`);
2389
+ } else {
2390
+ fields.push(`CAST('${columnValue}' AS ${columnType}) AS "${columnName}"`);
2391
+ }
2392
+ }
2393
+ rows.push(`SELECT ${fields.join(",\n ")}`);
2394
+ }
2395
+ let existingColumnsNames = Object.keys(this.existingColumns);
2396
+ const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
2397
+ const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
2398
+ let fullTableName = `${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`;
2399
+ let query = `MERGE INTO ${fullTableName} AS target
2400
+ USING (
2401
+ ${rows.join("\n UNION ALL\n ")}
2402
+ ) AS source
2403
+
2404
+ ON ${this.uniqueKeyColumns.map((item) => `target."${item}" = source."${item}"`).join("\n AND ")}
2405
+
2406
+ WHEN MATCHED THEN
2407
+ UPDATE SET
2408
+ ${existingColumnsNames.map((item) => `target."${item}" = source."${item}"`).join(",\n ")}
2409
+ WHEN NOT MATCHED THEN
2410
+ INSERT (
2411
+ ${existingColumnsNames.map((item) => `"${item}"`).join(", ")}
2412
+ )
2413
+ VALUES (
2414
+ ${existingColumnsNames.map((item) => `source."${item}"`).join(", ")}
2415
+ )`;
2416
+ return query;
2417
+ }
2418
+ //----------------------------------------------------------------
2419
+ //---- executeQuery ------------------------------------------------
2420
+ /**
2421
+ * Executes Snowflake Query and returns a result
2422
+ *
2423
+ * @param {query} string
2424
+ * @return Promise<Array>
2425
+ */
2426
+ async executeQuery(query) {
2427
+ if (!this.connection) {
2428
+ throw new Error("Snowflake connection not initialized");
2429
+ }
2430
+ return new Promise((resolve, reject) => {
2431
+ this.connection.execute({
2432
+ sqlText: query,
2433
+ complete: (err, stmt, rows) => {
2434
+ if (err) {
2435
+ reject(new Error(`Snowflake query failed: ${err.message}`));
2436
+ } else {
2437
+ resolve(rows || []);
2438
+ }
2439
+ }
2440
+ });
2441
+ });
2442
+ }
2443
+ //----------------------------------------------------------------
2444
+ //---- obfuscateSpecialCharacters ----------------------------------
2445
+ /**
2446
+ * Escape special characters for SQL string literals
2447
+ * @param {string} inputString - String to escape
2448
+ * @return {string} - Escaped string
2449
+ */
2450
+ obfuscateSpecialCharacters(inputString) {
2451
+ return String(inputString).replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/"/g, '\\"').replace(/[\x00-\x1F]/g, " ");
2452
+ }
2453
+ //----------------------------------------------------------------
2454
+ //---- getColumnType -----------------------------------------------
2455
+ /**
2456
+ * Get column type for Snowflake from schema
2457
+ * @param {string} columnName - Name of the column
2458
+ * @returns {string} Snowflake column type
2459
+ */
2460
+ getColumnType(columnName) {
2461
+ var _a;
2462
+ return this.schema[columnName]["SnowflakeFieldType"] || this._convertTypeToStorageType((_a = this.schema[columnName]["type"]) == null ? void 0 : _a.toLowerCase());
2463
+ }
2464
+ //----------------------------------------------------------------
2465
+ //---- _convertTypeToStorageType -----------------------------------
2466
+ /**
2467
+ * Converts generic type to Snowflake-specific type
2468
+ * @param {string} genericType - Generic type from schema
2469
+ * @returns {string} Snowflake column type
2470
+ */
2471
+ _convertTypeToStorageType(genericType) {
2472
+ if (!genericType) return "VARCHAR";
2473
+ switch (genericType.toLowerCase()) {
2474
+ // Integer types
2475
+ case "integer":
2476
+ case "int32":
2477
+ return "INTEGER";
2478
+ case "int64":
2479
+ case "long":
2480
+ return "BIGINT";
2481
+ // Float types
2482
+ case "float":
2483
+ case "number":
2484
+ case "double":
2485
+ return "FLOAT";
2486
+ case "decimal":
2487
+ return "NUMERIC";
2488
+ // Boolean types
2489
+ case "bool":
2490
+ case "boolean":
2491
+ return "BOOLEAN";
2492
+ // Date/time types
2493
+ case "date":
2494
+ return "DATE";
2495
+ case "datetime":
2496
+ return "TIMESTAMP_NTZ";
2497
+ // Timezone-naive
2498
+ case "timestamp":
2499
+ return "TIMESTAMP_TZ";
2500
+ // Timezone-aware
2501
+ // JSON/Object types
2502
+ case "json":
2503
+ case "object":
2504
+ case "array":
2505
+ return "VARIANT";
2506
+ // Default to VARCHAR for unknown types
2507
+ default:
2508
+ return "VARCHAR";
2509
+ }
2510
+ }
2511
+ //----------------------------------------------------------------
2512
+ };
2513
+ const manifest = {
2514
+ "name": "SnowflakeStorage",
2515
+ "description": "Storage for Snowflake Data Warehouse",
2516
+ "title": "Snowflake",
2517
+ "version": "0.0.0",
2518
+ "author": "OWOX, Inc.",
2519
+ "license": "MIT",
2520
+ "environment": {
2521
+ "node": {
2522
+ "enabled": true,
2523
+ "dependencies": [
2524
+ {
2525
+ "name": "snowflake-sdk",
2526
+ "version": "^1.6.20",
2527
+ "global": [
2528
+ "snowflake"
2529
+ ]
2530
+ }
2531
+ ]
2532
+ }
2533
+ }
2534
+ };
2535
+ return {
2536
+ SnowflakeStorage,
2537
+ manifest
2538
+ };
2539
+ })();
1990
2540
  const GoogleBigQuery = (function() {
1991
2541
  const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
1992
2542
  var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
@@ -3099,6 +3649,7 @@ const AwsAthena = (function() {
3099
3649
  };
3100
3650
  })();
3101
3651
  const Storages = {
3652
+ Snowflake,
3102
3653
  GoogleBigQuery,
3103
3654
  AwsAthena
3104
3655
  };
@@ -22038,6 +22589,7 @@ const AvailableConnectors = [
22038
22589
  "BankOfCanada"
22039
22590
  ];
22040
22591
  const AvailableStorages = [
22592
+ "Snowflake",
22041
22593
  "GoogleBigQuery",
22042
22594
  "AwsAthena"
22043
22595
  ];
@@ -22062,6 +22614,7 @@ const OWOX = {
22062
22614
  CriteoAds,
22063
22615
  BankOfCanada,
22064
22616
  // Individual storages
22617
+ Snowflake,
22065
22618
  GoogleBigQuery,
22066
22619
  AwsAthena
22067
22620
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@owox/connectors",
3
- "version": "0.14.0-next-20251127124948",
3
+ "version": "0.14.0-next-20251128101118",
4
4
  "description": "Connectors and storages for different data sources",
5
5
  "license": "MIT",
6
6
  "publishConfig": {
@@ -44,7 +44,8 @@
44
44
  "@aws-sdk/client-athena": "3.810.0",
45
45
  "@aws-sdk/client-s3": "3.810.0",
46
46
  "@aws-sdk/lib-storage": "3.810.0",
47
- "adm-zip": "0.5.16"
47
+ "adm-zip": "0.5.16",
48
+ "snowflake-sdk": "2.3.1"
48
49
  },
49
50
  "devDependencies": {
50
51
  "@types/node": "^20.0.0",