@owox/connectors 0.14.0-next-20251127124948 → 0.14.0-next-20251128101118
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connector-runner.cjs +2 -0
- package/dist/connector-runner.js +2 -0
- package/dist/index.cjs +553 -0
- package/dist/index.js +553 -0
- package/package.json +3 -2
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const OWOX = require("@owox/connectors");
|
|
4
4
|
const AdmZip = require("adm-zip");
|
|
5
5
|
const { BigQuery } = require("@google-cloud/bigquery");
|
|
6
|
+
const snowflake = require("snowflake-sdk");
|
|
6
7
|
const {
|
|
7
8
|
AthenaClient,
|
|
8
9
|
StartQueryExecutionCommand,
|
|
@@ -20,6 +21,7 @@ const { Upload } = require("@aws-sdk/lib-storage");
|
|
|
20
21
|
global.OWOX = OWOX;
|
|
21
22
|
global.AdmZip = AdmZip;
|
|
22
23
|
global.BigQuery = BigQuery;
|
|
24
|
+
global.snowflake = snowflake;
|
|
23
25
|
global.AthenaClient = AthenaClient;
|
|
24
26
|
global.StartQueryExecutionCommand = StartQueryExecutionCommand;
|
|
25
27
|
global.GetQueryExecutionCommand = GetQueryExecutionCommand;
|
package/dist/connector-runner.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
const OWOX = require("@owox/connectors");
|
|
3
3
|
const AdmZip = require("adm-zip");
|
|
4
4
|
const { BigQuery } = require("@google-cloud/bigquery");
|
|
5
|
+
const snowflake = require("snowflake-sdk");
|
|
5
6
|
const {
|
|
6
7
|
AthenaClient,
|
|
7
8
|
StartQueryExecutionCommand,
|
|
@@ -19,6 +20,7 @@ const { Upload } = require("@aws-sdk/lib-storage");
|
|
|
19
20
|
global.OWOX = OWOX;
|
|
20
21
|
global.AdmZip = AdmZip;
|
|
21
22
|
global.BigQuery = BigQuery;
|
|
23
|
+
global.snowflake = snowflake;
|
|
22
24
|
global.AthenaClient = AthenaClient;
|
|
23
25
|
global.StartQueryExecutionCommand = StartQueryExecutionCommand;
|
|
24
26
|
global.GetQueryExecutionCommand = GetQueryExecutionCommand;
|
package/dist/index.cjs
CHANGED
|
@@ -1677,6 +1677,8 @@ class StorageConfigDto {
|
|
|
1677
1677
|
return "GoogleBigQuery";
|
|
1678
1678
|
} else if (name === "AWS_ATHENA") {
|
|
1679
1679
|
return "AwsAthena";
|
|
1680
|
+
} else if (name === "SNOWFLAKE") {
|
|
1681
|
+
return "Snowflake";
|
|
1680
1682
|
}
|
|
1681
1683
|
return name;
|
|
1682
1684
|
}
|
|
@@ -1989,6 +1991,554 @@ const Core = {
|
|
|
1989
1991
|
OAUTH_CONSTANTS,
|
|
1990
1992
|
OAUTH_SOURCE_CREDENTIALS_KEY
|
|
1991
1993
|
};
|
|
1994
|
+
const Snowflake = (function() {
|
|
1995
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
|
|
1996
|
+
function quoteIdentifier(identifier) {
|
|
1997
|
+
if (!identifier) return identifier;
|
|
1998
|
+
if (identifier.startsWith('"') && identifier.endsWith('"')) {
|
|
1999
|
+
return identifier;
|
|
2000
|
+
}
|
|
2001
|
+
return `"${identifier}"`;
|
|
2002
|
+
}
|
|
2003
|
+
var SnowflakeStorage = class SnowflakeStorage extends AbstractStorage2 {
|
|
2004
|
+
//---- constructor -------------------------------------------------
|
|
2005
|
+
/**
|
|
2006
|
+
* Snowflake storage operations class
|
|
2007
|
+
*
|
|
2008
|
+
* @param config (object) instance of AbstractConfig
|
|
2009
|
+
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2010
|
+
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2011
|
+
* @param description (string) string with storage description }
|
|
2012
|
+
*/
|
|
2013
|
+
constructor(config, uniqueKeyColumns, schema = null, description = null) {
|
|
2014
|
+
super(
|
|
2015
|
+
config.mergeParameters({
|
|
2016
|
+
SnowflakeAccount: {
|
|
2017
|
+
isRequired: true,
|
|
2018
|
+
requiredType: "string"
|
|
2019
|
+
},
|
|
2020
|
+
SnowflakeWarehouse: {
|
|
2021
|
+
isRequired: true,
|
|
2022
|
+
requiredType: "string"
|
|
2023
|
+
},
|
|
2024
|
+
SnowflakeDatabase: {
|
|
2025
|
+
isRequired: true,
|
|
2026
|
+
requiredType: "string"
|
|
2027
|
+
},
|
|
2028
|
+
SnowflakeSchema: {
|
|
2029
|
+
isRequired: true,
|
|
2030
|
+
requiredType: "string"
|
|
2031
|
+
},
|
|
2032
|
+
SnowflakeRole: {
|
|
2033
|
+
isRequired: false,
|
|
2034
|
+
requiredType: "string",
|
|
2035
|
+
default: null
|
|
2036
|
+
},
|
|
2037
|
+
SnowflakeUsername: {
|
|
2038
|
+
isRequired: true,
|
|
2039
|
+
requiredType: "string"
|
|
2040
|
+
},
|
|
2041
|
+
SnowflakePassword: {
|
|
2042
|
+
isRequired: true,
|
|
2043
|
+
requiredType: "string"
|
|
2044
|
+
},
|
|
2045
|
+
SnowflakeAuthenticator: {
|
|
2046
|
+
isRequired: false,
|
|
2047
|
+
requiredType: "string",
|
|
2048
|
+
default: "SNOWFLAKE"
|
|
2049
|
+
},
|
|
2050
|
+
SnowflakePrivateKey: {
|
|
2051
|
+
isRequired: false,
|
|
2052
|
+
requiredType: "string",
|
|
2053
|
+
default: null
|
|
2054
|
+
},
|
|
2055
|
+
SnowflakePrivateKeyPassphrase: {
|
|
2056
|
+
isRequired: false,
|
|
2057
|
+
requiredType: "string",
|
|
2058
|
+
default: null
|
|
2059
|
+
},
|
|
2060
|
+
DestinationTableName: {
|
|
2061
|
+
isRequired: true,
|
|
2062
|
+
requiredType: "string",
|
|
2063
|
+
default: "Data"
|
|
2064
|
+
},
|
|
2065
|
+
MaxBufferSize: {
|
|
2066
|
+
isRequired: true,
|
|
2067
|
+
default: 250
|
|
2068
|
+
}
|
|
2069
|
+
}),
|
|
2070
|
+
uniqueKeyColumns,
|
|
2071
|
+
schema,
|
|
2072
|
+
description
|
|
2073
|
+
);
|
|
2074
|
+
this.updatedRecordsBuffer = {};
|
|
2075
|
+
this.totalRecordsProcessed = 0;
|
|
2076
|
+
this.connection = null;
|
|
2077
|
+
}
|
|
2078
|
+
//---- init --------------------------------------------------------
|
|
2079
|
+
/**
|
|
2080
|
+
* Initializing storage - establishes connection and creates table if needed
|
|
2081
|
+
*/
|
|
2082
|
+
async init() {
|
|
2083
|
+
this.checkIfSnowflakeIsConnected();
|
|
2084
|
+
await this.createConnection();
|
|
2085
|
+
await this.testConnection();
|
|
2086
|
+
await this.loadTableSchema();
|
|
2087
|
+
}
|
|
2088
|
+
//----------------------------------------------------------------
|
|
2089
|
+
//---- checkIfSnowflakeIsConnected ---------------------------------
|
|
2090
|
+
checkIfSnowflakeIsConnected() {
|
|
2091
|
+
if (typeof snowflake == "undefined") {
|
|
2092
|
+
throw new Error(`Snowflake SDK is not available. Ensure snowflake-sdk is installed.`);
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
//----------------------------------------------------------------
|
|
2096
|
+
//---- createConnection --------------------------------------------
|
|
2097
|
+
/**
|
|
2098
|
+
* Creates and connects to Snowflake
|
|
2099
|
+
*/
|
|
2100
|
+
async createConnection() {
|
|
2101
|
+
snowflake.configure({
|
|
2102
|
+
logLevel: "OFF"
|
|
2103
|
+
});
|
|
2104
|
+
const connectionConfig = {
|
|
2105
|
+
account: this.config.SnowflakeAccount.value,
|
|
2106
|
+
username: this.config.SnowflakeUsername.value,
|
|
2107
|
+
password: this.config.SnowflakePassword.value,
|
|
2108
|
+
warehouse: this.config.SnowflakeWarehouse.value,
|
|
2109
|
+
database: this.config.SnowflakeDatabase.value,
|
|
2110
|
+
schema: this.config.SnowflakeSchema.value,
|
|
2111
|
+
authenticator: this.config.SnowflakeAuthenticator.value || "SNOWFLAKE"
|
|
2112
|
+
};
|
|
2113
|
+
if (this.config.SnowflakeRole && this.config.SnowflakeRole.value) {
|
|
2114
|
+
connectionConfig.role = this.config.SnowflakeRole.value;
|
|
2115
|
+
}
|
|
2116
|
+
if (this.config.SnowflakePrivateKey && this.config.SnowflakePrivateKey.value) {
|
|
2117
|
+
connectionConfig.privateKey = this.config.SnowflakePrivateKey.value;
|
|
2118
|
+
if (this.config.SnowflakePrivateKeyPassphrase && this.config.SnowflakePrivateKeyPassphrase.value) {
|
|
2119
|
+
connectionConfig.privateKeyPassphrase = this.config.SnowflakePrivateKeyPassphrase.value;
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
this.connection = snowflake.createConnection(connectionConfig);
|
|
2123
|
+
return new Promise((resolve, reject) => {
|
|
2124
|
+
this.connection.connect((err, conn) => {
|
|
2125
|
+
if (err) {
|
|
2126
|
+
reject(new Error(`Failed to connect to Snowflake: ${err.message}`));
|
|
2127
|
+
} else {
|
|
2128
|
+
this.config.logMessage(`Connected to Snowflake (account: ${this.config.SnowflakeAccount.value})`);
|
|
2129
|
+
resolve(conn);
|
|
2130
|
+
}
|
|
2131
|
+
});
|
|
2132
|
+
});
|
|
2133
|
+
}
|
|
2134
|
+
//----------------------------------------------------------------
|
|
2135
|
+
//---- testConnection ----------------------------------------------
|
|
2136
|
+
/**
|
|
2137
|
+
* Tests the connection with a simple query
|
|
2138
|
+
*/
|
|
2139
|
+
async testConnection() {
|
|
2140
|
+
try {
|
|
2141
|
+
await this.executeQuery("SELECT 1 as test");
|
|
2142
|
+
this.config.logMessage("Snowflake connection established successfully");
|
|
2143
|
+
} catch (error) {
|
|
2144
|
+
throw new Error(`Snowflake connection test failed: ${error.message}`);
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
//----------------------------------------------------------------
|
|
2148
|
+
//---- loadTableSchema ---------------------------------------------
|
|
2149
|
+
/**
|
|
2150
|
+
* Loads existing table schema from Snowflake
|
|
2151
|
+
*/
|
|
2152
|
+
async loadTableSchema() {
|
|
2153
|
+
this.existingColumns = await this.getAListOfExistingColumns() || {};
|
|
2154
|
+
if (Object.keys(this.existingColumns).length == 0) {
|
|
2155
|
+
await this.createDatabaseAndSchemaIfNotExist();
|
|
2156
|
+
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
2157
|
+
} else {
|
|
2158
|
+
let selectedFields = this.getSelectedFields();
|
|
2159
|
+
let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2160
|
+
if (newFields.length > 0) {
|
|
2161
|
+
await this.addNewColumns(newFields);
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
//----------------------------------------------------------------
|
|
2166
|
+
//---- getAListOfExistingColumns -----------------------------------
|
|
2167
|
+
/**
|
|
2168
|
+
* Reads columns list of the table and returns it as object
|
|
2169
|
+
*
|
|
2170
|
+
* @return columns (object)
|
|
2171
|
+
*/
|
|
2172
|
+
async getAListOfExistingColumns() {
|
|
2173
|
+
let query = `SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE
|
|
2174
|
+
FROM ${this.config.SnowflakeDatabase.value}.INFORMATION_SCHEMA.COLUMNS
|
|
2175
|
+
WHERE TABLE_CATALOG = '${this.config.SnowflakeDatabase.value}'
|
|
2176
|
+
AND TABLE_SCHEMA = UPPER('${this.config.SnowflakeSchema.value}')
|
|
2177
|
+
AND TABLE_NAME = UPPER('${this.config.DestinationTableName.value}')
|
|
2178
|
+
ORDER BY ORDINAL_POSITION`;
|
|
2179
|
+
let queryResults = [];
|
|
2180
|
+
try {
|
|
2181
|
+
queryResults = await this.executeQuery(query);
|
|
2182
|
+
} catch (error) {
|
|
2183
|
+
if (error.message && error.message.includes("does not exist")) {
|
|
2184
|
+
return {};
|
|
2185
|
+
}
|
|
2186
|
+
throw error;
|
|
2187
|
+
}
|
|
2188
|
+
let columns = {};
|
|
2189
|
+
if (Array.isArray(queryResults)) {
|
|
2190
|
+
queryResults.forEach((row) => {
|
|
2191
|
+
const columnName = row.COLUMN_NAME;
|
|
2192
|
+
const dataType = row.DATA_TYPE;
|
|
2193
|
+
columns[columnName] = { "name": columnName, "type": dataType };
|
|
2194
|
+
});
|
|
2195
|
+
}
|
|
2196
|
+
return columns;
|
|
2197
|
+
}
|
|
2198
|
+
//----------------------------------------------------------------
|
|
2199
|
+
//---- createDatabaseAndSchemaIfNotExist ---------------------------
|
|
2200
|
+
async createDatabaseAndSchemaIfNotExist() {
|
|
2201
|
+
let createDbQuery = `CREATE DATABASE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}`;
|
|
2202
|
+
await this.executeQuery(createDbQuery);
|
|
2203
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2204
|
+
let createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}`;
|
|
2205
|
+
await this.executeQuery(createSchemaQuery);
|
|
2206
|
+
this.config.logMessage(`Database and schema ensured: ${this.config.SnowflakeDatabase.value}.${quotedSchema}`);
|
|
2207
|
+
}
|
|
2208
|
+
//----------------------------------------------------------------
|
|
2209
|
+
//---- createTableIfItDoesntExist ----------------------------------
|
|
2210
|
+
async createTableIfItDoesntExist() {
|
|
2211
|
+
let columns = [];
|
|
2212
|
+
let existingColumns = {};
|
|
2213
|
+
let selectedFields = this.getSelectedFields();
|
|
2214
|
+
let tableColumns = selectedFields.length > 0 ? selectedFields : this.uniqueKeyColumns;
|
|
2215
|
+
for (let i in tableColumns) {
|
|
2216
|
+
let columnName = tableColumns[i];
|
|
2217
|
+
let columnDescription = "";
|
|
2218
|
+
if (!(columnName in this.schema)) {
|
|
2219
|
+
throw new Error(`Required field ${columnName} not found in schema`);
|
|
2220
|
+
}
|
|
2221
|
+
let columnType = this.getColumnType(columnName);
|
|
2222
|
+
if ("description" in this.schema[columnName]) {
|
|
2223
|
+
columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
|
|
2224
|
+
}
|
|
2225
|
+
columns.push(`"${columnName}" ${columnType}${columnDescription}`);
|
|
2226
|
+
existingColumns[columnName] = { "name": columnName, "type": columnType };
|
|
2227
|
+
}
|
|
2228
|
+
columns.push(`PRIMARY KEY (${this.uniqueKeyColumns.map((col) => `"${col}"`).join(",")}) NOT ENFORCED`);
|
|
2229
|
+
let columnsStr = columns.join(",\n ");
|
|
2230
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2231
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2232
|
+
let query = `CREATE TABLE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} (
|
|
2233
|
+
${columnsStr}
|
|
2234
|
+
)`;
|
|
2235
|
+
if (this.description) {
|
|
2236
|
+
query += `
|
|
2237
|
+
COMMENT = '${this.description}'`;
|
|
2238
|
+
}
|
|
2239
|
+
await this.executeQuery(query);
|
|
2240
|
+
this.config.logMessage(`Table ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} was created`);
|
|
2241
|
+
return existingColumns;
|
|
2242
|
+
}
|
|
2243
|
+
//----------------------------------------------------------------
|
|
2244
|
+
//---- addNewColumns -----------------------------------------------
|
|
2245
|
+
/**
|
|
2246
|
+
* ALTER table by adding missed columns
|
|
2247
|
+
*
|
|
2248
|
+
* @param {newColumns} array with a list of new columns
|
|
2249
|
+
*/
|
|
2250
|
+
async addNewColumns(newColumns) {
|
|
2251
|
+
let columns = [];
|
|
2252
|
+
for (var i in newColumns) {
|
|
2253
|
+
let columnName = newColumns[i];
|
|
2254
|
+
if (columnName in this.schema) {
|
|
2255
|
+
let columnDescription = "";
|
|
2256
|
+
let columnType = this.getColumnType(columnName);
|
|
2257
|
+
if ("description" in this.schema[columnName]) {
|
|
2258
|
+
columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
|
|
2259
|
+
}
|
|
2260
|
+
columns.push(`ADD COLUMN IF NOT EXISTS "${columnName}" ${columnType}${columnDescription}`);
|
|
2261
|
+
this.existingColumns[columnName] = { "name": columnName, "type": columnType };
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
if (columns.length > 0) {
|
|
2265
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2266
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2267
|
+
let query = `ALTER TABLE ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}
|
|
2268
|
+
`;
|
|
2269
|
+
query += columns.join(",\n");
|
|
2270
|
+
await this.executeQuery(query);
|
|
2271
|
+
this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`);
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
//----------------------------------------------------------------
|
|
2275
|
+
//---- saveData ----------------------------------------------------
|
|
2276
|
+
/**
|
|
2277
|
+
* Saving data to storage
|
|
2278
|
+
* @param {data} array of assoc objects with records to save
|
|
2279
|
+
*/
|
|
2280
|
+
async saveData(data) {
|
|
2281
|
+
for (const row of data) {
|
|
2282
|
+
let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2283
|
+
if (newFields.length > 0) {
|
|
2284
|
+
await this.addNewColumns(newFields);
|
|
2285
|
+
}
|
|
2286
|
+
this.addRecordToBuffer(row);
|
|
2287
|
+
await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2288
|
+
}
|
|
2289
|
+
await this.saveRecordsAddedToBuffer();
|
|
2290
|
+
}
|
|
2291
|
+
//----------------------------------------------------------------
|
|
2292
|
+
//---- addRecordToBuffer -------------------------------------------
|
|
2293
|
+
/**
|
|
2294
|
+
* Adds record to buffer with deduplication
|
|
2295
|
+
* @param {record} object
|
|
2296
|
+
*/
|
|
2297
|
+
addRecordToBuffer(record) {
|
|
2298
|
+
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2299
|
+
this.updatedRecordsBuffer[uniqueKey] = record;
|
|
2300
|
+
}
|
|
2301
|
+
//----------------------------------------------------------------
|
|
2302
|
+
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
2303
|
+
/**
|
|
2304
|
+
* Add records from buffer to storage
|
|
2305
|
+
* @param (integer) {maxBufferSize} records will be added only if buffer size is larger than this parameter
|
|
2306
|
+
*/
|
|
2307
|
+
async saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2308
|
+
let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
|
|
2309
|
+
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
2310
|
+
await this.executeQueryWithSizeLimit();
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
//----------------------------------------------------------------
|
|
2314
|
+
//---- executeQueryWithSizeLimit -----------------------------------
|
|
2315
|
+
/**
|
|
2316
|
+
* Executes the MERGE query with automatic batching for large datasets
|
|
2317
|
+
*/
|
|
2318
|
+
async executeQueryWithSizeLimit() {
|
|
2319
|
+
const bufferKeys = Object.keys(this.updatedRecordsBuffer);
|
|
2320
|
+
const totalRecords = bufferKeys.length;
|
|
2321
|
+
if (totalRecords === 0) {
|
|
2322
|
+
return;
|
|
2323
|
+
}
|
|
2324
|
+
await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
2325
|
+
this.updatedRecordsBuffer = {};
|
|
2326
|
+
}
|
|
2327
|
+
//----------------------------------------------------------------
|
|
2328
|
+
//---- executeMergeQueryRecursively --------------------------------
|
|
2329
|
+
/**
|
|
2330
|
+
* Recursively attempts to execute MERGE queries, reducing batch size if needed
|
|
2331
|
+
* @param {Array} recordKeys - Array of record keys to process
|
|
2332
|
+
* @param {number} batchSize - Current batch size to attempt
|
|
2333
|
+
*/
|
|
2334
|
+
async executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
2335
|
+
if (recordKeys.length === 0) {
|
|
2336
|
+
return;
|
|
2337
|
+
}
|
|
2338
|
+
if (batchSize < 1) {
|
|
2339
|
+
throw new Error("Cannot process records: batch size reduced below 1");
|
|
2340
|
+
}
|
|
2341
|
+
const currentBatch = recordKeys.slice(0, batchSize);
|
|
2342
|
+
const remainingRecords = recordKeys.slice(batchSize);
|
|
2343
|
+
const query = this.buildMergeQuery(currentBatch);
|
|
2344
|
+
try {
|
|
2345
|
+
await this.executeQuery(query);
|
|
2346
|
+
this.totalRecordsProcessed += currentBatch.length;
|
|
2347
|
+
if (remainingRecords.length > 0) {
|
|
2348
|
+
await this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2349
|
+
}
|
|
2350
|
+
} catch (error) {
|
|
2351
|
+
if (batchSize > 1) {
|
|
2352
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2353
|
+
} else {
|
|
2354
|
+
throw error;
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
//----------------------------------------------------------------
|
|
2359
|
+
//---- buildMergeQuery ---------------------------------------------
|
|
2360
|
+
/**
|
|
2361
|
+
* Builds a MERGE query for the specified record keys
|
|
2362
|
+
* @param {Array} recordKeys - Array of record keys to include in the query
|
|
2363
|
+
* @return {string} - The constructed MERGE query
|
|
2364
|
+
*/
|
|
2365
|
+
buildMergeQuery(recordKeys) {
|
|
2366
|
+
let rows = [];
|
|
2367
|
+
for (let i = 0; i < recordKeys.length; i++) {
|
|
2368
|
+
const key = recordKeys[i];
|
|
2369
|
+
let record = this.stringifyNeastedFields(this.updatedRecordsBuffer[key]);
|
|
2370
|
+
let fields = [];
|
|
2371
|
+
for (var j in this.existingColumns) {
|
|
2372
|
+
let columnName = this.existingColumns[j]["name"];
|
|
2373
|
+
let columnType = this.existingColumns[j]["type"];
|
|
2374
|
+
let columnValue = null;
|
|
2375
|
+
if (record[columnName] === void 0 || record[columnName] === null) {
|
|
2376
|
+
columnValue = null;
|
|
2377
|
+
} else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
|
|
2378
|
+
columnValue = DateUtils3.formatDate(record[columnName]);
|
|
2379
|
+
} else if ((columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") && record[columnName] instanceof Date) {
|
|
2380
|
+
const isoString = record[columnName].toISOString();
|
|
2381
|
+
columnValue = isoString.replace("T", " ").substring(0, 19);
|
|
2382
|
+
} else {
|
|
2383
|
+
columnValue = this.obfuscateSpecialCharacters(record[columnName]);
|
|
2384
|
+
}
|
|
2385
|
+
if (columnValue === null) {
|
|
2386
|
+
fields.push(`CAST(NULL AS ${columnType}) AS "${columnName}"`);
|
|
2387
|
+
} else if (columnType.toUpperCase() == "DATE") {
|
|
2388
|
+
fields.push(`TO_DATE('${columnValue}', 'YYYY-MM-DD') AS "${columnName}"`);
|
|
2389
|
+
} else if (columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") {
|
|
2390
|
+
fields.push(`TO_TIMESTAMP('${columnValue}', 'YYYY-MM-DD HH24:MI:SS') AS "${columnName}"`);
|
|
2391
|
+
} else {
|
|
2392
|
+
fields.push(`CAST('${columnValue}' AS ${columnType}) AS "${columnName}"`);
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
rows.push(`SELECT ${fields.join(",\n ")}`);
|
|
2396
|
+
}
|
|
2397
|
+
let existingColumnsNames = Object.keys(this.existingColumns);
|
|
2398
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2399
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2400
|
+
let fullTableName = `${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`;
|
|
2401
|
+
let query = `MERGE INTO ${fullTableName} AS target
|
|
2402
|
+
USING (
|
|
2403
|
+
${rows.join("\n UNION ALL\n ")}
|
|
2404
|
+
) AS source
|
|
2405
|
+
|
|
2406
|
+
ON ${this.uniqueKeyColumns.map((item) => `target."${item}" = source."${item}"`).join("\n AND ")}
|
|
2407
|
+
|
|
2408
|
+
WHEN MATCHED THEN
|
|
2409
|
+
UPDATE SET
|
|
2410
|
+
${existingColumnsNames.map((item) => `target."${item}" = source."${item}"`).join(",\n ")}
|
|
2411
|
+
WHEN NOT MATCHED THEN
|
|
2412
|
+
INSERT (
|
|
2413
|
+
${existingColumnsNames.map((item) => `"${item}"`).join(", ")}
|
|
2414
|
+
)
|
|
2415
|
+
VALUES (
|
|
2416
|
+
${existingColumnsNames.map((item) => `source."${item}"`).join(", ")}
|
|
2417
|
+
)`;
|
|
2418
|
+
return query;
|
|
2419
|
+
}
|
|
2420
|
+
//----------------------------------------------------------------
|
|
2421
|
+
//---- executeQuery ------------------------------------------------
|
|
2422
|
+
/**
|
|
2423
|
+
* Executes Snowflake Query and returns a result
|
|
2424
|
+
*
|
|
2425
|
+
* @param {query} string
|
|
2426
|
+
* @return Promise<Array>
|
|
2427
|
+
*/
|
|
2428
|
+
async executeQuery(query) {
|
|
2429
|
+
if (!this.connection) {
|
|
2430
|
+
throw new Error("Snowflake connection not initialized");
|
|
2431
|
+
}
|
|
2432
|
+
return new Promise((resolve, reject) => {
|
|
2433
|
+
this.connection.execute({
|
|
2434
|
+
sqlText: query,
|
|
2435
|
+
complete: (err, stmt, rows) => {
|
|
2436
|
+
if (err) {
|
|
2437
|
+
reject(new Error(`Snowflake query failed: ${err.message}`));
|
|
2438
|
+
} else {
|
|
2439
|
+
resolve(rows || []);
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
});
|
|
2443
|
+
});
|
|
2444
|
+
}
|
|
2445
|
+
//----------------------------------------------------------------
|
|
2446
|
+
//---- obfuscateSpecialCharacters ----------------------------------
|
|
2447
|
+
/**
|
|
2448
|
+
* Escape special characters for SQL string literals
|
|
2449
|
+
* @param {string} inputString - String to escape
|
|
2450
|
+
* @return {string} - Escaped string
|
|
2451
|
+
*/
|
|
2452
|
+
obfuscateSpecialCharacters(inputString) {
|
|
2453
|
+
return String(inputString).replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/"/g, '\\"').replace(/[\x00-\x1F]/g, " ");
|
|
2454
|
+
}
|
|
2455
|
+
//----------------------------------------------------------------
|
|
2456
|
+
//---- getColumnType -----------------------------------------------
|
|
2457
|
+
/**
|
|
2458
|
+
* Get column type for Snowflake from schema
|
|
2459
|
+
* @param {string} columnName - Name of the column
|
|
2460
|
+
* @returns {string} Snowflake column type
|
|
2461
|
+
*/
|
|
2462
|
+
getColumnType(columnName) {
|
|
2463
|
+
var _a;
|
|
2464
|
+
return this.schema[columnName]["SnowflakeFieldType"] || this._convertTypeToStorageType((_a = this.schema[columnName]["type"]) == null ? void 0 : _a.toLowerCase());
|
|
2465
|
+
}
|
|
2466
|
+
//----------------------------------------------------------------
|
|
2467
|
+
//---- _convertTypeToStorageType -----------------------------------
|
|
2468
|
+
/**
|
|
2469
|
+
* Converts generic type to Snowflake-specific type
|
|
2470
|
+
* @param {string} genericType - Generic type from schema
|
|
2471
|
+
* @returns {string} Snowflake column type
|
|
2472
|
+
*/
|
|
2473
|
+
_convertTypeToStorageType(genericType) {
|
|
2474
|
+
if (!genericType) return "VARCHAR";
|
|
2475
|
+
switch (genericType.toLowerCase()) {
|
|
2476
|
+
// Integer types
|
|
2477
|
+
case "integer":
|
|
2478
|
+
case "int32":
|
|
2479
|
+
return "INTEGER";
|
|
2480
|
+
case "int64":
|
|
2481
|
+
case "long":
|
|
2482
|
+
return "BIGINT";
|
|
2483
|
+
// Float types
|
|
2484
|
+
case "float":
|
|
2485
|
+
case "number":
|
|
2486
|
+
case "double":
|
|
2487
|
+
return "FLOAT";
|
|
2488
|
+
case "decimal":
|
|
2489
|
+
return "NUMERIC";
|
|
2490
|
+
// Boolean types
|
|
2491
|
+
case "bool":
|
|
2492
|
+
case "boolean":
|
|
2493
|
+
return "BOOLEAN";
|
|
2494
|
+
// Date/time types
|
|
2495
|
+
case "date":
|
|
2496
|
+
return "DATE";
|
|
2497
|
+
case "datetime":
|
|
2498
|
+
return "TIMESTAMP_NTZ";
|
|
2499
|
+
// Timezone-naive
|
|
2500
|
+
case "timestamp":
|
|
2501
|
+
return "TIMESTAMP_TZ";
|
|
2502
|
+
// Timezone-aware
|
|
2503
|
+
// JSON/Object types
|
|
2504
|
+
case "json":
|
|
2505
|
+
case "object":
|
|
2506
|
+
case "array":
|
|
2507
|
+
return "VARIANT";
|
|
2508
|
+
// Default to VARCHAR for unknown types
|
|
2509
|
+
default:
|
|
2510
|
+
return "VARCHAR";
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
//----------------------------------------------------------------
|
|
2514
|
+
};
|
|
2515
|
+
const manifest = {
|
|
2516
|
+
"name": "SnowflakeStorage",
|
|
2517
|
+
"description": "Storage for Snowflake Data Warehouse",
|
|
2518
|
+
"title": "Snowflake",
|
|
2519
|
+
"version": "0.0.0",
|
|
2520
|
+
"author": "OWOX, Inc.",
|
|
2521
|
+
"license": "MIT",
|
|
2522
|
+
"environment": {
|
|
2523
|
+
"node": {
|
|
2524
|
+
"enabled": true,
|
|
2525
|
+
"dependencies": [
|
|
2526
|
+
{
|
|
2527
|
+
"name": "snowflake-sdk",
|
|
2528
|
+
"version": "^1.6.20",
|
|
2529
|
+
"global": [
|
|
2530
|
+
"snowflake"
|
|
2531
|
+
]
|
|
2532
|
+
}
|
|
2533
|
+
]
|
|
2534
|
+
}
|
|
2535
|
+
}
|
|
2536
|
+
};
|
|
2537
|
+
return {
|
|
2538
|
+
SnowflakeStorage,
|
|
2539
|
+
manifest
|
|
2540
|
+
};
|
|
2541
|
+
})();
|
|
1992
2542
|
const GoogleBigQuery = (function() {
|
|
1993
2543
|
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
|
|
1994
2544
|
var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
|
|
@@ -3101,6 +3651,7 @@ const AwsAthena = (function() {
|
|
|
3101
3651
|
};
|
|
3102
3652
|
})();
|
|
3103
3653
|
const Storages = {
|
|
3654
|
+
Snowflake,
|
|
3104
3655
|
GoogleBigQuery,
|
|
3105
3656
|
AwsAthena
|
|
3106
3657
|
};
|
|
@@ -22040,6 +22591,7 @@ const AvailableConnectors = [
|
|
|
22040
22591
|
"BankOfCanada"
|
|
22041
22592
|
];
|
|
22042
22593
|
const AvailableStorages = [
|
|
22594
|
+
"Snowflake",
|
|
22043
22595
|
"GoogleBigQuery",
|
|
22044
22596
|
"AwsAthena"
|
|
22045
22597
|
];
|
|
@@ -22064,6 +22616,7 @@ const OWOX = {
|
|
|
22064
22616
|
CriteoAds,
|
|
22065
22617
|
BankOfCanada,
|
|
22066
22618
|
// Individual storages
|
|
22619
|
+
Snowflake,
|
|
22067
22620
|
GoogleBigQuery,
|
|
22068
22621
|
AwsAthena
|
|
22069
22622
|
};
|
package/dist/index.js
CHANGED
|
@@ -1675,6 +1675,8 @@ class StorageConfigDto {
|
|
|
1675
1675
|
return "GoogleBigQuery";
|
|
1676
1676
|
} else if (name === "AWS_ATHENA") {
|
|
1677
1677
|
return "AwsAthena";
|
|
1678
|
+
} else if (name === "SNOWFLAKE") {
|
|
1679
|
+
return "Snowflake";
|
|
1678
1680
|
}
|
|
1679
1681
|
return name;
|
|
1680
1682
|
}
|
|
@@ -1987,6 +1989,554 @@ const Core = {
|
|
|
1987
1989
|
OAUTH_CONSTANTS,
|
|
1988
1990
|
OAUTH_SOURCE_CREDENTIALS_KEY
|
|
1989
1991
|
};
|
|
1992
|
+
const Snowflake = (function() {
|
|
1993
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
|
|
1994
|
+
function quoteIdentifier(identifier) {
|
|
1995
|
+
if (!identifier) return identifier;
|
|
1996
|
+
if (identifier.startsWith('"') && identifier.endsWith('"')) {
|
|
1997
|
+
return identifier;
|
|
1998
|
+
}
|
|
1999
|
+
return `"${identifier}"`;
|
|
2000
|
+
}
|
|
2001
|
+
var SnowflakeStorage = class SnowflakeStorage extends AbstractStorage2 {
|
|
2002
|
+
//---- constructor -------------------------------------------------
|
|
2003
|
+
/**
|
|
2004
|
+
* Snowflake storage operations class
|
|
2005
|
+
*
|
|
2006
|
+
* @param config (object) instance of AbstractConfig
|
|
2007
|
+
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2008
|
+
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2009
|
+
* @param description (string) string with storage description }
|
|
2010
|
+
*/
|
|
2011
|
+
constructor(config, uniqueKeyColumns, schema = null, description = null) {
|
|
2012
|
+
super(
|
|
2013
|
+
config.mergeParameters({
|
|
2014
|
+
SnowflakeAccount: {
|
|
2015
|
+
isRequired: true,
|
|
2016
|
+
requiredType: "string"
|
|
2017
|
+
},
|
|
2018
|
+
SnowflakeWarehouse: {
|
|
2019
|
+
isRequired: true,
|
|
2020
|
+
requiredType: "string"
|
|
2021
|
+
},
|
|
2022
|
+
SnowflakeDatabase: {
|
|
2023
|
+
isRequired: true,
|
|
2024
|
+
requiredType: "string"
|
|
2025
|
+
},
|
|
2026
|
+
SnowflakeSchema: {
|
|
2027
|
+
isRequired: true,
|
|
2028
|
+
requiredType: "string"
|
|
2029
|
+
},
|
|
2030
|
+
SnowflakeRole: {
|
|
2031
|
+
isRequired: false,
|
|
2032
|
+
requiredType: "string",
|
|
2033
|
+
default: null
|
|
2034
|
+
},
|
|
2035
|
+
SnowflakeUsername: {
|
|
2036
|
+
isRequired: true,
|
|
2037
|
+
requiredType: "string"
|
|
2038
|
+
},
|
|
2039
|
+
SnowflakePassword: {
|
|
2040
|
+
isRequired: true,
|
|
2041
|
+
requiredType: "string"
|
|
2042
|
+
},
|
|
2043
|
+
SnowflakeAuthenticator: {
|
|
2044
|
+
isRequired: false,
|
|
2045
|
+
requiredType: "string",
|
|
2046
|
+
default: "SNOWFLAKE"
|
|
2047
|
+
},
|
|
2048
|
+
SnowflakePrivateKey: {
|
|
2049
|
+
isRequired: false,
|
|
2050
|
+
requiredType: "string",
|
|
2051
|
+
default: null
|
|
2052
|
+
},
|
|
2053
|
+
SnowflakePrivateKeyPassphrase: {
|
|
2054
|
+
isRequired: false,
|
|
2055
|
+
requiredType: "string",
|
|
2056
|
+
default: null
|
|
2057
|
+
},
|
|
2058
|
+
DestinationTableName: {
|
|
2059
|
+
isRequired: true,
|
|
2060
|
+
requiredType: "string",
|
|
2061
|
+
default: "Data"
|
|
2062
|
+
},
|
|
2063
|
+
MaxBufferSize: {
|
|
2064
|
+
isRequired: true,
|
|
2065
|
+
default: 250
|
|
2066
|
+
}
|
|
2067
|
+
}),
|
|
2068
|
+
uniqueKeyColumns,
|
|
2069
|
+
schema,
|
|
2070
|
+
description
|
|
2071
|
+
);
|
|
2072
|
+
this.updatedRecordsBuffer = {};
|
|
2073
|
+
this.totalRecordsProcessed = 0;
|
|
2074
|
+
this.connection = null;
|
|
2075
|
+
}
|
|
2076
|
+
//---- init --------------------------------------------------------
|
|
2077
|
+
/**
|
|
2078
|
+
* Initializing storage - establishes connection and creates table if needed
|
|
2079
|
+
*/
|
|
2080
|
+
async init() {
|
|
2081
|
+
this.checkIfSnowflakeIsConnected();
|
|
2082
|
+
await this.createConnection();
|
|
2083
|
+
await this.testConnection();
|
|
2084
|
+
await this.loadTableSchema();
|
|
2085
|
+
}
|
|
2086
|
+
//----------------------------------------------------------------
|
|
2087
|
+
//---- checkIfSnowflakeIsConnected ---------------------------------
|
|
2088
|
+
checkIfSnowflakeIsConnected() {
|
|
2089
|
+
if (typeof snowflake == "undefined") {
|
|
2090
|
+
throw new Error(`Snowflake SDK is not available. Ensure snowflake-sdk is installed.`);
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
//----------------------------------------------------------------
|
|
2094
|
+
//---- createConnection --------------------------------------------
|
|
2095
|
+
/**
|
|
2096
|
+
* Creates and connects to Snowflake
|
|
2097
|
+
*/
|
|
2098
|
+
async createConnection() {
|
|
2099
|
+
snowflake.configure({
|
|
2100
|
+
logLevel: "OFF"
|
|
2101
|
+
});
|
|
2102
|
+
const connectionConfig = {
|
|
2103
|
+
account: this.config.SnowflakeAccount.value,
|
|
2104
|
+
username: this.config.SnowflakeUsername.value,
|
|
2105
|
+
password: this.config.SnowflakePassword.value,
|
|
2106
|
+
warehouse: this.config.SnowflakeWarehouse.value,
|
|
2107
|
+
database: this.config.SnowflakeDatabase.value,
|
|
2108
|
+
schema: this.config.SnowflakeSchema.value,
|
|
2109
|
+
authenticator: this.config.SnowflakeAuthenticator.value || "SNOWFLAKE"
|
|
2110
|
+
};
|
|
2111
|
+
if (this.config.SnowflakeRole && this.config.SnowflakeRole.value) {
|
|
2112
|
+
connectionConfig.role = this.config.SnowflakeRole.value;
|
|
2113
|
+
}
|
|
2114
|
+
if (this.config.SnowflakePrivateKey && this.config.SnowflakePrivateKey.value) {
|
|
2115
|
+
connectionConfig.privateKey = this.config.SnowflakePrivateKey.value;
|
|
2116
|
+
if (this.config.SnowflakePrivateKeyPassphrase && this.config.SnowflakePrivateKeyPassphrase.value) {
|
|
2117
|
+
connectionConfig.privateKeyPassphrase = this.config.SnowflakePrivateKeyPassphrase.value;
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
this.connection = snowflake.createConnection(connectionConfig);
|
|
2121
|
+
return new Promise((resolve, reject) => {
|
|
2122
|
+
this.connection.connect((err, conn) => {
|
|
2123
|
+
if (err) {
|
|
2124
|
+
reject(new Error(`Failed to connect to Snowflake: ${err.message}`));
|
|
2125
|
+
} else {
|
|
2126
|
+
this.config.logMessage(`Connected to Snowflake (account: ${this.config.SnowflakeAccount.value})`);
|
|
2127
|
+
resolve(conn);
|
|
2128
|
+
}
|
|
2129
|
+
});
|
|
2130
|
+
});
|
|
2131
|
+
}
|
|
2132
|
+
//----------------------------------------------------------------
|
|
2133
|
+
//---- testConnection ----------------------------------------------
|
|
2134
|
+
/**
|
|
2135
|
+
* Tests the connection with a simple query
|
|
2136
|
+
*/
|
|
2137
|
+
async testConnection() {
|
|
2138
|
+
try {
|
|
2139
|
+
await this.executeQuery("SELECT 1 as test");
|
|
2140
|
+
this.config.logMessage("Snowflake connection established successfully");
|
|
2141
|
+
} catch (error) {
|
|
2142
|
+
throw new Error(`Snowflake connection test failed: ${error.message}`);
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
//----------------------------------------------------------------
|
|
2146
|
+
//---- loadTableSchema ---------------------------------------------
|
|
2147
|
+
/**
|
|
2148
|
+
* Loads existing table schema from Snowflake
|
|
2149
|
+
*/
|
|
2150
|
+
async loadTableSchema() {
|
|
2151
|
+
this.existingColumns = await this.getAListOfExistingColumns() || {};
|
|
2152
|
+
if (Object.keys(this.existingColumns).length == 0) {
|
|
2153
|
+
await this.createDatabaseAndSchemaIfNotExist();
|
|
2154
|
+
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
2155
|
+
} else {
|
|
2156
|
+
let selectedFields = this.getSelectedFields();
|
|
2157
|
+
let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2158
|
+
if (newFields.length > 0) {
|
|
2159
|
+
await this.addNewColumns(newFields);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
//----------------------------------------------------------------
|
|
2164
|
+
//---- getAListOfExistingColumns -----------------------------------
|
|
2165
|
+
/**
|
|
2166
|
+
* Reads columns list of the table and returns it as object
|
|
2167
|
+
*
|
|
2168
|
+
* @return columns (object)
|
|
2169
|
+
*/
|
|
2170
|
+
async getAListOfExistingColumns() {
|
|
2171
|
+
let query = `SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE
|
|
2172
|
+
FROM ${this.config.SnowflakeDatabase.value}.INFORMATION_SCHEMA.COLUMNS
|
|
2173
|
+
WHERE TABLE_CATALOG = '${this.config.SnowflakeDatabase.value}'
|
|
2174
|
+
AND TABLE_SCHEMA = UPPER('${this.config.SnowflakeSchema.value}')
|
|
2175
|
+
AND TABLE_NAME = UPPER('${this.config.DestinationTableName.value}')
|
|
2176
|
+
ORDER BY ORDINAL_POSITION`;
|
|
2177
|
+
let queryResults = [];
|
|
2178
|
+
try {
|
|
2179
|
+
queryResults = await this.executeQuery(query);
|
|
2180
|
+
} catch (error) {
|
|
2181
|
+
if (error.message && error.message.includes("does not exist")) {
|
|
2182
|
+
return {};
|
|
2183
|
+
}
|
|
2184
|
+
throw error;
|
|
2185
|
+
}
|
|
2186
|
+
let columns = {};
|
|
2187
|
+
if (Array.isArray(queryResults)) {
|
|
2188
|
+
queryResults.forEach((row) => {
|
|
2189
|
+
const columnName = row.COLUMN_NAME;
|
|
2190
|
+
const dataType = row.DATA_TYPE;
|
|
2191
|
+
columns[columnName] = { "name": columnName, "type": dataType };
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
return columns;
|
|
2195
|
+
}
|
|
2196
|
+
//----------------------------------------------------------------
|
|
2197
|
+
//---- createDatabaseAndSchemaIfNotExist ---------------------------
|
|
2198
|
+
async createDatabaseAndSchemaIfNotExist() {
|
|
2199
|
+
let createDbQuery = `CREATE DATABASE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}`;
|
|
2200
|
+
await this.executeQuery(createDbQuery);
|
|
2201
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2202
|
+
let createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}`;
|
|
2203
|
+
await this.executeQuery(createSchemaQuery);
|
|
2204
|
+
this.config.logMessage(`Database and schema ensured: ${this.config.SnowflakeDatabase.value}.${quotedSchema}`);
|
|
2205
|
+
}
|
|
2206
|
+
//----------------------------------------------------------------
|
|
2207
|
+
//---- createTableIfItDoesntExist ----------------------------------
|
|
2208
|
+
async createTableIfItDoesntExist() {
|
|
2209
|
+
let columns = [];
|
|
2210
|
+
let existingColumns = {};
|
|
2211
|
+
let selectedFields = this.getSelectedFields();
|
|
2212
|
+
let tableColumns = selectedFields.length > 0 ? selectedFields : this.uniqueKeyColumns;
|
|
2213
|
+
for (let i in tableColumns) {
|
|
2214
|
+
let columnName = tableColumns[i];
|
|
2215
|
+
let columnDescription = "";
|
|
2216
|
+
if (!(columnName in this.schema)) {
|
|
2217
|
+
throw new Error(`Required field ${columnName} not found in schema`);
|
|
2218
|
+
}
|
|
2219
|
+
let columnType = this.getColumnType(columnName);
|
|
2220
|
+
if ("description" in this.schema[columnName]) {
|
|
2221
|
+
columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
|
|
2222
|
+
}
|
|
2223
|
+
columns.push(`"${columnName}" ${columnType}${columnDescription}`);
|
|
2224
|
+
existingColumns[columnName] = { "name": columnName, "type": columnType };
|
|
2225
|
+
}
|
|
2226
|
+
columns.push(`PRIMARY KEY (${this.uniqueKeyColumns.map((col) => `"${col}"`).join(",")}) NOT ENFORCED`);
|
|
2227
|
+
let columnsStr = columns.join(",\n ");
|
|
2228
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2229
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2230
|
+
let query = `CREATE TABLE IF NOT EXISTS ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} (
|
|
2231
|
+
${columnsStr}
|
|
2232
|
+
)`;
|
|
2233
|
+
if (this.description) {
|
|
2234
|
+
query += `
|
|
2235
|
+
COMMENT = '${this.description}'`;
|
|
2236
|
+
}
|
|
2237
|
+
await this.executeQuery(query);
|
|
2238
|
+
this.config.logMessage(`Table ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable} was created`);
|
|
2239
|
+
return existingColumns;
|
|
2240
|
+
}
|
|
2241
|
+
//----------------------------------------------------------------
|
|
2242
|
+
//---- addNewColumns -----------------------------------------------
|
|
2243
|
+
/**
|
|
2244
|
+
* ALTER table by adding missed columns
|
|
2245
|
+
*
|
|
2246
|
+
* @param {newColumns} array with a list of new columns
|
|
2247
|
+
*/
|
|
2248
|
+
async addNewColumns(newColumns) {
|
|
2249
|
+
let columns = [];
|
|
2250
|
+
for (var i in newColumns) {
|
|
2251
|
+
let columnName = newColumns[i];
|
|
2252
|
+
if (columnName in this.schema) {
|
|
2253
|
+
let columnDescription = "";
|
|
2254
|
+
let columnType = this.getColumnType(columnName);
|
|
2255
|
+
if ("description" in this.schema[columnName]) {
|
|
2256
|
+
columnDescription = ` COMMENT '${this.schema[columnName]["description"]}'`;
|
|
2257
|
+
}
|
|
2258
|
+
columns.push(`ADD COLUMN IF NOT EXISTS "${columnName}" ${columnType}${columnDescription}`);
|
|
2259
|
+
this.existingColumns[columnName] = { "name": columnName, "type": columnType };
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
if (columns.length > 0) {
|
|
2263
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2264
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2265
|
+
let query = `ALTER TABLE ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}
|
|
2266
|
+
`;
|
|
2267
|
+
query += columns.join(",\n");
|
|
2268
|
+
await this.executeQuery(query);
|
|
2269
|
+
this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`);
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
//----------------------------------------------------------------
|
|
2273
|
+
//---- saveData ----------------------------------------------------
|
|
2274
|
+
/**
|
|
2275
|
+
* Saving data to storage
|
|
2276
|
+
* @param {data} array of assoc objects with records to save
|
|
2277
|
+
*/
|
|
2278
|
+
async saveData(data) {
|
|
2279
|
+
for (const row of data) {
|
|
2280
|
+
let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2281
|
+
if (newFields.length > 0) {
|
|
2282
|
+
await this.addNewColumns(newFields);
|
|
2283
|
+
}
|
|
2284
|
+
this.addRecordToBuffer(row);
|
|
2285
|
+
await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2286
|
+
}
|
|
2287
|
+
await this.saveRecordsAddedToBuffer();
|
|
2288
|
+
}
|
|
2289
|
+
//----------------------------------------------------------------
|
|
2290
|
+
//---- addRecordToBuffer -------------------------------------------
|
|
2291
|
+
/**
|
|
2292
|
+
* Adds record to buffer with deduplication
|
|
2293
|
+
* @param {record} object
|
|
2294
|
+
*/
|
|
2295
|
+
addRecordToBuffer(record) {
|
|
2296
|
+
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2297
|
+
this.updatedRecordsBuffer[uniqueKey] = record;
|
|
2298
|
+
}
|
|
2299
|
+
//----------------------------------------------------------------
|
|
2300
|
+
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
2301
|
+
/**
|
|
2302
|
+
* Add records from buffer to storage
|
|
2303
|
+
* @param (integer) {maxBufferSize} records will be added only if buffer size is larger than this parameter
|
|
2304
|
+
*/
|
|
2305
|
+
async saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2306
|
+
let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
|
|
2307
|
+
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
2308
|
+
await this.executeQueryWithSizeLimit();
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
//----------------------------------------------------------------
|
|
2312
|
+
//---- executeQueryWithSizeLimit -----------------------------------
|
|
2313
|
+
/**
|
|
2314
|
+
* Executes the MERGE query with automatic batching for large datasets
|
|
2315
|
+
*/
|
|
2316
|
+
async executeQueryWithSizeLimit() {
|
|
2317
|
+
const bufferKeys = Object.keys(this.updatedRecordsBuffer);
|
|
2318
|
+
const totalRecords = bufferKeys.length;
|
|
2319
|
+
if (totalRecords === 0) {
|
|
2320
|
+
return;
|
|
2321
|
+
}
|
|
2322
|
+
await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
2323
|
+
this.updatedRecordsBuffer = {};
|
|
2324
|
+
}
|
|
2325
|
+
//----------------------------------------------------------------
|
|
2326
|
+
//---- executeMergeQueryRecursively --------------------------------
|
|
2327
|
+
/**
|
|
2328
|
+
* Recursively attempts to execute MERGE queries, reducing batch size if needed
|
|
2329
|
+
* @param {Array} recordKeys - Array of record keys to process
|
|
2330
|
+
* @param {number} batchSize - Current batch size to attempt
|
|
2331
|
+
*/
|
|
2332
|
+
async executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
2333
|
+
if (recordKeys.length === 0) {
|
|
2334
|
+
return;
|
|
2335
|
+
}
|
|
2336
|
+
if (batchSize < 1) {
|
|
2337
|
+
throw new Error("Cannot process records: batch size reduced below 1");
|
|
2338
|
+
}
|
|
2339
|
+
const currentBatch = recordKeys.slice(0, batchSize);
|
|
2340
|
+
const remainingRecords = recordKeys.slice(batchSize);
|
|
2341
|
+
const query = this.buildMergeQuery(currentBatch);
|
|
2342
|
+
try {
|
|
2343
|
+
await this.executeQuery(query);
|
|
2344
|
+
this.totalRecordsProcessed += currentBatch.length;
|
|
2345
|
+
if (remainingRecords.length > 0) {
|
|
2346
|
+
await this.executeMergeQueryRecursively(remainingRecords, batchSize);
|
|
2347
|
+
}
|
|
2348
|
+
} catch (error) {
|
|
2349
|
+
if (batchSize > 1) {
|
|
2350
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2351
|
+
} else {
|
|
2352
|
+
throw error;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
//----------------------------------------------------------------
|
|
2357
|
+
//---- buildMergeQuery ---------------------------------------------
|
|
2358
|
+
/**
|
|
2359
|
+
* Builds a MERGE query for the specified record keys
|
|
2360
|
+
* @param {Array} recordKeys - Array of record keys to include in the query
|
|
2361
|
+
* @return {string} - The constructed MERGE query
|
|
2362
|
+
*/
|
|
2363
|
+
buildMergeQuery(recordKeys) {
|
|
2364
|
+
let rows = [];
|
|
2365
|
+
for (let i = 0; i < recordKeys.length; i++) {
|
|
2366
|
+
const key = recordKeys[i];
|
|
2367
|
+
let record = this.stringifyNeastedFields(this.updatedRecordsBuffer[key]);
|
|
2368
|
+
let fields = [];
|
|
2369
|
+
for (var j in this.existingColumns) {
|
|
2370
|
+
let columnName = this.existingColumns[j]["name"];
|
|
2371
|
+
let columnType = this.existingColumns[j]["type"];
|
|
2372
|
+
let columnValue = null;
|
|
2373
|
+
if (record[columnName] === void 0 || record[columnName] === null) {
|
|
2374
|
+
columnValue = null;
|
|
2375
|
+
} else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
|
|
2376
|
+
columnValue = DateUtils3.formatDate(record[columnName]);
|
|
2377
|
+
} else if ((columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") && record[columnName] instanceof Date) {
|
|
2378
|
+
const isoString = record[columnName].toISOString();
|
|
2379
|
+
columnValue = isoString.replace("T", " ").substring(0, 19);
|
|
2380
|
+
} else {
|
|
2381
|
+
columnValue = this.obfuscateSpecialCharacters(record[columnName]);
|
|
2382
|
+
}
|
|
2383
|
+
if (columnValue === null) {
|
|
2384
|
+
fields.push(`CAST(NULL AS ${columnType}) AS "${columnName}"`);
|
|
2385
|
+
} else if (columnType.toUpperCase() == "DATE") {
|
|
2386
|
+
fields.push(`TO_DATE('${columnValue}', 'YYYY-MM-DD') AS "${columnName}"`);
|
|
2387
|
+
} else if (columnType.toUpperCase().includes("TIMESTAMP") || columnType.toUpperCase() == "DATETIME") {
|
|
2388
|
+
fields.push(`TO_TIMESTAMP('${columnValue}', 'YYYY-MM-DD HH24:MI:SS') AS "${columnName}"`);
|
|
2389
|
+
} else {
|
|
2390
|
+
fields.push(`CAST('${columnValue}' AS ${columnType}) AS "${columnName}"`);
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
rows.push(`SELECT ${fields.join(",\n ")}`);
|
|
2394
|
+
}
|
|
2395
|
+
let existingColumnsNames = Object.keys(this.existingColumns);
|
|
2396
|
+
const quotedSchema = quoteIdentifier(this.config.SnowflakeSchema.value);
|
|
2397
|
+
const quotedTable = quoteIdentifier(this.config.DestinationTableName.value);
|
|
2398
|
+
let fullTableName = `${this.config.SnowflakeDatabase.value}.${quotedSchema}.${quotedTable}`;
|
|
2399
|
+
let query = `MERGE INTO ${fullTableName} AS target
|
|
2400
|
+
USING (
|
|
2401
|
+
${rows.join("\n UNION ALL\n ")}
|
|
2402
|
+
) AS source
|
|
2403
|
+
|
|
2404
|
+
ON ${this.uniqueKeyColumns.map((item) => `target."${item}" = source."${item}"`).join("\n AND ")}
|
|
2405
|
+
|
|
2406
|
+
WHEN MATCHED THEN
|
|
2407
|
+
UPDATE SET
|
|
2408
|
+
${existingColumnsNames.map((item) => `target."${item}" = source."${item}"`).join(",\n ")}
|
|
2409
|
+
WHEN NOT MATCHED THEN
|
|
2410
|
+
INSERT (
|
|
2411
|
+
${existingColumnsNames.map((item) => `"${item}"`).join(", ")}
|
|
2412
|
+
)
|
|
2413
|
+
VALUES (
|
|
2414
|
+
${existingColumnsNames.map((item) => `source."${item}"`).join(", ")}
|
|
2415
|
+
)`;
|
|
2416
|
+
return query;
|
|
2417
|
+
}
|
|
2418
|
+
//----------------------------------------------------------------
|
|
2419
|
+
//---- executeQuery ------------------------------------------------
|
|
2420
|
+
/**
|
|
2421
|
+
* Executes Snowflake Query and returns a result
|
|
2422
|
+
*
|
|
2423
|
+
* @param {query} string
|
|
2424
|
+
* @return Promise<Array>
|
|
2425
|
+
*/
|
|
2426
|
+
async executeQuery(query) {
|
|
2427
|
+
if (!this.connection) {
|
|
2428
|
+
throw new Error("Snowflake connection not initialized");
|
|
2429
|
+
}
|
|
2430
|
+
return new Promise((resolve, reject) => {
|
|
2431
|
+
this.connection.execute({
|
|
2432
|
+
sqlText: query,
|
|
2433
|
+
complete: (err, stmt, rows) => {
|
|
2434
|
+
if (err) {
|
|
2435
|
+
reject(new Error(`Snowflake query failed: ${err.message}`));
|
|
2436
|
+
} else {
|
|
2437
|
+
resolve(rows || []);
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
});
|
|
2441
|
+
});
|
|
2442
|
+
}
|
|
2443
|
+
//----------------------------------------------------------------
|
|
2444
|
+
//---- obfuscateSpecialCharacters ----------------------------------
|
|
2445
|
+
/**
|
|
2446
|
+
* Escape special characters for SQL string literals
|
|
2447
|
+
* @param {string} inputString - String to escape
|
|
2448
|
+
* @return {string} - Escaped string
|
|
2449
|
+
*/
|
|
2450
|
+
obfuscateSpecialCharacters(inputString) {
|
|
2451
|
+
return String(inputString).replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/"/g, '\\"').replace(/[\x00-\x1F]/g, " ");
|
|
2452
|
+
}
|
|
2453
|
+
//----------------------------------------------------------------
|
|
2454
|
+
//---- getColumnType -----------------------------------------------
|
|
2455
|
+
/**
|
|
2456
|
+
* Get column type for Snowflake from schema
|
|
2457
|
+
* @param {string} columnName - Name of the column
|
|
2458
|
+
* @returns {string} Snowflake column type
|
|
2459
|
+
*/
|
|
2460
|
+
getColumnType(columnName) {
|
|
2461
|
+
var _a;
|
|
2462
|
+
return this.schema[columnName]["SnowflakeFieldType"] || this._convertTypeToStorageType((_a = this.schema[columnName]["type"]) == null ? void 0 : _a.toLowerCase());
|
|
2463
|
+
}
|
|
2464
|
+
//----------------------------------------------------------------
|
|
2465
|
+
//---- _convertTypeToStorageType -----------------------------------
|
|
2466
|
+
/**
|
|
2467
|
+
* Converts generic type to Snowflake-specific type
|
|
2468
|
+
* @param {string} genericType - Generic type from schema
|
|
2469
|
+
* @returns {string} Snowflake column type
|
|
2470
|
+
*/
|
|
2471
|
+
_convertTypeToStorageType(genericType) {
|
|
2472
|
+
if (!genericType) return "VARCHAR";
|
|
2473
|
+
switch (genericType.toLowerCase()) {
|
|
2474
|
+
// Integer types
|
|
2475
|
+
case "integer":
|
|
2476
|
+
case "int32":
|
|
2477
|
+
return "INTEGER";
|
|
2478
|
+
case "int64":
|
|
2479
|
+
case "long":
|
|
2480
|
+
return "BIGINT";
|
|
2481
|
+
// Float types
|
|
2482
|
+
case "float":
|
|
2483
|
+
case "number":
|
|
2484
|
+
case "double":
|
|
2485
|
+
return "FLOAT";
|
|
2486
|
+
case "decimal":
|
|
2487
|
+
return "NUMERIC";
|
|
2488
|
+
// Boolean types
|
|
2489
|
+
case "bool":
|
|
2490
|
+
case "boolean":
|
|
2491
|
+
return "BOOLEAN";
|
|
2492
|
+
// Date/time types
|
|
2493
|
+
case "date":
|
|
2494
|
+
return "DATE";
|
|
2495
|
+
case "datetime":
|
|
2496
|
+
return "TIMESTAMP_NTZ";
|
|
2497
|
+
// Timezone-naive
|
|
2498
|
+
case "timestamp":
|
|
2499
|
+
return "TIMESTAMP_TZ";
|
|
2500
|
+
// Timezone-aware
|
|
2501
|
+
// JSON/Object types
|
|
2502
|
+
case "json":
|
|
2503
|
+
case "object":
|
|
2504
|
+
case "array":
|
|
2505
|
+
return "VARIANT";
|
|
2506
|
+
// Default to VARCHAR for unknown types
|
|
2507
|
+
default:
|
|
2508
|
+
return "VARCHAR";
|
|
2509
|
+
}
|
|
2510
|
+
}
|
|
2511
|
+
//----------------------------------------------------------------
|
|
2512
|
+
};
|
|
2513
|
+
const manifest = {
|
|
2514
|
+
"name": "SnowflakeStorage",
|
|
2515
|
+
"description": "Storage for Snowflake Data Warehouse",
|
|
2516
|
+
"title": "Snowflake",
|
|
2517
|
+
"version": "0.0.0",
|
|
2518
|
+
"author": "OWOX, Inc.",
|
|
2519
|
+
"license": "MIT",
|
|
2520
|
+
"environment": {
|
|
2521
|
+
"node": {
|
|
2522
|
+
"enabled": true,
|
|
2523
|
+
"dependencies": [
|
|
2524
|
+
{
|
|
2525
|
+
"name": "snowflake-sdk",
|
|
2526
|
+
"version": "^1.6.20",
|
|
2527
|
+
"global": [
|
|
2528
|
+
"snowflake"
|
|
2529
|
+
]
|
|
2530
|
+
}
|
|
2531
|
+
]
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
};
|
|
2535
|
+
return {
|
|
2536
|
+
SnowflakeStorage,
|
|
2537
|
+
manifest
|
|
2538
|
+
};
|
|
2539
|
+
})();
|
|
1990
2540
|
const GoogleBigQuery = (function() {
|
|
1991
2541
|
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, OauthFlowException: OauthFlowException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource3, AbstractRunConfig: AbstractRunConfig3, AbstractConnector: AbstractConnector3, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils3, FileUtils: FileUtils3, DateUtils: DateUtils3, CryptoUtils: CryptoUtils3, AsyncUtils: AsyncUtils3, RunConfigDto: RunConfigDto2, OauthCredentialsDto: OauthCredentialsDto2, OauthCredentialsDtoBuilder: OauthCredentialsDtoBuilder2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2, OAUTH_CONSTANTS: OAUTH_CONSTANTS2, OAUTH_SOURCE_CREDENTIALS_KEY: OAUTH_SOURCE_CREDENTIALS_KEY2 } = Core;
|
|
1992
2542
|
var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
|
|
@@ -3099,6 +3649,7 @@ const AwsAthena = (function() {
|
|
|
3099
3649
|
};
|
|
3100
3650
|
})();
|
|
3101
3651
|
const Storages = {
|
|
3652
|
+
Snowflake,
|
|
3102
3653
|
GoogleBigQuery,
|
|
3103
3654
|
AwsAthena
|
|
3104
3655
|
};
|
|
@@ -22038,6 +22589,7 @@ const AvailableConnectors = [
|
|
|
22038
22589
|
"BankOfCanada"
|
|
22039
22590
|
];
|
|
22040
22591
|
const AvailableStorages = [
|
|
22592
|
+
"Snowflake",
|
|
22041
22593
|
"GoogleBigQuery",
|
|
22042
22594
|
"AwsAthena"
|
|
22043
22595
|
];
|
|
@@ -22062,6 +22614,7 @@ const OWOX = {
|
|
|
22062
22614
|
CriteoAds,
|
|
22063
22615
|
BankOfCanada,
|
|
22064
22616
|
// Individual storages
|
|
22617
|
+
Snowflake,
|
|
22065
22618
|
GoogleBigQuery,
|
|
22066
22619
|
AwsAthena
|
|
22067
22620
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@owox/connectors",
|
|
3
|
-
"version": "0.14.0-next-
|
|
3
|
+
"version": "0.14.0-next-20251128101118",
|
|
4
4
|
"description": "Connectors and storages for different data sources",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -44,7 +44,8 @@
|
|
|
44
44
|
"@aws-sdk/client-athena": "3.810.0",
|
|
45
45
|
"@aws-sdk/client-s3": "3.810.0",
|
|
46
46
|
"@aws-sdk/lib-storage": "3.810.0",
|
|
47
|
-
"adm-zip": "0.5.16"
|
|
47
|
+
"adm-zip": "0.5.16",
|
|
48
|
+
"snowflake-sdk": "2.3.1"
|
|
48
49
|
},
|
|
49
50
|
"devDependencies": {
|
|
50
51
|
"@types/node": "^20.0.0",
|