@runnerty/executor-snowflake 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,10 @@
1
+ node_modules
2
+ .all-contributorsrc
3
+ .eslintignore
4
+ .gitignore
5
+ .prettierignore
6
+ LICENSE
7
+ *.sql
8
+ *.png
9
+ *.txt
10
+ TODO
@@ -0,0 +1,10 @@
1
+ {
2
+ "printWidth": 120,
3
+ "singleQuote": true,
4
+ "useTabs": false,
5
+ "tabWidth": 2,
6
+ "semi": true,
7
+ "bracketSpacing": true,
8
+ "trailingComma": "none",
9
+ "arrowParens": "avoid"
10
+ }
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Runnerty Tech S.L.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,277 @@
1
+ <p align="center">
2
+ <a href="http://runnerty.io">
3
+ <img height="257" src="https://runnerty.io/assets/header/logo-stroked.png">
4
+ </a>
5
+ <p align="center">Smart Processes Management</p>
6
+ </p>
7
+
8
+ [![NPM version][npm-image]][npm-url] [![Downloads][downloads-image]][npm-url]
9
+ <a href="#badge">
10
+ <img alt="code style: prettier" src="https://img.shields.io/badge/code_style-prettier-ff69b4.svg">
11
+ </a>
12
+
13
+ # Snowflake executor for [Runnerty]:
14
+
15
+ ### Installation:
16
+
17
+ Through NPM
18
+
19
+ ```bash
20
+ npm i @runnerty/executor-snowflake
21
+ ```
22
+
23
+ You can also add modules to your project with [runnerty-cli]
24
+
25
+ ```bash
26
+ npx runnerty-cli add @runnerty/executor-snowflake
27
+ ```
28
+
29
+ This command installs the module in your project, adds example configuration in your `config.json` and creates an example plan of use.
30
+
31
+ If you have installed [runnerty-cli] globally you can include the module with this command:
32
+
33
+ ```bash
34
+ rty add @runnerty/executor-snowflake
35
+ ```
36
+
37
+ ### Configuration:
38
+
39
+ Add in [config.json]:
40
+
41
+ #### OAuth Authentication (Recommended):
42
+
43
+ This executor uses OAuth token authentication by default. You need to configure your OAuth token provider and set the following environment variables:
44
+
45
+ ```bash
46
+ export SNOWFLAKE_ACCOUNT="myaccount.us-east-1"
47
+ export SNOWFLAKE_USERNAME="myuser@company.com"
48
+ export SNOWFLAKE_PASSWORD="mypassword"
49
+ ```
50
+
51
+ ```json
52
+ {
53
+ "id": "snowflake_default",
54
+ "type": "@runnerty-executor-snowflake",
55
+ "account": "@ENV(SNOWFLAKE_ACCOUNT)",
56
+ "username": "@ENV(SNOWFLAKE_USERNAME)",
57
+ "database": "@ENV(SNOWFLAKE_DATABASE)",
58
+ "schema": "@ENV(SNOWFLAKE_SCHEMA)",
59
+ "warehouse": "@ENV(SNOWFLAKE_WAREHOUSE)",
60
+ "role": "@ENV(SNOWFLAKE_ROLE)"
61
+ }
62
+ ```
63
+
64
+ #### Direct Configuration:
65
+
66
+ ```json
67
+ {
68
+ "id": "snowflake_default",
69
+ "type": "@runnerty-executor-snowflake",
70
+ "account": "myaccount.us-east-1",
71
+ "username": "myuser@company.com",
72
+ "database": "MYDATABASE",
73
+ "schema": "PUBLIC",
74
+ "warehouse": "COMPUTE_WH",
75
+ "role": "MYROLE"
76
+ }
77
+ ```
78
+
79
+ #### Configuration params:
80
+
81
+ | Parameter | Description |
82
+ | :---------- | :-------------------------------------------------------------- |
83
+ | account | Snowflake account identifier (e.g., "myaccount.us-east-1") |
84
+ | username | The Snowflake user to authenticate as. |
85
+ | database | Name of the database to use for this connection. (Optional) |
86
+ | schema | Name of the schema to use for this connection. (Optional) |
87
+ | warehouse | Name of the warehouse to use for this connection. (Optional) |
88
+ | role | Name of the role to use for this connection. (Optional) |
89
+ | timeout | Connection timeout in milliseconds. (Default: 60000) |
90
+ | application | Application name for connection tracking. (Default: "runnerty") |
91
+
92
+ ### Plan samples:
93
+
94
+ Add in [plan.json]:
95
+
96
+ #### Basic query:
97
+
98
+ ```json
99
+ {
100
+ "id": "snowflake_default",
101
+ "command_file": "./sql/my_query.sql"
102
+ }
103
+ ```
104
+
105
+ ```json
106
+ {
107
+ "id": "snowflake_default",
108
+ "command": "SELECT CURRENT_TIMESTAMP() as now, CURRENT_USER() as user"
109
+ }
110
+ ```
111
+
112
+ #### Query with parameters:
113
+
114
+ ```json
115
+ {
116
+ "id": "snowflake_default",
117
+ "command": "SELECT * FROM users WHERE created_date >= :start_date AND status = :status",
118
+ "args": {
119
+ "start_date": "2023-01-01",
120
+ "status": "active"
121
+ }
122
+ }
123
+ ```
124
+
125
+ #### Export to files:
126
+
127
+ ```json
128
+ {
129
+ "id": "snowflake_default",
130
+ "command": "SELECT * FROM sales_data WHERE year = 2023",
131
+ "xlsxFileExport": "./reports/sales_2023.xlsx",
132
+ "xlsxSheetName": "Sales Report",
133
+ "xlsxAuthorName": "Data Team"
134
+ }
135
+ ```
136
+
137
+ ### Generation of files:
138
+
139
+ The results can be exported to csv, xlsx and json format files. These files are generated using streams for optimal performance with large datasets.
140
+ You only have to indicate the corresponding property in the parameters:
141
+
142
+ #### XLSX
143
+
144
+ XLSX Format with streaming support for large datasets.
145
+
146
+ | Parameter | Description |
147
+ | :------------- | :---------------------------- |
148
+ | xlsxFileExport | Path of xlsx file export. |
149
+ | xlsxAuthorName | Author file name. (Optional) |
150
+ | xlsxSheetName | Name of the sheet. (Optional) |
151
+
152
+ Sample:
153
+
154
+ ```json
155
+ {
156
+ "id": "snowflake_default",
157
+ "command": "SELECT * FROM users",
158
+ "xlsxFileExport": "./exports/users.xlsx",
159
+ "xlsxAuthorName": "Runnerty",
160
+ "xlsxSheetName": "Users Data"
161
+ }
162
+ ```
163
+
164
+ #### CSV
165
+
166
+ CSV Format with streaming support and customizable options.
167
+
168
+ | Parameter | Description |
169
+ | :---------------------- | :-------------------------------------------------------------------- |
170
+ | csvFileExport | Path of csv file export. |
171
+ | csvOptions/headers | Type: boolean. Whether to include headers in the CSV. (Default: true) |
172
+ | csvOptions/delimiter | Alternate delimiter. (Default: ',') |
173
+ | csvOptions/quote | Alternate quote. (Default: '"') |
174
+ | csvOptions/rowDelimiter | Specify an alternate row delimiter (i.e \r\n). (Default: '\n') |
175
+ | csvOptions/escape | Alternate escaping value. (Default: '"') |
176
+
177
+ Sample:
178
+
179
+ ```json
180
+ {
181
+ "id": "snowflake_default",
182
+ "command": "SELECT * FROM users",
183
+ "csvFileExport": "@GV(WORK_DIR)/users.csv",
184
+ "csvOptions": {
185
+ "delimiter": ";",
186
+ "quote": "\"",
187
+ "headers": true
188
+ }
189
+ }
190
+ ```
191
+
192
+ #### JSON
193
+
194
+ JSON Format with streaming support.
195
+
196
+ | Parameter | Description |
197
+ | :------------- | :------------------------ |
198
+ | jsonFileExport | Path of json file export. |
199
+
200
+ Sample:
201
+
202
+ ```json
203
+ {
204
+ "id": "snowflake_default",
205
+ "command": "SELECT * FROM users",
206
+ "jsonFileExport": "@GV(WORK_DIR)/users.json"
207
+ }
208
+ ```
209
+
210
+ ### Output (Process values):
211
+
212
+ #### Standard
213
+
214
+ - `PROCESS_EXEC_MSG_OUTPUT`: Snowflake output message.
215
+ - `PROCESS_EXEC_ERR_OUTPUT`: Error output message.
216
+ - `PROCESS_EXEC_COMMAND_EXECUTED`: Executed SQL command.
217
+
218
+ #### Query output
219
+
220
+ - `PROCESS_EXEC_DATA_OUTPUT`: Snowflake query output data (array of objects).
221
+ - `PROCESS_EXEC_DB_COUNTROWS`: Snowflake query count rows.
222
+ - `PROCESS_EXEC_DB_FIRSTROW`: Snowflake query first row data (object).
223
+ - `PROCESS_EXEC_DB_FIRSTROW_[FIELD_NAME]`: Snowflake first row field data.
224
+
225
+ Example of first row field access:
226
+
227
+ ```json
228
+ {
229
+ "id": "snowflake_default",
230
+ "command": "SELECT user_id, email, created_date FROM users LIMIT 1"
231
+ }
232
+ ```
233
+
234
+ Available variables after execution:
235
+
236
+ - `PROCESS_EXEC_DB_FIRSTROW_USER_ID`: First row user_id value
237
+ - `PROCESS_EXEC_DB_FIRSTROW_EMAIL`: First row email value
238
+ - `PROCESS_EXEC_DB_FIRSTROW_CREATED_DATE`: First row created_date value
239
+
240
+ ### Authentication Setup:
241
+
242
+ This executor requires OAuth authentication setup. Make sure you have:
243
+
244
+ 1. **Environment variables configured:**
245
+
246
+ ```bash
247
+ export SNOWFLAKE_ACCOUNT="your-account"
248
+ export SNOWFLAKE_USERNAME="your-username"
249
+ export SNOWFLAKE_PASSWORD="your-password"
250
+ ```
251
+
252
+ 2. **OAuth token provider configured** (internal API endpoint)
253
+
254
+ 3. **Required dependencies installed:**
255
+ ```bash
256
+ npm install snowflake-sdk axios exceljs fast-csv jsonstream
257
+ ```
258
+
259
+ ### Features:
260
+
261
+ - ✅ **OAuth Authentication** - Secure token-based authentication
262
+ - ✅ **Streaming Support** - Efficient processing of large datasets
263
+ - ✅ **Multiple Export Formats** - XLSX, CSV, JSON
264
+ - ✅ **Parameterized Queries** - Support for `:parameter` placeholders
265
+ - ✅ **Environment Variables** - Secure configuration management
266
+ - ✅ **Error Handling** - Comprehensive error reporting
267
+ - ✅ **Connection Pooling** - Optimized connection management
268
+
269
+ [runnerty]: http://www.runnerty.io
270
+ [downloads-image]: https://img.shields.io/npm/dm/@runnerty/executor-snowflake.svg
271
+ [npm-url]: https://www.npmjs.com/package/@runnerty/executor-snowflake
272
+ [npm-image]: https://img.shields.io/npm/v/@runnerty/executor-snowflake.svg
273
+ [david-badge]: https://david-dm.org/runnerty/executor-snowflake.svg
274
+ [david-badge-url]: https://david-dm.org/runnerty/executor-snowflake
275
+ [config.json]: http://docs.runnerty.io/config/
276
+ [runnerty-cli]: https://www.npmjs.com/package/runnerty-cli
277
+ [plan.json]: http://docs.runnerty.io/plan/
package/get-token.js ADDED
@@ -0,0 +1,48 @@
1
+ 'use strict';
2
+
3
+ const axios = require('axios');
4
+
5
+ /**
6
+ * Obtiene un token OAuth desde la API de AWS
7
+ * @returns {Promise<string>} Token OAuth
8
+ */
9
+ async function getToken(params) {
10
+ const url = params.url;
11
+
12
+ const headers = {
13
+ 'Content-Type': 'application/json'
14
+ };
15
+
16
+ const body = JSON.stringify({
17
+ username: params.user,
18
+ password: params.password
19
+ });
20
+
21
+ try {
22
+ const response = await axios({
23
+ method: 'post',
24
+ url: url,
25
+ headers: headers,
26
+ data: body
27
+ });
28
+
29
+ const token = response.data?.token;
30
+
31
+ if (!token) {
32
+ throw new Error('Token not found in response');
33
+ }
34
+
35
+ return token;
36
+ } catch (error) {
37
+ console.error('❌ Error getting token:', error.message);
38
+ if (error.response) {
39
+ console.error('Response status:', error.response.status);
40
+ console.error('Response data:', error.response.data);
41
+ }
42
+ throw error;
43
+ }
44
+ }
45
+ // Exportar las funciones
46
+ module.exports = {
47
+ getToken
48
+ };
package/index.js ADDED
@@ -0,0 +1,455 @@
1
+ 'use strict';
2
+
3
+ const snowflake = require('snowflake-sdk');
4
+ const fs = require('fs');
5
+ const fsp = require('fs').promises;
6
+ const path = require('path');
7
+ const JSONStream = require('jsonstream');
8
+ const Excel = require('exceljs');
9
+ const csv = require('fast-csv');
10
+ const { getToken } = require('./get-token');
11
+
12
+ const Executor = require('@runnerty/module-core').Executor;
13
+
14
+ class snowflakeExecutor extends Executor {
15
+ constructor(process) {
16
+ super(process);
17
+ this.ended = false;
18
+ this.endOptions = {
19
+ end: 'end'
20
+ };
21
+ }
22
+
23
+ async exec(params) {
24
+ try {
25
+ // Cargar comando SQL
26
+ if (!params.command) {
27
+ if (params.command_file) {
28
+ try {
29
+ await fsp.access(params.command_file, fs.constants.F_OK | fs.constants.W_OK);
30
+ params.command = await fsp.readFile(params.command_file, 'utf8');
31
+ } catch (err) {
32
+ throw new Error(`Load SQLFile: ${err}`);
33
+ }
34
+ } else {
35
+ this.endOptions.end = 'error';
36
+ this.endOptions.messageLog = 'execute-snowflake dont have command or command_file';
37
+ this.endOptions.err_output = 'execute-snowflake dont have command or command_file';
38
+ this._end(this.endOptions);
39
+ return;
40
+ }
41
+ }
42
+
43
+ const query = await this.prepareQuery(params);
44
+ this.endOptions.command_executed = query;
45
+
46
+ // Crear conexión y ejecutar consulta
47
+ const connection = await this.createConnection(params);
48
+ await this.executeQuery(connection, query);
49
+ } catch (error) {
50
+ this.error(error);
51
+ }
52
+ }
53
+
54
+ async createConnection(params) {
55
+ try {
56
+ const token = await getToken(params);
57
+
58
+ const connectionOptions = this.getConnectionOptions(params, token);
59
+
60
+ return new Promise((resolve, reject) => {
61
+ const connection = snowflake.createConnection(connectionOptions);
62
+
63
+ // Usar connect normal con token OAuth
64
+ connection.connect((err, conn) => {
65
+ if (err) {
66
+ reject(new Error(`Snowflake connection error: ${err.message}`));
67
+ } else {
68
+ resolve(conn);
69
+ }
70
+ });
71
+ });
72
+ } catch (error) {
73
+ throw new Error(`Failed to get token or connect: ${error.message}`);
74
+ }
75
+ }
76
+
77
+ getConnectionOptions(params, token) {
78
+ return {
79
+ account: params.account,
80
+ username: params.user,
81
+ authenticator: 'oauth', // Especificar que usamos OAuth
82
+ token: token, // Token OAuth obtenido de la API
83
+ database: params.database,
84
+ schema: params.schema,
85
+ warehouse: params.warehouse,
86
+ role: params.role,
87
+ timeout: params.timeout || 60000,
88
+ application: params.application || 'runnerty'
89
+ };
90
+ }
91
+
92
+ async executeQuery(connection, query) {
93
+ return new Promise((resolve, reject) => {
94
+ connection.execute({
95
+ sqlText: query,
96
+ streamResult: true,
97
+ complete: (err, stmt, rows) => {
98
+ if (err) {
99
+ reject(err);
100
+ return;
101
+ }
102
+
103
+ // Procesar resultados con streaming si está disponible
104
+ if (stmt.streamRows) {
105
+ const results = [];
106
+ let firstRow = {};
107
+ let rowCounter = 0;
108
+ const stream = stmt.streamRows();
109
+
110
+ stream.on('data', row => {
111
+ if (rowCounter === 0) firstRow = row;
112
+ results.push(row);
113
+ rowCounter++;
114
+ });
115
+
116
+ stream.on('end', () => {
117
+ this.prepareEndOptions(firstRow, rowCounter, results);
118
+ this._end(this.endOptions);
119
+ connection.destroy();
120
+ resolve();
121
+ });
122
+
123
+ stream.on('error', error => {
124
+ reject(error);
125
+ });
126
+ } else {
127
+ // Fallback sin streaming
128
+ this.prepareEndOptions(rows[0], rows ? rows.length : 0, rows);
129
+ this._end(this.endOptions);
130
+ connection.destroy();
131
+ resolve();
132
+ }
133
+ }
134
+ });
135
+ });
136
+ }
137
+
138
+ error(err, connection) {
139
+ console.error('❌ Snowflake Error:', err.message || err);
140
+ this.endOptions.end = 'error';
141
+ this.endOptions.messageLog = `execute-snowflake: ${err.message || err}`;
142
+ this.endOptions.err_output = `execute-snowflake: ${err.message || err}`;
143
+ if (connection) connection.destroy();
144
+ this._end(this.endOptions);
145
+ }
146
+
147
+ _end(endOptions) {
148
+ if (!this.ended) {
149
+ this.ended = true;
150
+ super.end(endOptions);
151
+ }
152
+ }
153
+
154
+ async prepareQuery(values) {
155
+ let query = values.command;
156
+
157
+ // Reemplazar argumentos en la consulta
158
+ if (values.args) {
159
+ for (const key in values.args) {
160
+ const regex = new RegExp(`:${key}`, 'g');
161
+ query = query.replace(regex, values.args[key]);
162
+ }
163
+ }
164
+
165
+ return query;
166
+ }
167
+
168
+ prepareEndOptions(firstRow, rowCounter, results) {
169
+ this.endOptions.data_output = results || [];
170
+ this.endOptions.extra_output = {
171
+ db_countrows: rowCounter || 0,
172
+ db_firstrow: firstRow || {}
173
+ };
174
+
175
+ // Variables globales para Runnerty
176
+ if (firstRow) {
177
+ Object.keys(firstRow).forEach(key => {
178
+ this.endOptions[`db_firstrow_${key.toLowerCase()}`] = firstRow[key];
179
+ });
180
+ }
181
+ }
182
+
183
+ async queryToJSON(connection, query, params) {
184
+ try {
185
+ // Verificar que el directorio del archivo de exportación existe
186
+ await fsp.access(path.dirname(params.jsonFileExport));
187
+
188
+ const fileStreamWriter = fs.createWriteStream(params.jsonFileExport);
189
+
190
+ fileStreamWriter.on('error', error => {
191
+ this.error(error, connection);
192
+ });
193
+
194
+ return new Promise((resolve, reject) => {
195
+ connection.execute({
196
+ sqlText: query,
197
+ streamResult: true,
198
+ complete: (err, stmt, rows) => {
199
+ if (err) {
200
+ reject(err);
201
+ return;
202
+ }
203
+
204
+ // Usar streaming si está disponible
205
+ if (stmt.streamRows) {
206
+ let firstRow = {};
207
+ let rowCounter = 0;
208
+ let isFirstRow = true;
209
+ const stream = stmt.streamRows();
210
+
211
+ stream.on('data', row => {
212
+ if (isFirstRow) {
213
+ firstRow = row;
214
+ isFirstRow = false;
215
+ }
216
+ rowCounter++;
217
+ });
218
+
219
+ stream.on('end', () => {
220
+ this.prepareEndOptions(firstRow, rowCounter);
221
+ this._end(this.endOptions);
222
+ connection.destroy();
223
+ resolve();
224
+ });
225
+
226
+ stream.on('error', error => {
227
+ this.error(error, connection);
228
+ reject(error);
229
+ });
230
+
231
+ // Pipe los datos a JSON y luego al archivo
232
+ stream.pipe(JSONStream.stringify()).pipe(fileStreamWriter);
233
+ } else {
234
+ // Fallback sin streaming - escribir directamente los rows
235
+ fileStreamWriter.write(JSON.stringify(rows, null, 2));
236
+ fileStreamWriter.end();
237
+
238
+ fileStreamWriter.on('finish', () => {
239
+ this.prepareEndOptions(rows[0], rows ? rows.length : 0);
240
+ this._end(this.endOptions);
241
+ connection.destroy();
242
+ resolve();
243
+ });
244
+ }
245
+ }
246
+ });
247
+ });
248
+ } catch (err) {
249
+ this.error(err, connection);
250
+ }
251
+ }
252
+
253
+ async queryToXLSX(connection, query, params) {
254
+ try {
255
+ // Verificar que el directorio del archivo de exportación existe
256
+ await fsp.access(path.dirname(params.xlsxFileExport));
257
+
258
+ const fileStreamWriter = fs.createWriteStream(params.xlsxFileExport);
259
+
260
+ const options = {
261
+ stream: fileStreamWriter,
262
+ useStyles: true,
263
+ useSharedStrings: true
264
+ };
265
+ const workbook = new Excel.stream.xlsx.WorkbookWriter(options);
266
+
267
+ const author = 'Runnerty';
268
+ const sheetName = 'Sheet';
269
+ const sheet = workbook.addWorksheet(params.xlsxSheetName ? params.xlsxSheetName : sheetName);
270
+ workbook.creator = params.xlsxAuthorName ? params.xlsxAuthorName : author;
271
+ workbook.lastPrinted = new Date();
272
+
273
+ fileStreamWriter.on('error', error => {
274
+ this.error(error, connection);
275
+ });
276
+
277
+ return new Promise((resolve, reject) => {
278
+ connection.execute({
279
+ sqlText: query,
280
+ streamResult: true,
281
+ complete: (err, stmt, rows) => {
282
+ if (err) {
283
+ reject(err);
284
+ return;
285
+ }
286
+
287
+ // Usar streaming si está disponible
288
+ if (stmt.streamRows) {
289
+ let firstRow = {};
290
+ let rowCounter = 0;
291
+ let isFirstRow = true;
292
+ const stream = stmt.streamRows();
293
+
294
+ stream.on('data', row => {
295
+ if (isFirstRow) {
296
+ firstRow = row;
297
+ sheet.columns = this.generateHeader(row);
298
+ isFirstRow = false;
299
+ }
300
+ sheet.addRow(row).commit();
301
+ rowCounter++;
302
+ });
303
+
304
+ stream.on('end', async () => {
305
+ try {
306
+ await workbook.commit();
307
+ this.prepareEndOptions(firstRow, rowCounter);
308
+ this._end(this.endOptions);
309
+ connection.destroy();
310
+ resolve();
311
+ } catch (commitError) {
312
+ this.error(commitError, connection);
313
+ reject(commitError);
314
+ }
315
+ });
316
+
317
+ stream.on('error', error => {
318
+ this.error(error, connection);
319
+ reject(error);
320
+ });
321
+ } else {
322
+ // Fallback sin streaming
323
+ try {
324
+ if (rows && rows.length > 0) {
325
+ sheet.columns = this.generateHeader(rows[0]);
326
+ rows.forEach(row => {
327
+ sheet.addRow(row).commit();
328
+ });
329
+ }
330
+
331
+ workbook
332
+ .commit()
333
+ .then(() => {
334
+ this.prepareEndOptions(rows[0], rows ? rows.length : 0);
335
+ this._end(this.endOptions);
336
+ connection.destroy();
337
+ resolve();
338
+ })
339
+ .catch(commitError => {
340
+ this.error(commitError, connection);
341
+ reject(commitError);
342
+ });
343
+ } catch (fallbackError) {
344
+ this.error(fallbackError, connection);
345
+ reject(fallbackError);
346
+ }
347
+ }
348
+ }
349
+ });
350
+ });
351
+ } catch (err) {
352
+ this.error(err, connection);
353
+ }
354
+ }
355
+
356
+ async queryToCSV(connection, query, params) {
357
+ try {
358
+ // Verificar que el directorio del archivo de exportación existe
359
+ await fsp.access(path.dirname(params.csvFileExport));
360
+
361
+ const fileStreamWriter = fs.createWriteStream(params.csvFileExport);
362
+
363
+ const paramsCSV = params.csvOptions || {};
364
+ if (!paramsCSV.hasOwnProperty('headers')) paramsCSV.headers = true;
365
+
366
+ const csvStream = csv.format(paramsCSV).on('error', err => {
367
+ this.error(err, connection);
368
+ });
369
+
370
+ fileStreamWriter.on('error', error => {
371
+ this.error(error, connection);
372
+ });
373
+
374
+ return new Promise((resolve, reject) => {
375
+ connection.execute({
376
+ sqlText: query,
377
+ streamResult: true,
378
+ complete: (err, stmt, rows) => {
379
+ if (err) {
380
+ reject(err);
381
+ return;
382
+ }
383
+
384
+ // Usar streaming si está disponible
385
+ if (stmt.streamRows) {
386
+ let firstRow = {};
387
+ let rowCounter = 0;
388
+ let isFirstRow = true;
389
+ const stream = stmt.streamRows();
390
+
391
+ stream.on('data', row => {
392
+ if (isFirstRow) {
393
+ firstRow = row;
394
+ isFirstRow = false;
395
+ }
396
+ rowCounter++;
397
+ });
398
+
399
+ stream.on('end', () => {
400
+ this.prepareEndOptions(firstRow, rowCounter);
401
+ this._end(this.endOptions);
402
+ connection.destroy();
403
+ resolve();
404
+ });
405
+
406
+ stream.on('error', error => {
407
+ this.error(error, connection);
408
+ reject(error);
409
+ });
410
+
411
+ // Pipe los datos a CSV y luego al archivo
412
+ stream.pipe(csvStream).pipe(fileStreamWriter);
413
+ } else {
414
+ // Fallback sin streaming
415
+ try {
416
+ if (rows && rows.length > 0) {
417
+ rows.forEach(row => {
418
+ csvStream.write(row);
419
+ });
420
+ }
421
+ csvStream.end();
422
+
423
+ csvStream.on('finish', () => {
424
+ this.prepareEndOptions(rows[0], rows ? rows.length : 0);
425
+ this._end(this.endOptions);
426
+ connection.destroy();
427
+ resolve();
428
+ });
429
+ } catch (fallbackError) {
430
+ this.error(fallbackError, connection);
431
+ reject(fallbackError);
432
+ }
433
+ }
434
+ }
435
+ });
436
+ });
437
+ } catch (err) {
438
+ this.error(err, connection);
439
+ }
440
+ }
441
+
442
+ generateHeader(row) {
443
+ const headers = [];
444
+ Object.keys(row).forEach(key => {
445
+ headers.push({
446
+ header: key,
447
+ key: key,
448
+ width: 20
449
+ });
450
+ });
451
+ return headers;
452
+ }
453
+ }
454
+
455
+ module.exports = snowflakeExecutor;
package/package.json ADDED
@@ -0,0 +1,27 @@
1
+ {
2
+ "name": "@runnerty/executor-snowflake",
3
+ "version": "1.0.0",
4
+ "description": "Runnerty module: Snowflake executor (MVP)",
5
+ "author": "Coderty",
6
+ "license": "MIT",
7
+ "main": "index.js",
8
+ "scripts": {
9
+ "test": "node test-simple.js"
10
+ },
11
+ "dependencies": {
12
+ "@runnerty/module-core": "~3.1.1",
13
+ "exceljs": "^4.4.0",
14
+ "jsonstream": "^1.0.3",
15
+ "snowflake-sdk": "^1.9.0",
16
+ "fast-csv": "^4.3.6"
17
+ },
18
+ "devDependencies": {
19
+ "eslint": "^8.19.0",
20
+ "prettier": "^2.7.1"
21
+ },
22
+ "keywords": [
23
+ "runnerty",
24
+ "executor",
25
+ "snowflake"
26
+ ]
27
+ }
@@ -0,0 +1,14 @@
1
+ -- Example SQL file for Snowflake
2
+ SELECT
3
+ 'Runnerty Snowflake Test' as test_name,
4
+ CURRENT_TIMESTAMP() as execution_time,
5
+ CURRENT_USER() as executed_by,
6
+ CURRENT_DATABASE() as database_name,
7
+ CURRENT_SCHEMA() as schema_name;
8
+
9
+ -- Generate some sample data
10
+ SELECT
11
+ seq4() as row_number,
12
+ UNIFORM(1, 100, RANDOM()) as random_number,
13
+ TO_CHAR(DATEADD(day, seq4(), CURRENT_DATE()), 'YYYY-MM-DD') as future_date
14
+ FROM TABLE(GENERATOR(ROWCOUNT => 3));
@@ -0,0 +1,15 @@
1
+ {
2
+ "executors": [
3
+ {
4
+ "id": "snowflake_default",
5
+ "type": "@runnerty-executor-snowflake",
6
+ "account": "@ENV(SNOWFLAKE_ACCOUNT)",
7
+ "username": "@ENV(SNOWFLAKE_USERNAME)",
8
+ "password": "@ENV(SNOWFLAKE_PASSWORD)",
9
+ "database": "@ENV(SNOWFLAKE_DATABASE)",
10
+ "schema": "@ENV(SNOWFLAKE_SCHEMA)",
11
+ "warehouse": "@ENV(SNOWFLAKE_WAREHOUSE)",
12
+ "role": "@ENV(SNOWFLAKE_ROLE)"
13
+ }
14
+ ]
15
+ }
@@ -0,0 +1,63 @@
1
+ {
2
+ "$schema": "https://raw.githubusercontent.com/runnerty/schemas/master/schemas/3.1/plan.json",
3
+ "chains": [
4
+ {
5
+ "id": "CHAIN_SNOWFLAKE_SSO",
6
+ "name": "SNOWFLAKE SSO Examples",
7
+ "defaults_processes": {
8
+ "notifications": {
9
+ "on_start": [
10
+ {
11
+ "id": "console_default",
12
+ "message": "🚀 START: @GV(PROCESS_ID) - @GETDATE('YYYY-MM-DD HH:mm:ss')"
13
+ }
14
+ ],
15
+ "on_fail": [
16
+ {
17
+ "id": "console_default",
18
+ "message": "❌ ERROR: @GV(PROCESS_ID) - @GV(PROCESS_EXEC_ERR_OUTPUT)",
19
+ "mode": "error"
20
+ }
21
+ ],
22
+ "on_end": [
23
+ {
24
+ "id": "console_default",
25
+ "message": "✅ END: @GV(PROCESS_ID) - Rows: @GV(PROCESS_EXEC_DB_COUNTROWS) - @GETDATE('YYYY-MM-DD HH:mm:ss')"
26
+ }
27
+ ]
28
+ }
29
+ },
30
+ "processes": [
31
+ {
32
+ "id": "SNOWFLAKE_SSO_TEST",
33
+ "name": "Test SSO Connection",
34
+ "exec": {
35
+ "id": "snowflake_default",
36
+ "command": "SELECT CURRENT_TIMESTAMP() as now, CURRENT_USER() as user, CURRENT_DATABASE() as db, CURRENT_SCHEMA() as schema"
37
+ }
38
+ },
39
+ {
40
+ "id": "SNOWFLAKE_WITH_PARAMS",
41
+ "name": "Query with Parameters",
42
+ "exec": {
43
+ "id": "snowflake_default",
44
+ "command": "SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = :schema AND TABLE_NAME LIKE :table_pattern LIMIT :limit",
45
+ "args": {
46
+ "schema": "'REPORTING_PUB'",
47
+ "table_pattern": "'%'",
48
+ "limit": "5"
49
+ }
50
+ }
51
+ },
52
+ {
53
+ "id": "SNOWFLAKE_FROM_FILE",
54
+ "name": "Execute SQL from File",
55
+ "exec": {
56
+ "id": "snowflake_default",
57
+ "command_file": "./sql/example_query.sql"
58
+ }
59
+ }
60
+ ]
61
+ }
62
+ ]
63
+ }
package/schema.json ADDED
@@ -0,0 +1,112 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-06/schema#",
3
+ "title": "@runnerty-executor-snowflake",
4
+ "definitions": {
5
+ "config": {
6
+ "type": "object",
7
+ "properties": {
8
+ "id": {
9
+ "type": "string"
10
+ },
11
+ "type": {
12
+ "type": "string",
13
+ "pattern": "@runnerty-executor-snowflake"
14
+ },
15
+ "user": {
16
+ "type": "string"
17
+ },
18
+ "password": {
19
+ "type": "string"
20
+ },
21
+ "database": {
22
+ "type": "string"
23
+ },
24
+ "host": {
25
+ "type": "string"
26
+ },
27
+ "port": {
28
+ "type": "string"
29
+ },
30
+ "debug": {
31
+ "type": "boolean"
32
+ },
33
+ "ssl": {
34
+ "type": "object"
35
+ },
36
+ "connectionTimeoutMillis": {
37
+ "type": "number"
38
+ },
39
+ "keepAliveInitialDelayMillis": {
40
+ "type": "number"
41
+ },
42
+ "keepAlive": {
43
+ "type": "boolean"
44
+ },
45
+ "parseInputDatesAsUTC": {
46
+ "type": "boolean"
47
+ },
48
+ "query_timeout": {
49
+ "type": ["boolean", "number"]
50
+ },
51
+ "statement_timeout": {
52
+ "type": ["boolean", "number"]
53
+ },
54
+ "idle_in_transaction_session_timeout": {
55
+ "type": ["boolean", "number"]
56
+ },
57
+ "application_name": {
58
+ "type": "string"
59
+ },
60
+ "encoding": {
61
+ "type": "string"
62
+ }
63
+ }
64
+ },
65
+ "params": {
66
+ "oneOf": [
67
+ {
68
+ "type": "object",
69
+ "required": ["id", "command"],
70
+ "properties": {
71
+ "id": {
72
+ "type": "string"
73
+ },
74
+ "command": {
75
+ "type": "string"
76
+ },
77
+ "args": {
78
+ "type": "object"
79
+ },
80
+ "fileExport": {
81
+ "type": "string"
82
+ },
83
+ "localInFile": {
84
+ "type": "string"
85
+ }
86
+ }
87
+ },
88
+ {
89
+ "type": "object",
90
+ "required": ["id", "command_file"],
91
+ "properties": {
92
+ "id": {
93
+ "type": "string"
94
+ },
95
+ "command_file": {
96
+ "type": "string"
97
+ },
98
+ "args": {
99
+ "type": "object"
100
+ },
101
+ "fileExport": {
102
+ "type": "string"
103
+ },
104
+ "localInFile": {
105
+ "type": "string"
106
+ }
107
+ }
108
+ }
109
+ ]
110
+ }
111
+ }
112
+ }