@forzalabs/remora 1.0.21 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/automap.js +26 -42
- package/actions/compile.js +27 -43
- package/actions/create_consumer.js +24 -40
- package/actions/create_producer.js +16 -32
- package/actions/debug.js +18 -34
- package/actions/deploy.js +30 -46
- package/actions/discover.js +13 -29
- package/actions/init.js +29 -45
- package/actions/mock.js +16 -32
- package/actions/run.js +34 -52
- package/actions/sample.js +42 -58
- package/index.js +38 -43
- package/package.json +4 -4
- package/workers/ExecutorWorker.js +18 -32
- package/Constants.js +0 -34
- package/core/Affirm.js +0 -42
- package/core/Algo.js +0 -160
- package/core/dste/DSTE.js +0 -113
- package/core/logger/DebugLogService.js +0 -48
- package/core/logger/DevelopmentLogService.js +0 -70
- package/core/logger/LocalLogService.js +0 -70
- package/core/logger/Logger.js +0 -54
- package/database/DatabaseEngine.js +0 -149
- package/database/DatabaseStructure.js +0 -27
- package/definitions/DatasetDefinitions.js +0 -2
- package/definitions/ExecutorDefinitions.js +0 -2
- package/definitions/ProcessENV.js +0 -2
- package/definitions/agents/DestinationDriver.js +0 -2
- package/definitions/agents/SourceDriver.js +0 -2
- package/definitions/cli.js +0 -2
- package/definitions/database/ApiKeys.js +0 -2
- package/definitions/database/Stored.js +0 -7
- package/definitions/database/UsageStat.js +0 -2
- package/definitions/database/User.js +0 -2
- package/definitions/json_schemas/consumer-schema.json +0 -1226
- package/definitions/json_schemas/producer-schema.json +0 -308
- package/definitions/json_schemas/project-schema.json +0 -100
- package/definitions/json_schemas/source-schema.json +0 -249
- package/definitions/requests/ConsumerRequest.js +0 -2
- package/definitions/requests/Developer.js +0 -2
- package/definitions/requests/Mapping.js +0 -2
- package/definitions/requests/ProducerRequest.js +0 -2
- package/definitions/requests/Request.js +0 -2
- package/definitions/resources/Compiled.js +0 -2
- package/definitions/resources/Consumer.js +0 -2
- package/definitions/resources/Environment.js +0 -2
- package/definitions/resources/Library.js +0 -2
- package/definitions/resources/Producer.js +0 -2
- package/definitions/resources/Project.js +0 -2
- package/definitions/resources/Schema.js +0 -2
- package/definitions/resources/Source.js +0 -2
- package/definitions/temp.js +0 -2
- package/definitions/transform/Transformations.js +0 -2
- package/drivers/DeltaShareDriver.js +0 -186
- package/drivers/DriverFactory.js +0 -72
- package/drivers/DriverHelper.js +0 -248
- package/drivers/HttpApiDriver.js +0 -208
- package/drivers/RedshiftDriver.js +0 -184
- package/drivers/files/LocalDestinationDriver.js +0 -146
- package/drivers/files/LocalSourceDriver.js +0 -405
- package/drivers/s3/S3DestinationDriver.js +0 -197
- package/drivers/s3/S3SourceDriver.js +0 -495
- package/engines/CryptoEngine.js +0 -75
- package/engines/Environment.js +0 -170
- package/engines/ProcessENVManager.js +0 -83
- package/engines/RandomEngine.js +0 -47
- package/engines/SecretManager.js +0 -23
- package/engines/UserManager.js +0 -66
- package/engines/ai/AutoMapperEngine.js +0 -37
- package/engines/ai/DeveloperEngine.js +0 -497
- package/engines/ai/LLM.js +0 -255
- package/engines/consumer/ConsumerManager.js +0 -218
- package/engines/consumer/ConsumerOnFinishManager.js +0 -202
- package/engines/dataset/Dataset.js +0 -824
- package/engines/dataset/DatasetManager.js +0 -211
- package/engines/dataset/DatasetRecord.js +0 -120
- package/engines/dataset/DatasetRecordPool.js +0 -77
- package/engines/execution/RequestExecutor.js +0 -67
- package/engines/parsing/CSVParser.js +0 -60
- package/engines/parsing/LineParser.js +0 -71
- package/engines/parsing/ParseCompression.js +0 -101
- package/engines/parsing/ParseHelper.js +0 -18
- package/engines/parsing/ParseManager.js +0 -54
- package/engines/parsing/XLSParser.js +0 -87
- package/engines/parsing/XMLParser.js +0 -115
- package/engines/producer/ProducerEngine.js +0 -127
- package/engines/producer/ProducerManager.js +0 -43
- package/engines/scheduler/CronScheduler.js +0 -222
- package/engines/scheduler/QueueManager.js +0 -314
- package/engines/schema/SchemaValidator.js +0 -67
- package/engines/transform/JoinEngine.js +0 -232
- package/engines/transform/TransformationEngine.js +0 -277
- package/engines/transform/TypeCaster.js +0 -59
- package/engines/usage/DataframeManager.js +0 -55
- package/engines/usage/UsageDataManager.js +0 -151
- package/engines/usage/UsageManager.js +0 -65
- package/engines/validation/Validator.js +0 -216
- package/executors/ConsumerExecutor.js +0 -280
- package/executors/Executor.js +0 -177
- package/executors/ExecutorOrchestrator.js +0 -331
- package/executors/ExecutorPerformance.js +0 -17
- package/executors/ExecutorProgress.js +0 -54
- package/executors/ExecutorScope.js +0 -52
- package/executors/OutputExecutor.js +0 -118
- package/executors/ProducerExecutor.js +0 -108
- package/helper/Helper.js +0 -149
- package/helper/Logger.js +0 -84
- package/helper/Runtime.js +0 -20
- package/helper/Settings.js +0 -13
- package/licencing/LicenceManager.js +0 -64
- package/settings.js +0 -12
package/drivers/HttpApiDriver.js
DELETED
|
@@ -1,208 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.HttpApiSourceDriver = void 0;
|
|
16
|
-
const Affirm_1 = __importDefault(require("../core/Affirm"));
|
|
17
|
-
const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
|
|
18
|
-
const Algo_1 = __importDefault(require("../core/Algo"));
|
|
19
|
-
const Logger_1 = __importDefault(require("../helper/Logger"));
|
|
20
|
-
const DriverHelper_1 = __importDefault(require("./DriverHelper"));
|
|
21
|
-
class HttpApiSourceDriver {
|
|
22
|
-
constructor() {
|
|
23
|
-
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
|
|
24
|
-
(0, Affirm_1.default)(source, 'Invalid source');
|
|
25
|
-
(0, Affirm_1.default)(source.authentication, 'Invalid authentication');
|
|
26
|
-
(0, Affirm_1.default)(source.authentication.url, 'HTTP API source requires a URL in authentication.url');
|
|
27
|
-
this._source = source;
|
|
28
|
-
this._baseUrl = SecretManager_1.default.replaceSecret(source.authentication.url);
|
|
29
|
-
this._httpMethod = source.authentication.httpMethod || 'GET';
|
|
30
|
-
this._timeout = source.authentication.timeout || 30000; // 30 seconds default
|
|
31
|
-
this._headers = source.authentication.headers ? Object.assign({}, source.authentication.headers) : {};
|
|
32
|
-
this._queryParams = source.authentication.queryParams ? Object.assign({}, source.authentication.queryParams) : {};
|
|
33
|
-
// Handle different authentication methods
|
|
34
|
-
switch (source.authentication.method) {
|
|
35
|
-
case 'bearer-token': {
|
|
36
|
-
(0, Affirm_1.default)(source.authentication.bearerToken, 'Bearer token authentication requires bearerToken');
|
|
37
|
-
this._headers['Authorization'] = `Bearer ${SecretManager_1.default.replaceSecret(source.authentication.bearerToken)}`;
|
|
38
|
-
break;
|
|
39
|
-
}
|
|
40
|
-
case 'api-key': {
|
|
41
|
-
(0, Affirm_1.default)(source.authentication.apiKey, 'API key authentication requires apiKey');
|
|
42
|
-
const apiKeyHeader = source.authentication.apiKeyHeader || 'X-API-Key';
|
|
43
|
-
this._headers[apiKeyHeader] = SecretManager_1.default.replaceSecret(source.authentication.apiKey);
|
|
44
|
-
break;
|
|
45
|
-
}
|
|
46
|
-
case 'username-password': {
|
|
47
|
-
(0, Affirm_1.default)(source.authentication.user && source.authentication.password, 'Username-password authentication requires user and password');
|
|
48
|
-
const credentials = Buffer.from(`${SecretManager_1.default.replaceSecret(source.authentication.user)}:${SecretManager_1.default.replaceSecret(source.authentication.password)}`).toString('base64');
|
|
49
|
-
this._headers['Authorization'] = `Basic ${credentials}`;
|
|
50
|
-
break;
|
|
51
|
-
}
|
|
52
|
-
case 'none':
|
|
53
|
-
// No authentication required
|
|
54
|
-
break;
|
|
55
|
-
default:
|
|
56
|
-
throw new Error(`Authentication method "${source.authentication.method}" is not supported for HTTP API sources`);
|
|
57
|
-
}
|
|
58
|
-
// Test connection
|
|
59
|
-
try {
|
|
60
|
-
yield this._makeRequest(this._baseUrl);
|
|
61
|
-
Logger_1.default.log(`HTTP API connection to ${this._baseUrl} successful`);
|
|
62
|
-
}
|
|
63
|
-
catch (error) {
|
|
64
|
-
throw new Error(`Failed to connect to HTTP API at ${this._baseUrl}: ${error.message}`);
|
|
65
|
-
}
|
|
66
|
-
return this;
|
|
67
|
-
});
|
|
68
|
-
this._makeRequest = (url, options) => __awaiter(this, void 0, void 0, function* () {
|
|
69
|
-
const method = (options === null || options === void 0 ? void 0 : options.method) || this._httpMethod;
|
|
70
|
-
const headers = Object.assign(Object.assign({}, this._headers), options === null || options === void 0 ? void 0 : options.additionalHeaders);
|
|
71
|
-
const queryParams = Object.assign(Object.assign({}, this._queryParams), options === null || options === void 0 ? void 0 : options.additionalQueryParams);
|
|
72
|
-
// Build URL with query parameters
|
|
73
|
-
const urlWithParams = new URL(url);
|
|
74
|
-
Object.entries(queryParams).forEach(([key, value]) => {
|
|
75
|
-
urlWithParams.searchParams.append(key, value);
|
|
76
|
-
});
|
|
77
|
-
const fetchOptions = {
|
|
78
|
-
method,
|
|
79
|
-
headers,
|
|
80
|
-
signal: AbortSignal.timeout(this._timeout)
|
|
81
|
-
};
|
|
82
|
-
if ((options === null || options === void 0 ? void 0 : options.body) && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
|
|
83
|
-
fetchOptions.body = typeof options.body === 'string'
|
|
84
|
-
? options.body
|
|
85
|
-
: JSON.stringify(options.body);
|
|
86
|
-
if (!headers['Content-Type']) {
|
|
87
|
-
headers['Content-Type'] = 'application/json';
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
const response = yield fetch(urlWithParams.toString(), fetchOptions);
|
|
91
|
-
if (!response.ok) {
|
|
92
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
93
|
-
}
|
|
94
|
-
const contentType = response.headers.get('content-type');
|
|
95
|
-
if (contentType === null || contentType === void 0 ? void 0 : contentType.includes('application/json')) {
|
|
96
|
-
return yield response.json();
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
return yield response.text();
|
|
100
|
-
}
|
|
101
|
-
});
|
|
102
|
-
this.execute = (_sql) => __awaiter(this, void 0, void 0, function* () {
|
|
103
|
-
void _sql;
|
|
104
|
-
throw new Error('SQL execution is not supported for HTTP API sources. Use query() or readAll() instead.');
|
|
105
|
-
});
|
|
106
|
-
this.query = (_sql, _values) => __awaiter(this, void 0, void 0, function* () {
|
|
107
|
-
void _sql;
|
|
108
|
-
void _values;
|
|
109
|
-
throw new Error('SQL queries are not supported for HTTP API sources. Use readAll() to fetch data from the API.');
|
|
110
|
-
});
|
|
111
|
-
this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
|
|
112
|
-
try {
|
|
113
|
-
const endpoint = producer.settings.fileKey || '';
|
|
114
|
-
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
|
|
115
|
-
yield this._makeRequest(url, { method: 'HEAD' });
|
|
116
|
-
return true;
|
|
117
|
-
}
|
|
118
|
-
catch (error) {
|
|
119
|
-
if (error.message.includes('404')) {
|
|
120
|
-
return false;
|
|
121
|
-
}
|
|
122
|
-
throw error;
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
this.readAll = (request, values) => __awaiter(this, void 0, void 0, function* () {
|
|
126
|
-
(0, Affirm_1.default)(request, 'Invalid read request');
|
|
127
|
-
(0, Affirm_1.default)(request.fileKey, 'Invalid file key (endpoint path)');
|
|
128
|
-
const endpoint = request.fileKey;
|
|
129
|
-
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
|
|
130
|
-
// Convert IQueryParameter[] to query params if provided
|
|
131
|
-
const additionalQueryParams = {};
|
|
132
|
-
if (values && values.length > 0) {
|
|
133
|
-
values.forEach(param => {
|
|
134
|
-
additionalQueryParams[param.name] = param.value;
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
const data = yield this._makeRequest(url, { additionalQueryParams });
|
|
138
|
-
// Convert response to string array (lines)
|
|
139
|
-
return this._extractObjectsFromResponse(data, request.httpApi).map(x => JSON.stringify(x));
|
|
140
|
-
});
|
|
141
|
-
this.readLinesInRange = (request) => __awaiter(this, void 0, void 0, function* () {
|
|
142
|
-
(0, Affirm_1.default)(request, 'Invalid read request');
|
|
143
|
-
(0, Affirm_1.default)(request.options, 'Invalid read request options');
|
|
144
|
-
const allLines = yield this.readAll(request);
|
|
145
|
-
const { lineFrom, lineTo } = request.options;
|
|
146
|
-
if (Algo_1.default.hasVal(lineFrom) && Algo_1.default.hasVal(lineTo)) {
|
|
147
|
-
return allLines.slice(lineFrom, lineTo);
|
|
148
|
-
}
|
|
149
|
-
return allLines;
|
|
150
|
-
});
|
|
151
|
-
this.download = (dataset) => __awaiter(this, void 0, void 0, function* () {
|
|
152
|
-
(0, Affirm_1.default)(dataset, 'Invalid dataset');
|
|
153
|
-
const file = dataset.getFile();
|
|
154
|
-
(0, Affirm_1.default)(file, 'Invalid dataset file');
|
|
155
|
-
(0, Affirm_1.default)(file.fileKey, 'Invalid file key (endpoint path)');
|
|
156
|
-
const endpoint = file.fileKey;
|
|
157
|
-
const url = endpoint.startsWith('http') ? endpoint : `${this._baseUrl}${endpoint}`;
|
|
158
|
-
const data = yield this._makeRequest(url);
|
|
159
|
-
const apiObjects = this._extractObjectsFromResponse(data, file.httpApi);
|
|
160
|
-
dataset.setFirstLine(JSON.stringify(apiObjects[0]));
|
|
161
|
-
const totalLineCount = yield DriverHelper_1.default.appendObjectsToUnifiedFile({
|
|
162
|
-
append: true,
|
|
163
|
-
delimiter: dataset.getDelimiter(),
|
|
164
|
-
destinationPath: dataset.getPath(),
|
|
165
|
-
objects: apiObjects
|
|
166
|
-
});
|
|
167
|
-
dataset.setCount(totalLineCount);
|
|
168
|
-
return dataset;
|
|
169
|
-
});
|
|
170
|
-
this._extractObjectsFromResponse = (data, httpApi) => {
|
|
171
|
-
let itemsData = [];
|
|
172
|
-
if (httpApi && httpApi.dataProperty && httpApi.dataProperty.length > 0) {
|
|
173
|
-
itemsData = data[httpApi.dataProperty];
|
|
174
|
-
}
|
|
175
|
-
else {
|
|
176
|
-
if (typeof data === 'string') {
|
|
177
|
-
itemsData = data.split('\n').filter(line => line.trim().length > 0);
|
|
178
|
-
}
|
|
179
|
-
else if (Array.isArray(data)) {
|
|
180
|
-
itemsData = data;
|
|
181
|
-
}
|
|
182
|
-
else if (typeof data === 'object' && data !== null) {
|
|
183
|
-
const dataObj = data;
|
|
184
|
-
if (dataObj.data && Array.isArray(dataObj.data)) {
|
|
185
|
-
itemsData = dataObj.data;
|
|
186
|
-
}
|
|
187
|
-
else if (dataObj.results && Array.isArray(dataObj.results)) {
|
|
188
|
-
itemsData = dataObj.results;
|
|
189
|
-
}
|
|
190
|
-
else if (dataObj.items && Array.isArray(dataObj.items)) {
|
|
191
|
-
itemsData = dataObj.items;
|
|
192
|
-
}
|
|
193
|
-
else {
|
|
194
|
-
// Single object, return as single line
|
|
195
|
-
itemsData = [data];
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
return itemsData;
|
|
200
|
-
};
|
|
201
|
-
this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
|
|
202
|
-
void request;
|
|
203
|
-
throw new Error('Not implemented yet');
|
|
204
|
-
});
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
exports.HttpApiSourceDriver = HttpApiSourceDriver;
|
|
208
|
-
exports.default = HttpApiSourceDriver;
|
|
@@ -1,184 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
const client_redshift_data_1 = require("@aws-sdk/client-redshift-data");
|
|
16
|
-
const Affirm_1 = __importDefault(require("../core/Affirm"));
|
|
17
|
-
const Algo_1 = __importDefault(require("../core/Algo"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
|
|
20
|
-
class RedshiftDriver {
|
|
21
|
-
constructor() {
|
|
22
|
-
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
|
|
23
|
-
(0, Affirm_1.default)(source, `Invalid source`);
|
|
24
|
-
this._SQL_MAX_QUERY_ROWS = parseInt(Environment_1.default.get('SQL_MAX_QUERY_ROWS'));
|
|
25
|
-
this._dbName = source.authentication['database'];
|
|
26
|
-
this._workgroup = source.authentication['workgroup'];
|
|
27
|
-
const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
|
|
28
|
-
const config = {
|
|
29
|
-
region: source.authentication['region'],
|
|
30
|
-
credentials: {
|
|
31
|
-
accessKeyId: SecretManager_1.default.replaceSecret(source.authentication['accessKey']),
|
|
32
|
-
secretAccessKey: SecretManager_1.default.replaceSecret(source.authentication['secretKey']),
|
|
33
|
-
sessionToken: sessionToken ? sessionToken : undefined
|
|
34
|
-
}
|
|
35
|
-
};
|
|
36
|
-
try {
|
|
37
|
-
this._client = new client_redshift_data_1.RedshiftData(config);
|
|
38
|
-
// This is just to check that the connection was successful and warm up the connection
|
|
39
|
-
yield this.query('SELECT :diss AS number', [{ name: ':diss', value: '1' }]);
|
|
40
|
-
}
|
|
41
|
-
catch (error) {
|
|
42
|
-
const myError = error;
|
|
43
|
-
if (myError.__type === 'UnrecognizedClientException')
|
|
44
|
-
throw new Error(`Invalid AWS credentials for source "${source.name}": ${myError.message}`);
|
|
45
|
-
else
|
|
46
|
-
throw error;
|
|
47
|
-
}
|
|
48
|
-
return this;
|
|
49
|
-
});
|
|
50
|
-
this.execute = (sql) => __awaiter(this, void 0, void 0, function* () {
|
|
51
|
-
var _a;
|
|
52
|
-
(0, Affirm_1.default)(sql, `Invalid SQL`);
|
|
53
|
-
const input = {
|
|
54
|
-
Sql: sql,
|
|
55
|
-
Database: this._dbName,
|
|
56
|
-
WorkgroupName: this._workgroup
|
|
57
|
-
};
|
|
58
|
-
const executeStatementRes = yield this._client.executeStatement(input);
|
|
59
|
-
const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
|
|
60
|
-
const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
|
|
61
|
-
return {
|
|
62
|
-
rows: res.result
|
|
63
|
-
};
|
|
64
|
-
});
|
|
65
|
-
this.query = (sql, values) => __awaiter(this, void 0, void 0, function* () {
|
|
66
|
-
var _a, _b;
|
|
67
|
-
(0, Affirm_1.default)(sql, `Invalid SQL`);
|
|
68
|
-
const wrapperSql = `SELECT * FROM (${sql}) LIMIT ${this._SQL_MAX_QUERY_ROWS}`;
|
|
69
|
-
const input = {
|
|
70
|
-
Sql: wrapperSql,
|
|
71
|
-
Database: this._dbName,
|
|
72
|
-
WorkgroupName: this._workgroup,
|
|
73
|
-
Parameters: (_a = values === null || values === void 0 ? void 0 : values.map(x => ({ name: Algo_1.default.replaceAll(x.name, ':', ''), value: x.value }))) !== null && _a !== void 0 ? _a : undefined
|
|
74
|
-
};
|
|
75
|
-
const executeStatementRes = yield this._client.executeStatement(input);
|
|
76
|
-
const queryId = (_b = executeStatementRes.Id) !== null && _b !== void 0 ? _b : '';
|
|
77
|
-
// TODO: handle errors (permission, wrong query, ...)
|
|
78
|
-
const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
|
|
79
|
-
return {
|
|
80
|
-
rows: res.result
|
|
81
|
-
};
|
|
82
|
-
});
|
|
83
|
-
this.exist = (producer) => __awaiter(this, void 0, void 0, function* () {
|
|
84
|
-
var _a;
|
|
85
|
-
(0, Affirm_1.default)(producer, `Invalid producer`);
|
|
86
|
-
const sql = `SELECT * FROM "${producer.settings.sqlTable}" LIMIT 1`;
|
|
87
|
-
const input = {
|
|
88
|
-
Sql: sql,
|
|
89
|
-
Database: this._dbName,
|
|
90
|
-
WorkgroupName: this._workgroup
|
|
91
|
-
};
|
|
92
|
-
const executeStatementRes = yield this._client.executeStatement(input);
|
|
93
|
-
const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
|
|
94
|
-
try {
|
|
95
|
-
yield this.executeStatement(queryId, { retry: 5 });
|
|
96
|
-
return true;
|
|
97
|
-
}
|
|
98
|
-
catch (e) {
|
|
99
|
-
console.error(e);
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
this.executeStatement = (queryId, options) => __awaiter(this, void 0, void 0, function* () {
|
|
104
|
-
const MAX_WAIT_CYCLES = 20;
|
|
105
|
-
for (let i = 0; i < MAX_WAIT_CYCLES; i++) {
|
|
106
|
-
const statement = yield this._client.describeStatement({ Id: queryId });
|
|
107
|
-
(0, Affirm_1.default)(statement, 'SQL query failed: describe statement call failed.');
|
|
108
|
-
(0, Affirm_1.default)(statement.Status !== 'FAILED', `SQL query failed: ${queryId}:\n\tSQL: ${statement.QueryString}\n\tError: ${statement.Error}.`);
|
|
109
|
-
const { Status: status, SubStatements: subStatements } = statement;
|
|
110
|
-
if (status === 'FINISHED') {
|
|
111
|
-
if (subStatements && subStatements.length > 0) {
|
|
112
|
-
const result = [];
|
|
113
|
-
for (let i = 0; i < subStatements.length; i++) {
|
|
114
|
-
const subStatement = subStatements[i];
|
|
115
|
-
result.push({
|
|
116
|
-
statement: { hasResultSet: statement.HasResultSet, queryString: statement.QueryString, resultRows: statement.ResultRows, resultSize: statement.ResultSize },
|
|
117
|
-
subStatement: { hasResultSet: subStatement.HasResultSet, queryString: subStatement.QueryString, resultRows: subStatement.ResultRows, resultSize: subStatement.ResultSize },
|
|
118
|
-
result: yield this.getStatementResult(subStatement)
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
return result;
|
|
122
|
-
}
|
|
123
|
-
else {
|
|
124
|
-
return [{
|
|
125
|
-
statement: { hasResultSet: statement.HasResultSet, queryString: statement.QueryString, resultRows: statement.ResultRows, resultSize: statement.ResultSize },
|
|
126
|
-
result: yield this.getStatementResult(statement)
|
|
127
|
-
}];
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
yield Algo_1.default.sleep(1);
|
|
131
|
-
}
|
|
132
|
-
if (options && options.retry > 0)
|
|
133
|
-
return yield this.executeStatement(queryId, Object.assign(Object.assign({}, options), { retry: options.retry - 1 }));
|
|
134
|
-
else
|
|
135
|
-
throw new Error('SQL timeout: query failed because timeout exceeded max wait cycles');
|
|
136
|
-
});
|
|
137
|
-
this.getStatementResult = (statement) => __awaiter(this, void 0, void 0, function* () {
|
|
138
|
-
(0, Affirm_1.default)(statement, `SQL failed: can't extract result from invalid statement`);
|
|
139
|
-
if (statement.HasResultSet) {
|
|
140
|
-
const result = yield this._client.getStatementResult({ Id: statement.Id });
|
|
141
|
-
(0, Affirm_1.default)(result, 'SQL failed: invalid result statement');
|
|
142
|
-
(0, Affirm_1.default)(result.Records, 'SQL failed: no records in result statement');
|
|
143
|
-
return this.parseQueryResult(result);
|
|
144
|
-
}
|
|
145
|
-
else {
|
|
146
|
-
return [];
|
|
147
|
-
}
|
|
148
|
-
});
|
|
149
|
-
this.parseQueryResult = (queryResult) => {
|
|
150
|
-
(0, Affirm_1.default)(queryResult, `Parse SQL failed: invalid query result`);
|
|
151
|
-
(0, Affirm_1.default)(queryResult.ColumnMetadata, `Parse SQL failed: invalid column metadata in query result`);
|
|
152
|
-
(0, Affirm_1.default)(queryResult.Records, `Parse SQL failed: invalid records in query result`);
|
|
153
|
-
const columnNames = queryResult.ColumnMetadata.map(column => column.name);
|
|
154
|
-
const records = [];
|
|
155
|
-
for (const record of queryResult.Records) {
|
|
156
|
-
const recordData = {};
|
|
157
|
-
for (let i = 0; i < columnNames.length; i++) {
|
|
158
|
-
const columnName = columnNames[i];
|
|
159
|
-
(0, Affirm_1.default)(columnName, `Parse SQL failed: invalid column name`);
|
|
160
|
-
const value = record[i];
|
|
161
|
-
if (value.longValue !== undefined)
|
|
162
|
-
recordData[columnName] = value.longValue;
|
|
163
|
-
else if (value.stringValue !== undefined)
|
|
164
|
-
recordData[columnName] = value.stringValue;
|
|
165
|
-
else if (value.booleanValue !== undefined)
|
|
166
|
-
recordData[columnName] = value.booleanValue;
|
|
167
|
-
else if (value.doubleValue !== undefined)
|
|
168
|
-
recordData[columnName] = value.doubleValue;
|
|
169
|
-
else if (value.isNull === true)
|
|
170
|
-
recordData[columnName] = null;
|
|
171
|
-
else
|
|
172
|
-
recordData[columnName] = null;
|
|
173
|
-
}
|
|
174
|
-
records.push(recordData);
|
|
175
|
-
}
|
|
176
|
-
return records;
|
|
177
|
-
};
|
|
178
|
-
this.ready = (request) => __awaiter(this, void 0, void 0, function* () {
|
|
179
|
-
void request;
|
|
180
|
-
throw new Error('Not implemented yet');
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
exports.default = RedshiftDriver;
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
-
});
|
|
43
|
-
};
|
|
44
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
45
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
46
|
-
var m = o[Symbol.asyncIterator], i;
|
|
47
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
48
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
49
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
50
|
-
};
|
|
51
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
-
};
|
|
54
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
-
const fs = __importStar(require("fs"));
|
|
56
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
57
|
-
const readline_1 = __importDefault(require("readline"));
|
|
58
|
-
const path_1 = __importDefault(require("path"));
|
|
59
|
-
const Affirm_1 = __importDefault(require("../../core/Affirm"));
|
|
60
|
-
const Logger_1 = __importDefault(require("../../helper/Logger"));
|
|
61
|
-
class LocalDestinationDriver {
|
|
62
|
-
constructor() {
|
|
63
|
-
this.init = (source) => __awaiter(this, void 0, void 0, function* () {
|
|
64
|
-
(0, Affirm_1.default)(source, `Invalid source`);
|
|
65
|
-
const fileURL = source.authentication['path'];
|
|
66
|
-
(0, Affirm_1.default)(fileURL, `Missing file path in the authentication of source "${source.name}"`);
|
|
67
|
-
const exist = fs.existsSync(fileURL);
|
|
68
|
-
(0, Affirm_1.default)(exist, `The path (${fileURL}) for source "${source.name}" does NOT exist.`);
|
|
69
|
-
this._path = source.authentication['path'];
|
|
70
|
-
return this;
|
|
71
|
-
});
|
|
72
|
-
this.saveFile = (fileKey, content) => {
|
|
73
|
-
(0, Affirm_1.default)(this._path, 'Path not initialized');
|
|
74
|
-
(0, Affirm_1.default)(fileKey, 'Invalid file key');
|
|
75
|
-
(0, Affirm_1.default)(content, 'Invalid content');
|
|
76
|
-
const filePath = path_1.default.join(this._path, fileKey);
|
|
77
|
-
const directory = path_1.default.dirname(filePath);
|
|
78
|
-
// Create directory if it doesn't exist
|
|
79
|
-
if (!fs.existsSync(directory)) {
|
|
80
|
-
fs.mkdirSync(directory, { recursive: true });
|
|
81
|
-
}
|
|
82
|
-
fs.writeFileSync(filePath, content);
|
|
83
|
-
return Promise.resolve();
|
|
84
|
-
};
|
|
85
|
-
this.copyFromLocal = (sourceFilePath, destinationFileKey) => {
|
|
86
|
-
(0, Affirm_1.default)(this._path, 'Path not initialized');
|
|
87
|
-
(0, Affirm_1.default)(sourceFilePath, 'Invalid source file path');
|
|
88
|
-
(0, Affirm_1.default)(destinationFileKey, 'Invalid destination file key');
|
|
89
|
-
const destinationFilePath = path_1.default.join(this._path, destinationFileKey);
|
|
90
|
-
const destinationDir = path_1.default.dirname(destinationFilePath);
|
|
91
|
-
// Ensure destination directory exists
|
|
92
|
-
if (!fs.existsSync(destinationDir)) {
|
|
93
|
-
fs.mkdirSync(destinationDir, { recursive: true });
|
|
94
|
-
}
|
|
95
|
-
fs.copyFileSync(sourceFilePath, destinationFilePath);
|
|
96
|
-
};
|
|
97
|
-
this.copyFromS3 = (s3Driver, sourceFileKey, destinationFileKey) => __awaiter(this, void 0, void 0, function* () {
|
|
98
|
-
const fileContent = yield s3Driver.downloadFile(sourceFileKey);
|
|
99
|
-
yield this.saveFile(destinationFileKey, fileContent);
|
|
100
|
-
});
|
|
101
|
-
this.move = (fromPath, toName) => __awaiter(this, void 0, void 0, function* () {
|
|
102
|
-
try {
|
|
103
|
-
const toFilePath = path_1.default.join(this._path, toName);
|
|
104
|
-
yield promises_1.default.copyFile(fromPath, toFilePath);
|
|
105
|
-
yield promises_1.default.unlink(fromPath);
|
|
106
|
-
return { bucket: '', key: toFilePath, res: true };
|
|
107
|
-
}
|
|
108
|
-
catch (error) {
|
|
109
|
-
Logger_1.default.error(error);
|
|
110
|
-
return { bucket: '', key: '', res: false };
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
|
-
this.transformAndMove = (fromPath, transform, toName) => __awaiter(this, void 0, void 0, function* () {
|
|
114
|
-
var _a, e_1, _b, _c;
|
|
115
|
-
try {
|
|
116
|
-
const toFilePath = path_1.default.join(this._path, toName);
|
|
117
|
-
const reader = fs.createReadStream(fromPath);
|
|
118
|
-
const lineReader = readline_1.default.createInterface({ input: reader, crlfDelay: Infinity });
|
|
119
|
-
const writer = fs.createWriteStream(toFilePath);
|
|
120
|
-
try {
|
|
121
|
-
for (var _d = true, lineReader_1 = __asyncValues(lineReader), lineReader_1_1; lineReader_1_1 = yield lineReader_1.next(), _a = lineReader_1_1.done, !_a; _d = true) {
|
|
122
|
-
_c = lineReader_1_1.value;
|
|
123
|
-
_d = false;
|
|
124
|
-
const line = _c;
|
|
125
|
-
writer.write(transform(line) + '\n');
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
129
|
-
finally {
|
|
130
|
-
try {
|
|
131
|
-
if (!_d && !_a && (_b = lineReader_1.return)) yield _b.call(lineReader_1);
|
|
132
|
-
}
|
|
133
|
-
finally { if (e_1) throw e_1.error; }
|
|
134
|
-
}
|
|
135
|
-
writer.close();
|
|
136
|
-
reader.close();
|
|
137
|
-
return { bucket: '', key: toFilePath, res: true };
|
|
138
|
-
}
|
|
139
|
-
catch (error) {
|
|
140
|
-
Logger_1.default.error(error);
|
|
141
|
-
return { bucket: '', key: '', res: false };
|
|
142
|
-
}
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
exports.default = LocalDestinationDriver;
|