@owox/connectors 0.12.0-next-20251105102829 → 0.12.0-next-20251105123137
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connector-runner.cjs +1 -5
- package/dist/connector-runner.js +1 -5
- package/dist/index.cjs +800 -1763
- package/dist/index.js +800 -1763
- package/package.json +1 -3
package/dist/index.js
CHANGED
|
@@ -25,11 +25,6 @@ var require_index = __commonJS({
|
|
|
25
25
|
SERVER_ERROR_MIN: 500,
|
|
26
26
|
SERVER_ERROR_MAX: 599
|
|
27
27
|
};
|
|
28
|
-
var ENVIRONMENT = {
|
|
29
|
-
UNKNOWN: 1,
|
|
30
|
-
APPS_SCRIPT: 2,
|
|
31
|
-
NODE: 3
|
|
32
|
-
};
|
|
33
28
|
var EXECUTION_STATUS = {
|
|
34
29
|
IMPORT_IN_PROGRESS: 1,
|
|
35
30
|
CLEANUP_IN_PROGRESS: 2,
|
|
@@ -77,269 +72,11 @@ var require_index = __commonJS({
|
|
|
77
72
|
}
|
|
78
73
|
class UnsupportedEnvironmentException extends AbstractException {
|
|
79
74
|
}
|
|
80
|
-
var EnvironmentAdapter = class EnvironmentAdapter {
|
|
81
|
-
/**
|
|
82
|
-
* Mac algorithm constants.
|
|
83
|
-
*
|
|
84
|
-
* @type {Object}
|
|
85
|
-
*/
|
|
86
|
-
static get MacAlgorithm() {
|
|
87
|
-
return {
|
|
88
|
-
HMAC_SHA_256: "HMAC_SHA_256",
|
|
89
|
-
HMAC_SHA_384: "HMAC_SHA_384",
|
|
90
|
-
HMAC_SHA_512: "HMAC_SHA_512",
|
|
91
|
-
HMAC_SHA_1: "HMAC_SHA_1",
|
|
92
|
-
HMAC_MD5: "HMAC_MD5"
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
constructor() {
|
|
96
|
-
this.environment = this.getEnvironment();
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Get the current environment.
|
|
100
|
-
* Detects whether code is running in Google Apps Script or Node.js environment.
|
|
101
|
-
*
|
|
102
|
-
* @returns {ENVIRONMENT} The detected environment (APPS_SCRIPT, NODE, or UNKNOWN)
|
|
103
|
-
* @throws {UnsupportedEnvironmentException} If environment cannot be determined
|
|
104
|
-
*/
|
|
105
|
-
static getEnvironment() {
|
|
106
|
-
if (typeof this.environment !== "undefined") {
|
|
107
|
-
return this.environment;
|
|
108
|
-
}
|
|
109
|
-
if (typeof UrlFetchApp !== "undefined") {
|
|
110
|
-
this.environment = ENVIRONMENT.APPS_SCRIPT;
|
|
111
|
-
} else if (typeof process !== "undefined") {
|
|
112
|
-
this.environment = ENVIRONMENT.NODE;
|
|
113
|
-
} else {
|
|
114
|
-
this.environment = ENVIRONMENT.UNKNOWN;
|
|
115
|
-
}
|
|
116
|
-
return this.environment;
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Fetch data from the given URL.
|
|
120
|
-
*
|
|
121
|
-
* @param {string} url - The URL to fetch data from.
|
|
122
|
-
* @param {Object} options - Options for the fetch request.
|
|
123
|
-
* @returns {FetchResponse}
|
|
124
|
-
*
|
|
125
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
126
|
-
*/
|
|
127
|
-
static fetch(url, options = {}) {
|
|
128
|
-
const env = this.getEnvironment();
|
|
129
|
-
if (env === ENVIRONMENT.APPS_SCRIPT) {
|
|
130
|
-
const response = UrlFetchApp.fetch(url, options);
|
|
131
|
-
return this._wrapAppsScriptResponse(response);
|
|
132
|
-
}
|
|
133
|
-
if (env === ENVIRONMENT.NODE) {
|
|
134
|
-
const method = options.method || "GET";
|
|
135
|
-
const response = request(method, url, options);
|
|
136
|
-
return this._wrapNodeResponse(response);
|
|
137
|
-
}
|
|
138
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* Sleep for the given number of milliseconds.
|
|
142
|
-
*
|
|
143
|
-
* @param {number} ms - The number of milliseconds to sleep.
|
|
144
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
145
|
-
*/
|
|
146
|
-
static sleep(ms) {
|
|
147
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
148
|
-
Utilities.sleep(ms);
|
|
149
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
150
|
-
let done = false;
|
|
151
|
-
new Promise((resolve) => {
|
|
152
|
-
setTimeout(() => {
|
|
153
|
-
done = true;
|
|
154
|
-
resolve();
|
|
155
|
-
}, ms);
|
|
156
|
-
});
|
|
157
|
-
deasync.loopWhile(() => !done);
|
|
158
|
-
} else {
|
|
159
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Format the given date.
|
|
164
|
-
*
|
|
165
|
-
* @param {Date} date - The date to format.
|
|
166
|
-
* @param {string} timezone - The timezone to format the date in.
|
|
167
|
-
* @param {string} format - The format to format the date in.
|
|
168
|
-
* @returns {string}
|
|
169
|
-
*
|
|
170
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
171
|
-
*/
|
|
172
|
-
static formatDate(date, timezone, format) {
|
|
173
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
174
|
-
return Utilities.formatDate(date, timezone, format);
|
|
175
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
176
|
-
return date.toISOString().split("T")[0];
|
|
177
|
-
} else {
|
|
178
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
|
|
183
|
-
*
|
|
184
|
-
* @returns {string} UUID
|
|
185
|
-
*
|
|
186
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
187
|
-
*/
|
|
188
|
-
static getUuid() {
|
|
189
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
190
|
-
return Utilities.getUuid();
|
|
191
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
192
|
-
const crypto = require("node:crypto");
|
|
193
|
-
return crypto.randomUUID();
|
|
194
|
-
} else {
|
|
195
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
/**
|
|
199
|
-
* Encode the given data to base64.
|
|
200
|
-
*
|
|
201
|
-
* @param {string} data - The data to encode.
|
|
202
|
-
* @returns {string}
|
|
203
|
-
*
|
|
204
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
205
|
-
*/
|
|
206
|
-
static base64Encode(data) {
|
|
207
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
208
|
-
return Utilities.base64Encode(data);
|
|
209
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
210
|
-
return Buffer.from(data).toString("base64");
|
|
211
|
-
} else {
|
|
212
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
/**
|
|
216
|
-
* Compute the HMAC signature for the given data.
|
|
217
|
-
*
|
|
218
|
-
* @param {string} algorithm - The algorithm to use.
|
|
219
|
-
* @param {string} data - The data to compute the signature for.
|
|
220
|
-
* @param {string} key - The key to use.
|
|
221
|
-
* @returns {string}
|
|
222
|
-
*
|
|
223
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported.
|
|
224
|
-
*/
|
|
225
|
-
static computeHmacSignature(algorithm, data, key) {
|
|
226
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
227
|
-
if (typeof algorithm === "string") {
|
|
228
|
-
algorithm = Utilities.MacAlgorithm[algorithm];
|
|
229
|
-
}
|
|
230
|
-
return Utilities.computeHmacSignature(algorithm, data, key);
|
|
231
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
232
|
-
const crypto = require("node:crypto");
|
|
233
|
-
const algorithmMap = {
|
|
234
|
-
"HMAC_SHA_256": "sha256",
|
|
235
|
-
"HMAC_SHA_384": "sha384",
|
|
236
|
-
"HMAC_SHA_512": "sha512",
|
|
237
|
-
"HMAC_SHA_1": "sha1",
|
|
238
|
-
"HMAC_MD5": "md5"
|
|
239
|
-
};
|
|
240
|
-
const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
|
|
241
|
-
const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
|
|
242
|
-
return Array.from(buffer);
|
|
243
|
-
} else {
|
|
244
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
/**
|
|
248
|
-
* Parse CSV string into array of arrays
|
|
249
|
-
*
|
|
250
|
-
* @param {string} csvString - The CSV string to parse
|
|
251
|
-
* @param {string} [delimiter=','] - The delimiter to use for parsing CSV
|
|
252
|
-
* @returns {Array<Array<string>>} Parsed CSV data
|
|
253
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported
|
|
254
|
-
*/
|
|
255
|
-
static parseCsv(csvString, delimiter = ",") {
|
|
256
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
257
|
-
return Utilities.parseCsv(csvString, delimiter);
|
|
258
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
259
|
-
return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
|
|
260
|
-
const trimmed = cell.trim();
|
|
261
|
-
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
262
|
-
return trimmed.slice(1, -1).replace(/""/g, '"');
|
|
263
|
-
}
|
|
264
|
-
return trimmed;
|
|
265
|
-
}));
|
|
266
|
-
} else {
|
|
267
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
/**
|
|
271
|
-
* Unzip a blob/buffer
|
|
272
|
-
*
|
|
273
|
-
* @param {Blob|Buffer} data - The data to unzip
|
|
274
|
-
* @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
|
|
275
|
-
* @throws {UnsupportedEnvironmentException} If the environment is not supported
|
|
276
|
-
*/
|
|
277
|
-
static unzip(data) {
|
|
278
|
-
if (this.getEnvironment() === ENVIRONMENT.APPS_SCRIPT) {
|
|
279
|
-
return Utilities.unzip(data);
|
|
280
|
-
} else if (this.getEnvironment() === ENVIRONMENT.NODE) {
|
|
281
|
-
const zip = new AdmZip(data);
|
|
282
|
-
return zip.getEntries().map((entry) => ({
|
|
283
|
-
getDataAsString: () => entry.getData().toString("utf8")
|
|
284
|
-
}));
|
|
285
|
-
} else {
|
|
286
|
-
throw new UnsupportedEnvironmentException("Unsupported environment");
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
/**
|
|
290
|
-
* Wraps the response from the Apps Script environment.
|
|
291
|
-
* Not use directly, only for internal purposes.
|
|
292
|
-
*
|
|
293
|
-
* @param {Object} response
|
|
294
|
-
* @returns {FetchResponse}
|
|
295
|
-
*/
|
|
296
|
-
static _wrapAppsScriptResponse(response) {
|
|
297
|
-
return {
|
|
298
|
-
getHeaders: () => response.getAllHeaders(),
|
|
299
|
-
getAsJson: () => {
|
|
300
|
-
try {
|
|
301
|
-
return JSON.parse(response.getContentText());
|
|
302
|
-
} catch (e) {
|
|
303
|
-
throw new Error("Invalid JSON response");
|
|
304
|
-
}
|
|
305
|
-
},
|
|
306
|
-
getContent: () => response.getContent(),
|
|
307
|
-
getContentText: () => response.getContentText(),
|
|
308
|
-
getBlob: () => response.getBlob(),
|
|
309
|
-
getResponseCode: () => response.getResponseCode()
|
|
310
|
-
};
|
|
311
|
-
}
|
|
312
|
-
/**
|
|
313
|
-
* Wraps the response from the Node environment.
|
|
314
|
-
* Not use directly, only for internal purposes.
|
|
315
|
-
*
|
|
316
|
-
* @param {Object} response
|
|
317
|
-
* @returns {FetchResponse}
|
|
318
|
-
*/
|
|
319
|
-
static _wrapNodeResponse(response) {
|
|
320
|
-
const headers = response.headers || {};
|
|
321
|
-
const text = response.body ? response.body.toString() : "";
|
|
322
|
-
return {
|
|
323
|
-
getHeaders: () => headers,
|
|
324
|
-
getAsJson: () => {
|
|
325
|
-
try {
|
|
326
|
-
return JSON.parse(text);
|
|
327
|
-
} catch (e) {
|
|
328
|
-
throw new Error("Invalid JSON response");
|
|
329
|
-
}
|
|
330
|
-
},
|
|
331
|
-
getContent: () => text,
|
|
332
|
-
getContentText: () => text,
|
|
333
|
-
getBlob: () => response.body,
|
|
334
|
-
getResponseCode: () => response.statusCode
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
};
|
|
338
75
|
class AbstractStorage {
|
|
339
76
|
//---- constructor -------------------------------------------------
|
|
340
77
|
/**
|
|
341
|
-
*
|
|
342
|
-
* @param config (object) instance of
|
|
78
|
+
* Abstract class for storage operations providing common methods for data persistence
|
|
79
|
+
* @param config (object) instance of AbstractConfig
|
|
343
80
|
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
344
81
|
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
345
82
|
* @param description (string) string with storage description }
|
|
@@ -363,6 +100,14 @@ var require_index = __commonJS({
|
|
|
363
100
|
}
|
|
364
101
|
}
|
|
365
102
|
//----------------------------------------------------------------
|
|
103
|
+
//---- init --------------------------------------------------------
|
|
104
|
+
/**
|
|
105
|
+
* Initializing storage
|
|
106
|
+
*/
|
|
107
|
+
async init() {
|
|
108
|
+
throw new Error("Method init() has to be implemented in a child class of AbstractStorage");
|
|
109
|
+
}
|
|
110
|
+
//----------------------------------------------------------------
|
|
366
111
|
//---- getUniqueKeyByRecordFields ----------------------------------
|
|
367
112
|
/**
|
|
368
113
|
* Calculcating unique key based on this.uniqueKeyColumns
|
|
@@ -419,11 +164,12 @@ var require_index = __commonJS({
|
|
|
419
164
|
//----------------------------------------------------------------
|
|
420
165
|
//---- saveData ----------------------------------------------------
|
|
421
166
|
/**
|
|
422
|
-
* Saving data to a storage. Has to be implemented in
|
|
167
|
+
* Saving data to a storage. Has to be implemented in child class as async method.
|
|
423
168
|
* @param {data} array of assoc objects with records to save
|
|
169
|
+
* @returns {Promise<void>}
|
|
424
170
|
*/
|
|
425
|
-
saveData(data) {
|
|
426
|
-
throw new Error("Method
|
|
171
|
+
async saveData(data) {
|
|
172
|
+
throw new Error("Method saveData() has to be implemented in a child class of AbstractStorage");
|
|
427
173
|
}
|
|
428
174
|
//----------------------------------------------------------------
|
|
429
175
|
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
@@ -494,16 +240,6 @@ var require_index = __commonJS({
|
|
|
494
240
|
return record;
|
|
495
241
|
}
|
|
496
242
|
//----------------------------------------------------------------
|
|
497
|
-
//---- areHeadersNeeded --------------------------------------------
|
|
498
|
-
/**
|
|
499
|
-
* Checks if storage needs headers to be added
|
|
500
|
-
* By default returns false, should be overridden in child classes if needed
|
|
501
|
-
* @returns {boolean} true if headers need to be added, false otherwise
|
|
502
|
-
*/
|
|
503
|
-
areHeadersNeeded() {
|
|
504
|
-
return false;
|
|
505
|
-
}
|
|
506
|
-
//----------------------------------------------------------------
|
|
507
243
|
//---- getSelectedFields -------------------------------------------
|
|
508
244
|
/**
|
|
509
245
|
* Parse Fields config value and return array of selected field names
|
|
@@ -557,7 +293,7 @@ var require_index = __commonJS({
|
|
|
557
293
|
* A Data Source-specific methid is used to fetch new data and return it as an array of objects, where each property of an object corresponds to a column name.
|
|
558
294
|
* @return data array
|
|
559
295
|
*/
|
|
560
|
-
fetchData() {
|
|
296
|
+
async fetchData() {
|
|
561
297
|
throw new Error("Method fetchData must be implemented in Class inheritor of AbstractSource");
|
|
562
298
|
}
|
|
563
299
|
//----------------------------------------------------------------
|
|
@@ -582,16 +318,16 @@ var require_index = __commonJS({
|
|
|
582
318
|
* @return {HTTPResponse} The response object from the fetch
|
|
583
319
|
* @throws {HttpRequestException} After exhausting all retries
|
|
584
320
|
*/
|
|
585
|
-
urlFetchWithRetry(url, options) {
|
|
321
|
+
async urlFetchWithRetry(url, options) {
|
|
586
322
|
for (let attempt = 1; attempt <= this.config.MaxFetchRetries.value; attempt++) {
|
|
587
323
|
try {
|
|
588
|
-
const response =
|
|
589
|
-
return this._validateResponse(response);
|
|
324
|
+
const response = await HttpUtils.fetch(url, { ...options, muteHttpExceptions: true });
|
|
325
|
+
return await this._validateResponse(response);
|
|
590
326
|
} catch (error) {
|
|
591
327
|
if (!this._shouldRetry(error, attempt)) {
|
|
592
328
|
throw error;
|
|
593
329
|
}
|
|
594
|
-
this._waitBeforeRetry(attempt);
|
|
330
|
+
await this._waitBeforeRetry(attempt);
|
|
595
331
|
}
|
|
596
332
|
}
|
|
597
333
|
}
|
|
@@ -603,12 +339,12 @@ var require_index = __commonJS({
|
|
|
603
339
|
* @throws {HttpRequestException} If the response indicates an error
|
|
604
340
|
* @private
|
|
605
341
|
*/
|
|
606
|
-
_validateResponse(response) {
|
|
342
|
+
async _validateResponse(response) {
|
|
607
343
|
const code = response.getResponseCode();
|
|
608
344
|
if (code >= HTTP_STATUS.SUCCESS_MIN && code <= HTTP_STATUS.SUCCESS_MAX) {
|
|
609
345
|
return response;
|
|
610
346
|
}
|
|
611
|
-
const errorInfo = this._extractErrorInfo(response);
|
|
347
|
+
const errorInfo = await this._extractErrorInfo(response);
|
|
612
348
|
throw new HttpRequestException({
|
|
613
349
|
message: errorInfo.message,
|
|
614
350
|
statusCode: code,
|
|
@@ -622,9 +358,9 @@ var require_index = __commonJS({
|
|
|
622
358
|
* @return {Object} Object containing error message and JSON data if available
|
|
623
359
|
* @private
|
|
624
360
|
*/
|
|
625
|
-
_extractErrorInfo(response) {
|
|
361
|
+
async _extractErrorInfo(response) {
|
|
626
362
|
var _a, _b;
|
|
627
|
-
const text = response.getContentText();
|
|
363
|
+
const text = await response.getContentText();
|
|
628
364
|
let parsedJson = null;
|
|
629
365
|
let message = text;
|
|
630
366
|
try {
|
|
@@ -661,10 +397,10 @@ var require_index = __commonJS({
|
|
|
661
397
|
* @param {number} attempt - The current attempt number
|
|
662
398
|
* @private
|
|
663
399
|
*/
|
|
664
|
-
_waitBeforeRetry(attempt) {
|
|
400
|
+
async _waitBeforeRetry(attempt) {
|
|
665
401
|
const delay = this.calculateBackoff(attempt);
|
|
666
402
|
console.log(`Retrying after ${Math.round(delay / 1e3)}s...`);
|
|
667
|
-
|
|
403
|
+
await AsyncUtils.delay(delay);
|
|
668
404
|
}
|
|
669
405
|
//---- calculateBackoff --------------------------------------------
|
|
670
406
|
/**
|
|
@@ -796,7 +532,7 @@ var require_index = __commonJS({
|
|
|
796
532
|
/**
|
|
797
533
|
* Initiates imports new data from a data source
|
|
798
534
|
*/
|
|
799
|
-
run() {
|
|
535
|
+
async run() {
|
|
800
536
|
try {
|
|
801
537
|
if (this.config.isInProgress()) {
|
|
802
538
|
this.config.logMessage("Import is already in progress");
|
|
@@ -806,11 +542,7 @@ var require_index = __commonJS({
|
|
|
806
542
|
this.config.handleStatusUpdate({ status: EXECUTION_STATUS.IMPORT_IN_PROGRESS });
|
|
807
543
|
this.config.updateLastImportDate();
|
|
808
544
|
this.config.logMessage("Start importing new data");
|
|
809
|
-
|
|
810
|
-
this.storage.addHeader(this.storage.uniqueKeyColumns);
|
|
811
|
-
this.config.logMessage(`Column(s) for unique key was added: ${this.storage.uniqueKeyColumns}`);
|
|
812
|
-
}
|
|
813
|
-
this.startImportProcess();
|
|
545
|
+
await this.startImportProcess();
|
|
814
546
|
this.config.logMessage("Import is finished");
|
|
815
547
|
this.config.handleStatusUpdate({
|
|
816
548
|
status: EXECUTION_STATUS.IMPORT_DONE
|
|
@@ -831,7 +563,7 @@ var require_index = __commonJS({
|
|
|
831
563
|
/**
|
|
832
564
|
* A method for calling from Root script for determining parameters needed to fetch new data.
|
|
833
565
|
*/
|
|
834
|
-
startImportProcess() {
|
|
566
|
+
async startImportProcess() {
|
|
835
567
|
var _a;
|
|
836
568
|
let startDate = null;
|
|
837
569
|
let endDate = /* @__PURE__ */ new Date();
|
|
@@ -842,10 +574,10 @@ var require_index = __commonJS({
|
|
|
842
574
|
return;
|
|
843
575
|
}
|
|
844
576
|
endDate.setDate(startDate.getDate() + daysToFetch);
|
|
845
|
-
let data = this.source.fetchData(startDate, endDate);
|
|
577
|
+
let data = await this.source.fetchData(startDate, endDate);
|
|
846
578
|
this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
|
|
847
579
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value) === "true") {
|
|
848
|
-
this.storage.saveData(data);
|
|
580
|
+
await this.storage.saveData(data);
|
|
849
581
|
}
|
|
850
582
|
if (this.runConfig.type === RUN_CONFIG_TYPE.INCREMENTAL) {
|
|
851
583
|
this.config.updateLastRequstedDate(endDate);
|
|
@@ -1002,31 +734,12 @@ var require_index = __commonJS({
|
|
|
1002
734
|
* @param (object) with config data. Properties are parameters names, values are values
|
|
1003
735
|
*/
|
|
1004
736
|
constructor(configData) {
|
|
1005
|
-
this.addParameter("Environment", {
|
|
1006
|
-
value: AbstractConfig.detectEnvironment(),
|
|
1007
|
-
requiredType: "number",
|
|
1008
|
-
attributes: [CONFIG_ATTRIBUTES.HIDE_IN_CONFIG_FORM, CONFIG_ATTRIBUTES.ADVANCED]
|
|
1009
|
-
});
|
|
1010
737
|
for (var name in configData) {
|
|
1011
738
|
this.addParameter(name, configData[name]);
|
|
1012
739
|
}
|
|
1013
740
|
return this;
|
|
1014
741
|
}
|
|
1015
742
|
//----------------------------------------------------------------
|
|
1016
|
-
//---- static helper -------------------------------------------------
|
|
1017
|
-
/**
|
|
1018
|
-
* Determines the runtime environment
|
|
1019
|
-
* @returns {ENVIRONMENT} The detected environment
|
|
1020
|
-
*/
|
|
1021
|
-
static detectEnvironment() {
|
|
1022
|
-
if (typeof UrlFetchApp !== "undefined") {
|
|
1023
|
-
return ENVIRONMENT.APPS_SCRIPT;
|
|
1024
|
-
}
|
|
1025
|
-
if (typeof process !== "undefined") {
|
|
1026
|
-
return ENVIRONMENT.NODE;
|
|
1027
|
-
}
|
|
1028
|
-
return ENVIRONMENT.UNKNOWN;
|
|
1029
|
-
}
|
|
1030
743
|
//---- mergeParameters ---------------------------------------------
|
|
1031
744
|
/**
|
|
1032
745
|
* Merge configuration to existing config
|
|
@@ -1233,11 +946,11 @@ var require_index = __commonJS({
|
|
|
1233
946
|
}
|
|
1234
947
|
//----------------------------------------------------------------
|
|
1235
948
|
}
|
|
1236
|
-
function processShortLinks(data, { shortLinkField, urlFieldName }) {
|
|
949
|
+
async function processShortLinks(data, { shortLinkField, urlFieldName }) {
|
|
1237
950
|
if (!Array.isArray(data) || data.length === 0) return data;
|
|
1238
951
|
const shortLinks = _collectUniqueShortLinks(data, shortLinkField, urlFieldName);
|
|
1239
952
|
if (shortLinks.length === 0) return data;
|
|
1240
|
-
const resolvedShortLinks = _resolveShortLinks(shortLinks);
|
|
953
|
+
const resolvedShortLinks = await _resolveShortLinks(shortLinks);
|
|
1241
954
|
return _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName);
|
|
1242
955
|
}
|
|
1243
956
|
function _collectUniqueShortLinks(data, shortLinkField, urlFieldName) {
|
|
@@ -1259,13 +972,11 @@ var require_index = __commonJS({
|
|
|
1259
972
|
if (hasParams) return false;
|
|
1260
973
|
return /^https:\/\/[^\/]+\/[^\/]+$/.test(url);
|
|
1261
974
|
}
|
|
1262
|
-
function _resolveShortLinks(shortLinks) {
|
|
1263
|
-
|
|
975
|
+
async function _resolveShortLinks(shortLinks) {
|
|
976
|
+
const promises = shortLinks.map(async (linkObj) => {
|
|
1264
977
|
try {
|
|
1265
|
-
const response =
|
|
1266
|
-
method: "GET"
|
|
1267
|
-
followRedirects: false,
|
|
1268
|
-
muteHttpExceptions: true
|
|
978
|
+
const response = await HttpUtils.fetch(linkObj.originalUrl, {
|
|
979
|
+
method: "GET"
|
|
1269
980
|
});
|
|
1270
981
|
const headers = response.getHeaders();
|
|
1271
982
|
const resolvedUrl = headers.Location || headers.location || linkObj.originalUrl;
|
|
@@ -1281,6 +992,7 @@ var require_index = __commonJS({
|
|
|
1281
992
|
};
|
|
1282
993
|
}
|
|
1283
994
|
});
|
|
995
|
+
return Promise.all(promises);
|
|
1284
996
|
}
|
|
1285
997
|
function _populateDataWithResolvedUrls(data, resolvedShortLinks, shortLinkField, urlFieldName) {
|
|
1286
998
|
return data.map((record) => {
|
|
@@ -1304,15 +1016,15 @@ var require_index = __commonJS({
|
|
|
1304
1016
|
var OAuthUtils = {
|
|
1305
1017
|
/**
|
|
1306
1018
|
* Universal OAuth access token retrieval method
|
|
1307
|
-
*
|
|
1019
|
+
*
|
|
1308
1020
|
* @param {Object} options - All configuration options
|
|
1309
1021
|
* @param {Object} options.config - Configuration object containing credentials
|
|
1310
1022
|
* @param {string} options.tokenUrl - OAuth token endpoint URL
|
|
1311
1023
|
* @param {Object} options.formData - Form data to send in request body
|
|
1312
1024
|
* @param {Object} [options.headers] - Request headers
|
|
1313
|
-
* @returns {string} - The access token
|
|
1025
|
+
* @returns {Promise<string>} - The access token
|
|
1314
1026
|
*/
|
|
1315
|
-
getAccessToken({ config, tokenUrl, formData, headers = {} }) {
|
|
1027
|
+
async getAccessToken({ config, tokenUrl, formData, headers = {} }) {
|
|
1316
1028
|
const requestHeaders = {
|
|
1317
1029
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
1318
1030
|
...headers
|
|
@@ -1325,8 +1037,9 @@ var require_index = __commonJS({
|
|
|
1325
1037
|
body: Object.entries(formData).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
|
|
1326
1038
|
};
|
|
1327
1039
|
try {
|
|
1328
|
-
const resp =
|
|
1329
|
-
const
|
|
1040
|
+
const resp = await HttpUtils.fetch(tokenUrl, options);
|
|
1041
|
+
const text = await resp.getContentText();
|
|
1042
|
+
const json = JSON.parse(text);
|
|
1330
1043
|
if (json.error) {
|
|
1331
1044
|
throw new Error(`Token error: ${json.error}`);
|
|
1332
1045
|
}
|
|
@@ -1339,7 +1052,7 @@ var require_index = __commonJS({
|
|
|
1339
1052
|
},
|
|
1340
1053
|
/**
|
|
1341
1054
|
* Get access token using Service Account JWT authentication
|
|
1342
|
-
*
|
|
1055
|
+
*
|
|
1343
1056
|
* @param {Object} options - Configuration options
|
|
1344
1057
|
* @param {Object} options.config - Configuration object
|
|
1345
1058
|
* @param {string} options.tokenUrl - Token URL
|
|
@@ -1347,7 +1060,7 @@ var require_index = __commonJS({
|
|
|
1347
1060
|
* @param {string} options.scope - OAuth scope (e.g., "https://www.googleapis.com/auth/adwords")
|
|
1348
1061
|
* @returns {string} - The access token
|
|
1349
1062
|
*/
|
|
1350
|
-
getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
|
|
1063
|
+
async getServiceAccountToken({ config, tokenUrl, serviceAccountKeyJson, scope }) {
|
|
1351
1064
|
try {
|
|
1352
1065
|
const serviceAccountData = JSON.parse(serviceAccountKeyJson);
|
|
1353
1066
|
const now = Math.floor(Date.now() / 1e3);
|
|
@@ -1365,7 +1078,7 @@ var require_index = __commonJS({
|
|
|
1365
1078
|
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
1366
1079
|
assertion: jwt
|
|
1367
1080
|
};
|
|
1368
|
-
const accessToken = this.getAccessToken({
|
|
1081
|
+
const accessToken = await this.getAccessToken({
|
|
1369
1082
|
config,
|
|
1370
1083
|
tokenUrl,
|
|
1371
1084
|
formData
|
|
@@ -1418,6 +1131,68 @@ var require_index = __commonJS({
|
|
|
1418
1131
|
return base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
|
|
1419
1132
|
}
|
|
1420
1133
|
};
|
|
1134
|
+
var HttpUtils = class HttpUtils {
|
|
1135
|
+
/**
|
|
1136
|
+
* Fetch data from the given URL.
|
|
1137
|
+
*
|
|
1138
|
+
* @param {string} url - The URL to fetch data from.
|
|
1139
|
+
* @param {Object} options - Options for the fetch request.
|
|
1140
|
+
* @returns {Promise<FetchResponse>}
|
|
1141
|
+
*/
|
|
1142
|
+
static async fetch(url, options = {}) {
|
|
1143
|
+
const method = options.method || "GET";
|
|
1144
|
+
const fetchOptions = {
|
|
1145
|
+
method: method.toUpperCase(),
|
|
1146
|
+
headers: options.headers || {}
|
|
1147
|
+
};
|
|
1148
|
+
if (options.body) {
|
|
1149
|
+
fetchOptions.body = options.body;
|
|
1150
|
+
}
|
|
1151
|
+
const response = await fetch(url, fetchOptions);
|
|
1152
|
+
return this._wrapNodeResponse(response);
|
|
1153
|
+
}
|
|
1154
|
+
/**
|
|
1155
|
+
* Wraps the response from the Node environment.
|
|
1156
|
+
* Not use directly, only for internal purposes.
|
|
1157
|
+
*
|
|
1158
|
+
* @param {Response} response - Native fetch Response object
|
|
1159
|
+
* @returns {FetchResponse}
|
|
1160
|
+
*/
|
|
1161
|
+
static _wrapNodeResponse(response) {
|
|
1162
|
+
let textCache = null;
|
|
1163
|
+
let blobCache = null;
|
|
1164
|
+
const getText = async () => {
|
|
1165
|
+
if (textCache === null) {
|
|
1166
|
+
textCache = await response.text();
|
|
1167
|
+
}
|
|
1168
|
+
return textCache;
|
|
1169
|
+
};
|
|
1170
|
+
const getBlob = async () => {
|
|
1171
|
+
if (blobCache === null) {
|
|
1172
|
+
blobCache = await response.arrayBuffer();
|
|
1173
|
+
}
|
|
1174
|
+
return Buffer.from(blobCache);
|
|
1175
|
+
};
|
|
1176
|
+
const headersObj = {};
|
|
1177
|
+
response.headers.forEach((value, key) => {
|
|
1178
|
+
headersObj[key] = value;
|
|
1179
|
+
});
|
|
1180
|
+
return {
|
|
1181
|
+
getHeaders: () => headersObj,
|
|
1182
|
+
getAsJson: () => getText().then((text) => {
|
|
1183
|
+
try {
|
|
1184
|
+
return JSON.parse(text);
|
|
1185
|
+
} catch (e) {
|
|
1186
|
+
throw new Error("Invalid JSON response");
|
|
1187
|
+
}
|
|
1188
|
+
}),
|
|
1189
|
+
getContent: () => getText(),
|
|
1190
|
+
getContentText: () => getText(),
|
|
1191
|
+
getBlob: () => getBlob(),
|
|
1192
|
+
getResponseCode: () => response.status
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
};
|
|
1421
1196
|
var FormatUtils = {
|
|
1422
1197
|
/**
|
|
1423
1198
|
* Universal ID parser: parses comma/semicolon separated string to array of numeric IDs
|
|
@@ -1460,6 +1235,102 @@ var require_index = __commonJS({
|
|
|
1460
1235
|
}, {});
|
|
1461
1236
|
}
|
|
1462
1237
|
};
|
|
1238
|
+
var FileUtils = class FileUtils {
|
|
1239
|
+
/**
|
|
1240
|
+
* Parse CSV string into array of arrays
|
|
1241
|
+
*
|
|
1242
|
+
* @param {string} csvString - The CSV string to parse
|
|
1243
|
+
* @param {string} [delimiter=','] - The delimiter to use for parsing CSV
|
|
1244
|
+
* @returns {Array<Array<string>>} Parsed CSV data
|
|
1245
|
+
*/
|
|
1246
|
+
static parseCsv(csvString, delimiter = ",") {
|
|
1247
|
+
return csvString.split("\n").filter((line) => line.trim() !== "").map((line) => line.split(delimiter).map((cell) => {
|
|
1248
|
+
const trimmed = cell.trim();
|
|
1249
|
+
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
1250
|
+
return trimmed.slice(1, -1).replace(/""/g, '"');
|
|
1251
|
+
}
|
|
1252
|
+
return trimmed;
|
|
1253
|
+
}));
|
|
1254
|
+
}
|
|
1255
|
+
/**
|
|
1256
|
+
* Unzip a blob/buffer
|
|
1257
|
+
*
|
|
1258
|
+
* @param {Buffer} data - The data to unzip
|
|
1259
|
+
* @returns {Array<{getDataAsString: Function}>} Array of file-like objects with getDataAsString method
|
|
1260
|
+
*/
|
|
1261
|
+
static unzip(data) {
|
|
1262
|
+
const zip = new AdmZip(data);
|
|
1263
|
+
return zip.getEntries().map((entry) => ({
|
|
1264
|
+
getDataAsString: () => entry.getData().toString("utf8")
|
|
1265
|
+
}));
|
|
1266
|
+
}
|
|
1267
|
+
};
|
|
1268
|
+
var DateUtils = class DateUtils {
|
|
1269
|
+
/**
|
|
1270
|
+
* Format the given date to ISO format (YYYY-MM-DD).
|
|
1271
|
+
*
|
|
1272
|
+
* @param {Date} date - The date to format.
|
|
1273
|
+
* @returns {string} ISO formatted date (YYYY-MM-DD)
|
|
1274
|
+
*/
|
|
1275
|
+
static formatDate(date) {
|
|
1276
|
+
return date.toISOString().split("T")[0];
|
|
1277
|
+
}
|
|
1278
|
+
};
|
|
1279
|
+
var CryptoUtils = class CryptoUtils {
|
|
1280
|
+
/**
|
|
1281
|
+
* Mac algorithm constants.
|
|
1282
|
+
*
|
|
1283
|
+
* @type {Object}
|
|
1284
|
+
*/
|
|
1285
|
+
static get MacAlgorithm() {
|
|
1286
|
+
return {
|
|
1287
|
+
HMAC_SHA_256: "HMAC_SHA_256",
|
|
1288
|
+
HMAC_SHA_384: "HMAC_SHA_384",
|
|
1289
|
+
HMAC_SHA_512: "HMAC_SHA_512",
|
|
1290
|
+
HMAC_SHA_1: "HMAC_SHA_1",
|
|
1291
|
+
HMAC_MD5: "HMAC_MD5"
|
|
1292
|
+
};
|
|
1293
|
+
}
|
|
1294
|
+
/**
|
|
1295
|
+
* Get a UUID. Format: `${string}-${string}-${string}-${string}-${string}`
|
|
1296
|
+
*
|
|
1297
|
+
* @returns {string} UUID
|
|
1298
|
+
*/
|
|
1299
|
+
static getUuid() {
|
|
1300
|
+
const crypto = require("node:crypto");
|
|
1301
|
+
return crypto.randomUUID();
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Encode the given data to base64.
|
|
1305
|
+
*
|
|
1306
|
+
* @param {string} data - The data to encode.
|
|
1307
|
+
* @returns {string}
|
|
1308
|
+
*/
|
|
1309
|
+
static base64Encode(data) {
|
|
1310
|
+
return Buffer.from(data).toString("base64");
|
|
1311
|
+
}
|
|
1312
|
+
/**
|
|
1313
|
+
* Compute the HMAC signature for the given data.
|
|
1314
|
+
*
|
|
1315
|
+
* @param {string} algorithm - The algorithm to use (e.g., 'HMAC_SHA_256', 'sha256').
|
|
1316
|
+
* @param {string} data - The data to compute the signature for.
|
|
1317
|
+
* @param {string} key - The key to use.
|
|
1318
|
+
* @returns {Array<number>} Byte array of the signature
|
|
1319
|
+
*/
|
|
1320
|
+
static computeHmacSignature(algorithm, data, key) {
|
|
1321
|
+
const crypto = require("node:crypto");
|
|
1322
|
+
const algorithmMap = {
|
|
1323
|
+
"HMAC_SHA_256": "sha256",
|
|
1324
|
+
"HMAC_SHA_384": "sha384",
|
|
1325
|
+
"HMAC_SHA_512": "sha512",
|
|
1326
|
+
"HMAC_SHA_1": "sha1",
|
|
1327
|
+
"HMAC_MD5": "md5"
|
|
1328
|
+
};
|
|
1329
|
+
const nodeAlgorithm = algorithmMap[algorithm] || algorithm.toLowerCase().replace("hmac_", "");
|
|
1330
|
+
const buffer = crypto.createHmac(nodeAlgorithm, key).update(data).digest();
|
|
1331
|
+
return Array.from(buffer);
|
|
1332
|
+
}
|
|
1333
|
+
};
|
|
1463
1334
|
var ConnectorUtils = {
|
|
1464
1335
|
/**
|
|
1465
1336
|
* Check if a node is a time series node
|
|
@@ -1482,6 +1353,17 @@ var require_index = __commonJS({
|
|
|
1482
1353
|
}, {});
|
|
1483
1354
|
}
|
|
1484
1355
|
};
|
|
1356
|
+
var AsyncUtils = class AsyncUtils {
|
|
1357
|
+
/**
|
|
1358
|
+
* Async delay for the given number of milliseconds.
|
|
1359
|
+
*
|
|
1360
|
+
* @param {number} ms - The number of milliseconds to delay.
|
|
1361
|
+
* @returns {Promise<void>}
|
|
1362
|
+
*/
|
|
1363
|
+
static async delay(ms) {
|
|
1364
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1365
|
+
}
|
|
1366
|
+
};
|
|
1485
1367
|
class RunConfigDto {
|
|
1486
1368
|
constructor(config) {
|
|
1487
1369
|
this._config = config;
|
|
@@ -1893,902 +1775,38 @@ var require_index = __commonJS({
|
|
|
1893
1775
|
);
|
|
1894
1776
|
}
|
|
1895
1777
|
}
|
|
1896
|
-
var GoogleSheetsConfig = class GoogleSheetsConfig extends AbstractConfig {
|
|
1897
|
-
//---- constructor -------------------------------------------------
|
|
1898
|
-
constructor(configRange) {
|
|
1899
|
-
if (typeof configRange.getA1Notation !== "function") {
|
|
1900
|
-
throw new Error(`Unable to create an GoogleSheetsConfig object. The first constructor's parameter must be an SpreadsheetApp.Sheet object`);
|
|
1901
|
-
}
|
|
1902
|
-
let configObject = {};
|
|
1903
|
-
configRange.getValues().forEach((row, index) => {
|
|
1904
|
-
var name = row[0].replaceAll(/[^a-zA-Z0-9]/gi, "");
|
|
1905
|
-
if (!["", "Parameters", "Status", "Name"].includes(name)) {
|
|
1906
|
-
var value = row[1];
|
|
1907
|
-
configObject[name] = {};
|
|
1908
|
-
if (value || value === 0) {
|
|
1909
|
-
configObject[name].value = value;
|
|
1910
|
-
}
|
|
1911
|
-
if (row[0].slice(-1) == "*") {
|
|
1912
|
-
configObject[name].isRequired = true;
|
|
1913
|
-
}
|
|
1914
|
-
if (["Log", "CurrentStatus", "LastImportDate", "LastRequestedDate", "DestinationSpreadsheet"].includes(name)) {
|
|
1915
|
-
configObject[name].cell = configRange.offset(index, 1, 1, 1);
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
|
-
});
|
|
1919
|
-
configObject.configSpreadsheet = configRange.getSheet().getParent();
|
|
1920
|
-
configObject.Log.timeZone = configRange.getSheet().getParent().getSpreadsheetTimeZone();
|
|
1921
|
-
super(configObject);
|
|
1922
|
-
this.mergeParameters({
|
|
1923
|
-
MaxRunTimeout: {
|
|
1924
|
-
isRequired: true,
|
|
1925
|
-
requiredType: "number",
|
|
1926
|
-
default: 30
|
|
1927
|
-
},
|
|
1928
|
-
NotifyByEmail: {
|
|
1929
|
-
isRequired: false,
|
|
1930
|
-
requiredType: "string",
|
|
1931
|
-
default: ""
|
|
1932
|
-
},
|
|
1933
|
-
NotifyByGoogleChat: {
|
|
1934
|
-
isRequired: false,
|
|
1935
|
-
requiredType: "string",
|
|
1936
|
-
default: ""
|
|
1937
|
-
},
|
|
1938
|
-
NotifyWhen: {
|
|
1939
|
-
isRequired: false,
|
|
1940
|
-
requiredType: "string",
|
|
1941
|
-
default: "Never"
|
|
1942
|
-
}
|
|
1943
|
-
});
|
|
1944
|
-
}
|
|
1945
|
-
//---- handleStatusUpdate -----------------------------------------------
|
|
1946
|
-
/**
|
|
1947
|
-
* @param {Object} params - Parameters object with status and other properties
|
|
1948
|
-
* @param {number} params.status - Status constant
|
|
1949
|
-
* @param {string} params.error - Error message for Error status
|
|
1950
|
-
*/
|
|
1951
|
-
handleStatusUpdate({ status, error }) {
|
|
1952
|
-
this.manageTimeoutTrigger(status);
|
|
1953
|
-
this.updateCurrentStatus(status);
|
|
1954
|
-
if (this.shouldSendNotifications(status)) {
|
|
1955
|
-
this.sendNotifications({ status, error });
|
|
1956
|
-
}
|
|
1957
|
-
}
|
|
1958
|
-
//----------------------------------------------------------------
|
|
1959
|
-
//---- manageTimeoutTrigger ----------------------------------------
|
|
1960
|
-
/**
|
|
1961
|
-
* Manage timeout trigger based on current status
|
|
1962
|
-
* @param {number} status - Status constant
|
|
1963
|
-
*/
|
|
1964
|
-
manageTimeoutTrigger(status) {
|
|
1965
|
-
if (status === EXECUTION_STATUS.IMPORT_IN_PROGRESS) {
|
|
1966
|
-
this.createTimeoutTrigger();
|
|
1967
|
-
} else if (status === EXECUTION_STATUS.IMPORT_DONE || status === EXECUTION_STATUS.ERROR) {
|
|
1968
|
-
this.removeTimeoutTrigger();
|
|
1969
|
-
}
|
|
1970
|
-
}
|
|
1971
|
-
//---- getStatusProperties ------------------------------------------
|
|
1972
|
-
/**
|
|
1973
|
-
* Get all properties for a given status
|
|
1974
|
-
* @param {number} status - Status constant
|
|
1975
|
-
* @returns {Object} - Object with all status properties
|
|
1976
|
-
*/
|
|
1977
|
-
getStatusProperties(status) {
|
|
1978
|
-
switch (status) {
|
|
1979
|
-
case EXECUTION_STATUS.IMPORT_IN_PROGRESS:
|
|
1980
|
-
return {
|
|
1981
|
-
displayText: "Import in progress",
|
|
1982
|
-
backgroundColor: "#c9e3f9",
|
|
1983
|
-
notificationMessage: "Import is in progress."
|
|
1984
|
-
};
|
|
1985
|
-
case EXECUTION_STATUS.CLEANUP_IN_PROGRESS:
|
|
1986
|
-
return {
|
|
1987
|
-
displayText: "CleanUp in progress",
|
|
1988
|
-
backgroundColor: "#c9e3f9",
|
|
1989
|
-
notificationMessage: "Cleanup is in progress."
|
|
1990
|
-
};
|
|
1991
|
-
case EXECUTION_STATUS.IMPORT_DONE:
|
|
1992
|
-
return {
|
|
1993
|
-
displayText: "Done",
|
|
1994
|
-
backgroundColor: "#d4efd5",
|
|
1995
|
-
notificationMessage: "Import completed successfully."
|
|
1996
|
-
};
|
|
1997
|
-
case EXECUTION_STATUS.CLEANUP_DONE:
|
|
1998
|
-
return {
|
|
1999
|
-
displayText: "Done",
|
|
2000
|
-
backgroundColor: "#d4efd5",
|
|
2001
|
-
notificationMessage: "Cleanup completed successfully."
|
|
2002
|
-
};
|
|
2003
|
-
case EXECUTION_STATUS.ERROR:
|
|
2004
|
-
return {
|
|
2005
|
-
displayText: "Error",
|
|
2006
|
-
backgroundColor: "#fdd2cf",
|
|
2007
|
-
notificationMessage: "Error occurred"
|
|
2008
|
-
};
|
|
2009
|
-
default:
|
|
2010
|
-
throw new Error(`Unknown status constant: ${status}`);
|
|
2011
|
-
}
|
|
2012
|
-
}
|
|
2013
|
-
//----------------------------------------------------------------
|
|
2014
|
-
//---- updateCurrentStatus -----------------------------------------
|
|
2015
|
-
/**
|
|
2016
|
-
* @param {number} status - Status constant
|
|
2017
|
-
*/
|
|
2018
|
-
updateCurrentStatus(status) {
|
|
2019
|
-
const statusProps = this.getStatusProperties(status);
|
|
2020
|
-
this.CurrentStatus.cell.setValue(statusProps.displayText);
|
|
2021
|
-
this.CurrentStatus.cell.setBackground(statusProps.backgroundColor);
|
|
2022
|
-
}
|
|
2023
|
-
//----------------------------------------------------------------
|
|
2024
|
-
//---- updateLastImportDate ----------------------------------------
|
|
2025
|
-
/**
|
|
2026
|
-
* updating the last import attempt date in a config sheet
|
|
2027
|
-
*/
|
|
2028
|
-
updateLastImportDate() {
|
|
2029
|
-
this.LastImportDate.cell.setValue(
|
|
2030
|
-
EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss")
|
|
2031
|
-
);
|
|
2032
|
-
}
|
|
2033
|
-
//----------------------------------------------------------------
|
|
2034
|
-
//---- updateLastRequstedDate --------------------------------------
|
|
2035
|
-
/**
|
|
2036
|
-
* Updating the last requested date in a config sheet
|
|
2037
|
-
* @param date Date requested date
|
|
2038
|
-
*/
|
|
2039
|
-
updateLastRequstedDate(date) {
|
|
2040
|
-
if ("LastRequestedDate" in this && (!this.LastRequestedDate.value || date.getTime() != this.LastRequestedDate.value.getTime())) {
|
|
2041
|
-
this.LastRequestedDate.value = new Date(date.getTime());
|
|
2042
|
-
this.LastRequestedDate.cell.setValue(EnvironmentAdapter.formatDate(date, this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"));
|
|
2043
|
-
}
|
|
2044
|
-
}
|
|
2045
|
-
//----------------------------------------------------------------
|
|
2046
|
-
//---- updateFieldsSheet -------------------------------------------
|
|
2047
|
-
/**
|
|
2048
|
-
* Updating the content of the fields list to simplify the selection of fields for import
|
|
2049
|
-
* @param connector AbstractSource
|
|
2050
|
-
* @param sheetName string, Fields by default
|
|
2051
|
-
*/
|
|
2052
|
-
updateFieldsSheet(source, sheetName = "Fields") {
|
|
2053
|
-
this.validate();
|
|
2054
|
-
var configSheetStorage = new GoogleSheetsStorage(
|
|
2055
|
-
this.mergeParameters({
|
|
2056
|
-
DestinationSheetName: {
|
|
2057
|
-
// @TODO: make sure it would affect destination sheet for import data. CONFIG is an object. Probably we might dublicate it
|
|
2058
|
-
value: sheetName
|
|
2059
|
-
}
|
|
2060
|
-
}),
|
|
2061
|
-
["id"]
|
|
2062
|
-
);
|
|
2063
|
-
var groups = source.getFieldsSchema();
|
|
2064
|
-
var data = [];
|
|
2065
|
-
for (var groupName in groups) {
|
|
2066
|
-
data.push({
|
|
2067
|
-
"id": groupName,
|
|
2068
|
-
"✔️": groupName,
|
|
2069
|
-
"name": "",
|
|
2070
|
-
"description": `=IF( COUNTIFS(A:A, "${groupName} *", B:B, TRUE) > 0, COUNTIFS(A:A, "${groupName} *", B:B, TRUE), "")`
|
|
2071
|
-
});
|
|
2072
|
-
data.push({ "id": `${groupName} !desc`, "✔️": groups[groupName].description });
|
|
2073
|
-
data.push({ "id": `${groupName} !doc`, "✔️": groups[groupName].documentation });
|
|
2074
|
-
if (groups[groupName].fields) {
|
|
2075
|
-
for (var fieldName in groups[groupName].fields) {
|
|
2076
|
-
data.push({
|
|
2077
|
-
"id": groupName + " " + fieldName,
|
|
2078
|
-
"name": fieldName,
|
|
2079
|
-
"description": groups[groupName].fields[fieldName].description
|
|
2080
|
-
});
|
|
2081
|
-
}
|
|
2082
|
-
data.push({ "id": `${groupName} zzz_separator` });
|
|
2083
|
-
}
|
|
2084
|
-
}
|
|
2085
|
-
for (var groupName in groups) {
|
|
2086
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2087
|
-
if (groupRow !== null) {
|
|
2088
|
-
let depth = configSheetStorage.SHEET.getRowGroupDepth(groupRow.rowIndex + 5);
|
|
2089
|
-
if (depth > 0) {
|
|
2090
|
-
configSheetStorage.SHEET.getRowGroup(groupRow.rowIndex + 5, depth).remove();
|
|
2091
|
-
}
|
|
2092
|
-
}
|
|
2093
|
-
}
|
|
2094
|
-
configSheetStorage.saveData(data);
|
|
2095
|
-
configSheetStorage.SHEET.sort(configSheetStorage.getColumnIndexByName("id"));
|
|
2096
|
-
configSheetStorage.loadDataFromSheet();
|
|
2097
|
-
configSheetStorage.SHEET.hideColumns(configSheetStorage.getColumnIndexByName("id"));
|
|
2098
|
-
configSheetStorage.SHEET.setColumnWidth(configSheetStorage.getColumnIndexByName("✔️"), 25);
|
|
2099
|
-
configSheetStorage.SHEET.autoResizeColumn(3);
|
|
2100
|
-
var checkboxRule = SpreadsheetApp.newDataValidation().requireCheckbox().build();
|
|
2101
|
-
for (var groupName in groups) {
|
|
2102
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2103
|
-
var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 2}`);
|
|
2104
|
-
range.setFontWeight("bold").setBackground("black").setFontColor("white");
|
|
2105
|
-
var groupRow = configSheetStorage.getRecordByRecordFields({ "id": `${groupName} !desc` });
|
|
2106
|
-
var range = configSheetStorage.SHEET.getRange(`${groupRow.rowIndex + 2}:${groupRow.rowIndex + 3}`);
|
|
2107
|
-
range.setBackground("#f3f3f3");
|
|
2108
|
-
var checkboxesColumnIndex = configSheetStorage.getColumnIndexByName("✔️");
|
|
2109
|
-
var groupNameRow = configSheetStorage.getRecordByRecordFields({ "id": groupName });
|
|
2110
|
-
var groupFieldsCount = this.getFieldsCountByGroupName(configSheetStorage, groupName);
|
|
2111
|
-
configSheetStorage.SHEET.getRange(groupNameRow.rowIndex + 5, checkboxesColumnIndex, groupFieldsCount).setDataValidation(checkboxRule);
|
|
2112
|
-
configSheetStorage.SHEET.getRange(`${groupNameRow.rowIndex + 5}:${groupNameRow.rowIndex + 4 + groupFieldsCount}`).shiftRowGroupDepth(1);
|
|
2113
|
-
}
|
|
2114
|
-
configSheetStorage.SHEET.collapseAllRowGroups();
|
|
2115
|
-
}
|
|
2116
|
-
//----------------------------------------------------------------
|
|
2117
|
-
//---- getFieldsCountByGroupName -----------------------------------
|
|
2118
|
-
/**
|
|
2119
|
-
* @param GoogleSheetsConfig object
|
|
2120
|
-
* @param groupName string name of the group
|
|
2121
|
-
* @return integer number of fields of requested group
|
|
2122
|
-
*/
|
|
2123
|
-
getFieldsCountByGroupName(configSheetStorage, groupName) {
|
|
2124
|
-
return Object.values(configSheetStorage.values).filter((element) => element.name && element.id.startsWith(`${groupName} `)).length;
|
|
2125
|
-
}
|
|
2126
|
-
//----------------------------------------------------------------
|
|
2127
|
-
//---- isInProgress ------------------------------------------------
|
|
2128
|
-
/**
|
|
2129
|
-
* Checking current status if it is in progress or not
|
|
2130
|
-
* @return boolean true is process in progress
|
|
2131
|
-
*/
|
|
2132
|
-
isInProgress() {
|
|
2133
|
-
let isInProgress = null;
|
|
2134
|
-
if (this.CurrentStatus.cell.getValue().indexOf("progress") !== -1) {
|
|
2135
|
-
let diff = (/* @__PURE__ */ new Date() - new Date(EnvironmentAdapter.formatDate(this.LastImportDate.cell.getValue(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss"))) / (1e3 * 60);
|
|
2136
|
-
isInProgress = diff < this.MaxRunTimeout.value;
|
|
2137
|
-
} else {
|
|
2138
|
-
isInProgress = false;
|
|
2139
|
-
}
|
|
2140
|
-
return isInProgress;
|
|
2141
|
-
}
|
|
2142
|
-
//----------------------------------------------------------------
|
|
2143
|
-
//---- addWarningToCurrentStatus -----------------------------------
|
|
2144
|
-
addWarningToCurrentStatus() {
|
|
2145
|
-
this.CurrentStatus.cell.setBackground("#fff0c4");
|
|
2146
|
-
}
|
|
2147
|
-
//----------------------------------------------------------------
|
|
2148
|
-
//---- validate ----------------------------------------------------
|
|
2149
|
-
/**
|
|
2150
|
-
* validating if google sheets config is correct
|
|
2151
|
-
*/
|
|
2152
|
-
validate() {
|
|
2153
|
-
let scriptTimeZone = Session.getScriptTimeZone();
|
|
2154
|
-
if (scriptTimeZone != "Etc/UTC") {
|
|
2155
|
-
throw new Error(`The Apps Script time zone must be set to UTC to avoid confusion with dates. Currently, it is set to ${scriptTimeZone} instead. Update the time zone in Project Settings`);
|
|
2156
|
-
}
|
|
2157
|
-
super.validate();
|
|
2158
|
-
}
|
|
2159
|
-
//----------------------------------------------------------------
|
|
2160
|
-
//---- logMessage --------------------------------------------------
|
|
2161
|
-
/**
|
|
2162
|
-
* @param string message to Log
|
|
2163
|
-
*/
|
|
2164
|
-
logMessage(message, removeExistingMessage = false) {
|
|
2165
|
-
console.log(message);
|
|
2166
|
-
let formattedDate = EnvironmentAdapter.formatDate(/* @__PURE__ */ new Date(), this.Log.timeZone, "yyyy-MM-dd HH:mm:ss");
|
|
2167
|
-
let currentLog = removeExistingMessage ? "" : this.Log.cell.getValue();
|
|
2168
|
-
currentLog ? currentLog += "\n" : "";
|
|
2169
|
-
let emoji = "☑️ ";
|
|
2170
|
-
let match;
|
|
2171
|
-
if (match = message.match(new RegExp("^(\\p{Emoji_Presentation}|\\p{Emoji}\\uFE0F|\\p{Extended_Pictographic})\\s+", "u"))) {
|
|
2172
|
-
emoji = match[0];
|
|
2173
|
-
message = message.slice(2).trim();
|
|
2174
|
-
}
|
|
2175
|
-
this.Log.cell.setValue(
|
|
2176
|
-
`${currentLog}${emoji}${formattedDate}: ${message}`
|
|
2177
|
-
);
|
|
2178
|
-
this.updateLastImportDate();
|
|
2179
|
-
}
|
|
2180
|
-
showCredentialsDialog(source) {
|
|
2181
|
-
const ui = SpreadsheetApp.getUi();
|
|
2182
|
-
const template = HtmlService.createTemplateFromFile("Views/credentials-input-dialog");
|
|
2183
|
-
template.source = source;
|
|
2184
|
-
const html = template.evaluate().setWidth(400).setHeight(450);
|
|
2185
|
-
ui.showModalDialog(html, `${source.constructor.name} Credentials`);
|
|
2186
|
-
}
|
|
2187
|
-
showManualBackfillDialog(source) {
|
|
2188
|
-
const ui = SpreadsheetApp.getUi();
|
|
2189
|
-
const template = HtmlService.createTemplateFromFile("Views/manual-backfill-dialog");
|
|
2190
|
-
template.source = source;
|
|
2191
|
-
const html = template.evaluate().setWidth(600).setHeight(300);
|
|
2192
|
-
ui.showModalDialog(html, "Manual Backfill");
|
|
2193
|
-
}
|
|
2194
|
-
//---- sendNotifications -------------------------------------------
|
|
2195
|
-
/**
|
|
2196
|
-
* Send notifications based on configuration settings
|
|
2197
|
-
* @param {Object} params - Parameters object
|
|
2198
|
-
* @param {string} params.status - Current status value
|
|
2199
|
-
* @param {string} params.error - Error message for Error status
|
|
2200
|
-
*/
|
|
2201
|
-
sendNotifications({ status, error }) {
|
|
2202
|
-
try {
|
|
2203
|
-
const { title, messageWithDetails } = this.prepareNotificationContent({ status, error });
|
|
2204
|
-
if (this.NotifyByEmail && this.NotifyByEmail.value && this.NotifyByEmail.value.trim()) {
|
|
2205
|
-
EmailNotification.send({
|
|
2206
|
-
to: this.NotifyByEmail.value,
|
|
2207
|
-
subject: title,
|
|
2208
|
-
message: messageWithDetails
|
|
2209
|
-
});
|
|
2210
|
-
}
|
|
2211
|
-
if (this.NotifyByGoogleChat && this.NotifyByGoogleChat.value && this.NotifyByGoogleChat.value.trim()) {
|
|
2212
|
-
GoogleChatNotification.send({
|
|
2213
|
-
webhookUrl: this.NotifyByGoogleChat.value.trim(),
|
|
2214
|
-
message: messageWithDetails
|
|
2215
|
-
});
|
|
2216
|
-
}
|
|
2217
|
-
} catch (error2) {
|
|
2218
|
-
this.logMessage(`⚠️ Notification error: ${error2.message}`);
|
|
2219
|
-
}
|
|
2220
|
-
}
|
|
2221
|
-
//----------------------------------------------------------------
|
|
2222
|
-
//---- prepareNotificationContent ----------------------------------
|
|
2223
|
-
/**
|
|
2224
|
-
* Prepare notification title and message content
|
|
2225
|
-
* @param {Object} params - Parameters object
|
|
2226
|
-
* @param {number} params.status - Status constant
|
|
2227
|
-
* @param {string} params.error - Error message for Error status
|
|
2228
|
-
* @returns {Object} - Object with title and messageWithDetails properties
|
|
2229
|
-
*/
|
|
2230
|
-
prepareNotificationContent({ status, error }) {
|
|
2231
|
-
const documentName = this.configSpreadsheet.getName();
|
|
2232
|
-
const documentUrl = this.configSpreadsheet.getUrl();
|
|
2233
|
-
const { displayText, notificationMessage } = this.getStatusProperties(status);
|
|
2234
|
-
const title = `${documentName} - Status: ${displayText}`;
|
|
2235
|
-
return {
|
|
2236
|
-
title,
|
|
2237
|
-
messageWithDetails: `${title}
|
|
2238
|
-
${documentUrl}
|
|
2239
|
-
|
|
2240
|
-
${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}` : ""}`
|
|
2241
|
-
};
|
|
2242
|
-
}
|
|
2243
|
-
//----------------------------------------------------------------
|
|
2244
|
-
//---- shouldSendNotifications -------------------------------------
|
|
2245
|
-
/**
|
|
2246
|
-
* Determine if notifications should be sent based on status and filter setting
|
|
2247
|
-
* @param {number} status - Status constant
|
|
2248
|
-
* @returns {boolean} - True if notifications should be sent
|
|
2249
|
-
*/
|
|
2250
|
-
shouldSendNotifications(status) {
|
|
2251
|
-
var _a;
|
|
2252
|
-
const notifyWhen = (_a = this.NotifyWhen) == null ? void 0 : _a.value;
|
|
2253
|
-
switch (notifyWhen) {
|
|
2254
|
-
case "On error":
|
|
2255
|
-
return status === EXECUTION_STATUS.ERROR;
|
|
2256
|
-
case "On success":
|
|
2257
|
-
return status === EXECUTION_STATUS.IMPORT_DONE;
|
|
2258
|
-
case "Always":
|
|
2259
|
-
return status === EXECUTION_STATUS.ERROR || status === EXECUTION_STATUS.IMPORT_DONE;
|
|
2260
|
-
case "Never":
|
|
2261
|
-
case "":
|
|
2262
|
-
default:
|
|
2263
|
-
return false;
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
//----------------------------------------------------------------
|
|
2267
|
-
//---- createTimeoutTrigger ----------------------------------------
|
|
2268
|
-
createTimeoutTrigger() {
|
|
2269
|
-
this.removeTimeoutTrigger();
|
|
2270
|
-
ScriptApp.newTrigger("checkForTimeout").timeBased().after((this.MaxRunTimeout.value * 2 + 1) * 60 * 1e3).create();
|
|
2271
|
-
}
|
|
2272
|
-
//----------------------------------------------------------------
|
|
2273
|
-
//---- removeTimeoutTrigger ----------------------------------------
|
|
2274
|
-
removeTimeoutTrigger() {
|
|
2275
|
-
const triggers = ScriptApp.getProjectTriggers();
|
|
2276
|
-
let removedCount = 0;
|
|
2277
|
-
triggers.forEach((trigger) => {
|
|
2278
|
-
if (trigger.getHandlerFunction() === "checkForTimeout") {
|
|
2279
|
-
ScriptApp.deleteTrigger(trigger);
|
|
2280
|
-
removedCount++;
|
|
2281
|
-
}
|
|
2282
|
-
});
|
|
2283
|
-
console.log(`[TimeoutTrigger] ${removedCount > 0 ? `Removed ${removedCount} timeout trigger(s)` : "No timeout triggers found to remove"}`);
|
|
2284
|
-
}
|
|
2285
|
-
//----------------------------------------------------------------
|
|
2286
|
-
//---- checkForTimeout ---------------------------------------------
|
|
2287
|
-
checkForTimeout() {
|
|
2288
|
-
if (!this.isInProgress()) {
|
|
2289
|
-
console.log("[TimeoutTrigger] Status is NOT in progress, setting to Error and sending notification");
|
|
2290
|
-
this.handleStatusUpdate({
|
|
2291
|
-
status: EXECUTION_STATUS.ERROR,
|
|
2292
|
-
error: "Import was interrupted (likely due to timeout)"
|
|
2293
|
-
});
|
|
2294
|
-
} else {
|
|
2295
|
-
console.log("[TimeoutTrigger] Status is still in progress");
|
|
2296
|
-
}
|
|
2297
|
-
}
|
|
2298
|
-
//----------------------------------------------------------------
|
|
2299
|
-
};
|
|
2300
|
-
var GoogleChatNotification = class GoogleChatNotification {
|
|
2301
|
-
/**
|
|
2302
|
-
* Send Google Chat notification
|
|
2303
|
-
* @param {Object} params - Parameters object
|
|
2304
|
-
* @param {string} params.webhookUrl - Google Chat webhook URL
|
|
2305
|
-
* @param {string} params.message - Formatted notification message
|
|
2306
|
-
*/
|
|
2307
|
-
static send(params) {
|
|
2308
|
-
const { webhookUrl, message } = params;
|
|
2309
|
-
if (!webhookUrl || !webhookUrl.trim()) {
|
|
2310
|
-
return;
|
|
2311
|
-
}
|
|
2312
|
-
try {
|
|
2313
|
-
const response = EnvironmentAdapter.fetch(webhookUrl.trim(), {
|
|
2314
|
-
method: "POST",
|
|
2315
|
-
headers: {
|
|
2316
|
-
"Content-Type": "application/json; charset=UTF-8"
|
|
2317
|
-
},
|
|
2318
|
-
payload: JSON.stringify({ text: message })
|
|
2319
|
-
});
|
|
2320
|
-
if (response.getResponseCode() === 200) {
|
|
2321
|
-
console.log("Google Chat notification sent successfully");
|
|
2322
|
-
} else {
|
|
2323
|
-
console.error(`Google Chat notification failed with status: ${response.getResponseCode()}`);
|
|
2324
|
-
}
|
|
2325
|
-
} catch (error) {
|
|
2326
|
-
console.error("Failed to send Google Chat notification:", error);
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
};
|
|
2330
|
-
var EmailNotification = class EmailNotification {
|
|
2331
|
-
/**
|
|
2332
|
-
* Send email notification
|
|
2333
|
-
* @param {Object} params - Parameters object
|
|
2334
|
-
* @param {string} params.to - Email address(es) to send to (can be multiple separated by commas)
|
|
2335
|
-
* @param {string} params.subject - Email subject line
|
|
2336
|
-
* @param {string} params.message - Formatted notification message
|
|
2337
|
-
*/
|
|
2338
|
-
static send(params) {
|
|
2339
|
-
const { to, subject, message } = params;
|
|
2340
|
-
if (!to || !to.trim()) {
|
|
2341
|
-
return;
|
|
2342
|
-
}
|
|
2343
|
-
try {
|
|
2344
|
-
const emailAddresses = to.split(",").map((email) => email.trim()).filter((email) => email.length > 0).filter((email) => this.isValidEmail(email)).join(",");
|
|
2345
|
-
if (!emailAddresses) {
|
|
2346
|
-
console.log("Email notification skipped: no valid email addresses found");
|
|
2347
|
-
return;
|
|
2348
|
-
}
|
|
2349
|
-
MailApp.sendEmail(
|
|
2350
|
-
emailAddresses,
|
|
2351
|
-
subject,
|
|
2352
|
-
message,
|
|
2353
|
-
{ noReply: true }
|
|
2354
|
-
);
|
|
2355
|
-
console.log(`Email notification sent successfully to: ${emailAddresses}`);
|
|
2356
|
-
} catch (error) {
|
|
2357
|
-
console.error(`Failed to send email notification: ${error.message}`);
|
|
2358
|
-
}
|
|
2359
|
-
}
|
|
2360
|
-
/**
|
|
2361
|
-
* Validate email address format
|
|
2362
|
-
* @param {string} email - Email address to validate
|
|
2363
|
-
* @returns {boolean} - True if email is valid
|
|
2364
|
-
*/
|
|
2365
|
-
static isValidEmail(email) {
|
|
2366
|
-
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
|
2367
|
-
const isValid = emailRegex.test(email);
|
|
2368
|
-
if (!isValid) {
|
|
2369
|
-
console.log(`Invalid email address skipped: ${email}`);
|
|
2370
|
-
}
|
|
2371
|
-
return isValid;
|
|
2372
|
-
}
|
|
2373
|
-
};
|
|
2374
1778
|
const Core = {
|
|
2375
1779
|
AbstractException,
|
|
2376
1780
|
HttpRequestException,
|
|
2377
1781
|
UnsupportedEnvironmentException,
|
|
2378
|
-
EnvironmentAdapter,
|
|
2379
1782
|
AbstractStorage,
|
|
2380
1783
|
AbstractSource,
|
|
2381
1784
|
AbstractRunConfig,
|
|
2382
1785
|
AbstractConnector,
|
|
2383
1786
|
AbstractConfig,
|
|
1787
|
+
HttpUtils,
|
|
1788
|
+
FileUtils,
|
|
1789
|
+
DateUtils,
|
|
1790
|
+
CryptoUtils,
|
|
1791
|
+
AsyncUtils,
|
|
2384
1792
|
RunConfigDto,
|
|
2385
1793
|
SourceConfigDto,
|
|
2386
1794
|
StorageConfigDto,
|
|
2387
1795
|
ConfigDto,
|
|
2388
1796
|
NodeJsConfig,
|
|
2389
|
-
GoogleSheetsConfig,
|
|
2390
|
-
GoogleChatNotification,
|
|
2391
|
-
EmailNotification,
|
|
2392
1797
|
HTTP_STATUS,
|
|
2393
|
-
ENVIRONMENT,
|
|
2394
1798
|
EXECUTION_STATUS,
|
|
2395
1799
|
RUN_CONFIG_TYPE,
|
|
2396
1800
|
CONFIG_ATTRIBUTES
|
|
2397
1801
|
};
|
|
2398
|
-
const GoogleSheets = (function() {
|
|
2399
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, EnvironmentAdapter: EnvironmentAdapter2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, ENVIRONMENT: ENVIRONMENT2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
2400
|
-
var GoogleSheetsStorage2 = class GoogleSheetsStorage extends AbstractStorage2 {
|
|
2401
|
-
//---- constructor -------------------------------------------------
|
|
2402
|
-
/**
|
|
2403
|
-
* Asbstract class making Google Sheets data active in Apps Script to simplity read/write operations
|
|
2404
|
-
* @param config (object) instance of AbscractConfig
|
|
2405
|
-
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2406
|
-
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2407
|
-
*/
|
|
2408
|
-
constructor(config, uniqueKeyColumns, schema = null) {
|
|
2409
|
-
super(
|
|
2410
|
-
config.mergeParameters({
|
|
2411
|
-
CleanUpToKeepWindow: {
|
|
2412
|
-
requiredType: "number"
|
|
2413
|
-
},
|
|
2414
|
-
DestinationSheetName: {
|
|
2415
|
-
isRequired: true,
|
|
2416
|
-
default: "Data"
|
|
2417
|
-
}
|
|
2418
|
-
}),
|
|
2419
|
-
uniqueKeyColumns,
|
|
2420
|
-
schema
|
|
2421
|
-
);
|
|
2422
|
-
this.SHEET = this.getDestinationSheet(config);
|
|
2423
|
-
this.loadDataFromSheet();
|
|
2424
|
-
}
|
|
2425
|
-
//----------------------------------------------------------------
|
|
2426
|
-
//---- loadDataFromSheet -------------------------------------------
|
|
2427
|
-
/**
|
|
2428
|
-
* reading Data from the source sheet and loading it to this.values
|
|
2429
|
-
*/
|
|
2430
|
-
loadDataFromSheet() {
|
|
2431
|
-
const values = this.SHEET.getDataRange().getValues();
|
|
2432
|
-
this.columnNames = values.shift();
|
|
2433
|
-
if (this.uniqueKeyColumns.some((column) => !this.columnNames.includes(column))) {
|
|
2434
|
-
throw new Error(`Sheet '${this.SHEET.getName()}' is missing one the folling columns required for unique key: column '${this.uniqueKeyColumns}'`);
|
|
2435
|
-
}
|
|
2436
|
-
this.values = values.reduce((acc, row, rowIndex) => {
|
|
2437
|
-
const uniqueKey = this.uniqueKeyColumns.reduce((accumulator, columnName) => {
|
|
2438
|
-
let index = this.columnNames.indexOf(columnName);
|
|
2439
|
-
accumulator += `|${row[index]}`;
|
|
2440
|
-
return accumulator;
|
|
2441
|
-
}, []);
|
|
2442
|
-
acc[uniqueKey] = {
|
|
2443
|
-
...this.columnNames.reduce((obj, col, colIndex) => ({
|
|
2444
|
-
...obj,
|
|
2445
|
-
[col]: row[colIndex]
|
|
2446
|
-
}), {}),
|
|
2447
|
-
rowIndex
|
|
2448
|
-
// Add row index for reference
|
|
2449
|
-
};
|
|
2450
|
-
return acc;
|
|
2451
|
-
}, {});
|
|
2452
|
-
this.addedRecordsBuffer = [];
|
|
2453
|
-
}
|
|
2454
|
-
//----------------------------------------------------------------
|
|
2455
|
-
//---- getDestinationSheet -----------------------------------------
|
|
2456
|
-
/**
|
|
2457
|
-
* @param object destination Spreadsheet Config
|
|
2458
|
-
* @param object destination Sheet Name Config
|
|
2459
|
-
* @return Sheet object for data Destination
|
|
2460
|
-
*/
|
|
2461
|
-
getDestinationSheet(config) {
|
|
2462
|
-
if (!this.SHEET) {
|
|
2463
|
-
if (!config.DestinationSpreadsheet || !config.DestinationSpreadsheet.value) {
|
|
2464
|
-
config.DestinationSpreadsheet = { "spreadsheet": config.configSpreadsheet };
|
|
2465
|
-
} else {
|
|
2466
|
-
let match = config.DestinationSpreadsheet.value.match(/\/d\/([a-zA-Z0-9-_]+)/);
|
|
2467
|
-
if (match && match[1]) {
|
|
2468
|
-
config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match[1]);
|
|
2469
|
-
} else {
|
|
2470
|
-
let match2 = config.DestinationSpreadsheet.cell.getRichTextValue().getLinkUrl().match(/\/d\/([a-zA-Z0-9-_]+)/);
|
|
2471
|
-
if (match2 && match2[1]) {
|
|
2472
|
-
config.DestinationSpreadsheet.spreadsheet = SpreadsheetApp.openById(match2[1]);
|
|
2473
|
-
} else {
|
|
2474
|
-
throw new Error(`Destination Spreadsheet must be specified in config either by Spreadsheet Id or by a link to spreadsheet`);
|
|
2475
|
-
}
|
|
2476
|
-
}
|
|
2477
|
-
}
|
|
2478
|
-
if (config.DestinationSpreadsheet.spreadsheet == null) {
|
|
2479
|
-
this.config.logMessage(`Cannot load destination sheet document`);
|
|
2480
|
-
}
|
|
2481
|
-
config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.getSheetByName(
|
|
2482
|
-
config.DestinationSheetName.value
|
|
2483
|
-
);
|
|
2484
|
-
if (!config.DestinationSpreadsheet.sheet) {
|
|
2485
|
-
config.DestinationSpreadsheet.sheet = config.DestinationSpreadsheet.spreadsheet.insertSheet(
|
|
2486
|
-
config.DestinationSheetName.value,
|
|
2487
|
-
config.DestinationSpreadsheet.spreadsheet.getSheets().length
|
|
2488
|
-
);
|
|
2489
|
-
this.config.logMessage(`Sheet '${config.DestinationSheetName.value}' was created.`);
|
|
2490
|
-
}
|
|
2491
|
-
this.SHEET = config.DestinationSpreadsheet.sheet;
|
|
2492
|
-
if (this.isEmpty()) {
|
|
2493
|
-
this.addHeader(this.uniqueKeyColumns);
|
|
2494
|
-
this.config.logMessage(`Columns for unique keys were added to '${config.DestinationSheetName.value}' sheet.`);
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
return this.SHEET;
|
|
2498
|
-
}
|
|
2499
|
-
//----------------------------------------------------------------
|
|
2500
|
-
//---- addRecord ---------------------------------------------------
|
|
2501
|
-
/**
|
|
2502
|
-
* Checking if record exists by id
|
|
2503
|
-
* @param object {record} object with record data
|
|
2504
|
-
* @param Boolean {useBuffer}: Set to `false` if the record must be saved instantly, or `true` to save it later using `this.saveAddedRecordsFromBuffer()`.
|
|
2505
|
-
* @return object {record} with added rowIndex property
|
|
2506
|
-
*/
|
|
2507
|
-
addRecord(record, useBuffer = false) {
|
|
2508
|
-
record = this.stringifyNeastedFields(record);
|
|
2509
|
-
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2510
|
-
if (useBuffer) {
|
|
2511
|
-
this.addedRecordsBuffer[uniqueKey] = record;
|
|
2512
|
-
} else {
|
|
2513
|
-
let data = this.columnNames.map((key) => record[key] || "");
|
|
2514
|
-
this.SHEET.appendRow(data);
|
|
2515
|
-
record.rowIndex = this.SHEET.getLastRow() - 2;
|
|
2516
|
-
this.values[uniqueKey] = record;
|
|
2517
|
-
}
|
|
2518
|
-
return record;
|
|
2519
|
-
}
|
|
2520
|
-
//----------------------------------------------------------------
|
|
2521
|
-
//---- saveData ----------------------------------------------------
|
|
2522
|
-
/**
|
|
2523
|
-
* Saving data to a storage
|
|
2524
|
-
* @param {data} array of assoc objects with records to save
|
|
2525
|
-
*/
|
|
2526
|
-
saveData(data) {
|
|
2527
|
-
var recordsAdded = 0;
|
|
2528
|
-
var recordsUpdated = 0;
|
|
2529
|
-
data.map((row) => {
|
|
2530
|
-
let newFields = Object.keys(row).filter((column2) => !this.columnNames.includes(column2));
|
|
2531
|
-
for (var column in newFields) {
|
|
2532
|
-
this.addColumn(newFields[column], this.columnNames.length + 1);
|
|
2533
|
-
this.config.logMessage(`Column '${newFields[column]}' was added to '${this.SHEET.getName()}' sheet`);
|
|
2534
|
-
}
|
|
2535
|
-
if (this.isRecordExists(row)) {
|
|
2536
|
-
if (this.updateRecord(row)) {
|
|
2537
|
-
recordsUpdated++;
|
|
2538
|
-
}
|
|
2539
|
-
} else {
|
|
2540
|
-
this.addRecord(row, true);
|
|
2541
|
-
recordsAdded += this.saveRecordsAddedToBuffer(100);
|
|
2542
|
-
}
|
|
2543
|
-
});
|
|
2544
|
-
recordsAdded += this.saveRecordsAddedToBuffer(0);
|
|
2545
|
-
if (recordsAdded > 0) {
|
|
2546
|
-
this.config.logMessage(`${recordsAdded} records were added`);
|
|
2547
|
-
}
|
|
2548
|
-
if (recordsUpdated > 0) {
|
|
2549
|
-
this.config.logMessage(`${recordsUpdated} records were updated`);
|
|
2550
|
-
}
|
|
2551
|
-
}
|
|
2552
|
-
//----------------------------------------------------------------
|
|
2553
|
-
//---- saveRecordsAddedToBuffer ------------------------------------
|
|
2554
|
-
/**
|
|
2555
|
-
* Add records from buffer to a sheet
|
|
2556
|
-
* @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
|
|
2557
|
-
*/
|
|
2558
|
-
saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2559
|
-
let recordsAdded = 0;
|
|
2560
|
-
let bufferSize = Object.keys(this.addedRecordsBuffer).length;
|
|
2561
|
-
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
2562
|
-
let startIndex = this.SHEET.getLastRow() - 2;
|
|
2563
|
-
let index = 1;
|
|
2564
|
-
let data = [];
|
|
2565
|
-
for (var uniqueKey in this.addedRecordsBuffer) {
|
|
2566
|
-
let record = this.addedRecordsBuffer[uniqueKey];
|
|
2567
|
-
record.rowIndex = startIndex + index++;
|
|
2568
|
-
this.values[uniqueKey] = record;
|
|
2569
|
-
data.push(this.columnNames.map((key) => record[key] || ""));
|
|
2570
|
-
}
|
|
2571
|
-
this.SHEET.getRange(startIndex + 3, 1, data.length, data[0].length).setValues(data);
|
|
2572
|
-
recordsAdded = bufferSize;
|
|
2573
|
-
this.addedRecordsBuffer = {};
|
|
2574
|
-
}
|
|
2575
|
-
return recordsAdded;
|
|
2576
|
-
}
|
|
2577
|
-
//----------------------------------------------------------------
|
|
2578
|
-
//---- updateRecord ------------------------------------------------
|
|
2579
|
-
/**
|
|
2580
|
-
* Update content of an existing record
|
|
2581
|
-
* @param object {record} object with record data
|
|
2582
|
-
* @return boolean Returns true if the record was updated; otherwise, returns false
|
|
2583
|
-
*/
|
|
2584
|
-
updateRecord(record) {
|
|
2585
|
-
record = this.stringifyNeastedFields(record);
|
|
2586
|
-
let uniqueKey = this.getUniqueKeyByRecordFields(record);
|
|
2587
|
-
var existingRecord = this.getRecordByUniqueKey(uniqueKey);
|
|
2588
|
-
var isRecordUpdated = false;
|
|
2589
|
-
this.columnNames.forEach((columnName, columnIndex) => {
|
|
2590
|
-
if (columnName in record && !this.areValuesEqual(record[columnName], existingRecord[columnName])) {
|
|
2591
|
-
console.log(`${uniqueKey}: ${existingRecord[columnName]} ${typeof existingRecord[columnName]} → ${record[columnName]} ${typeof record[columnName]}`);
|
|
2592
|
-
this.SHEET.getRange(existingRecord.rowIndex + 2, columnIndex + 1, 1, 1).setValue(record[columnName]);
|
|
2593
|
-
existingRecord[columnName] = record[columnName];
|
|
2594
|
-
isRecordUpdated = true;
|
|
2595
|
-
}
|
|
2596
|
-
});
|
|
2597
|
-
return isRecordUpdated;
|
|
2598
|
-
}
|
|
2599
|
-
//----------------------------------------------------------------
|
|
2600
|
-
//---- deleteRecord ------------------------------------------------
|
|
2601
|
-
/**
|
|
2602
|
-
* Delete record from a sheet
|
|
2603
|
-
* @param uniqueKey {string} unique key of the record to delete
|
|
2604
|
-
*/
|
|
2605
|
-
deleteRecord(uniqueKey) {
|
|
2606
|
-
if (!(uniqueKey in this.values)) {
|
|
2607
|
-
throw new Error(`Unable to delete the record with ID ${uniqueKey} because it was not found`);
|
|
2608
|
-
} else if (!("rowIndex" in this.values[uniqueKey])) {
|
|
2609
|
-
throw new Error(`Unable to delete the record with ID ${uniqueKey} because it does not have a rowIndex`);
|
|
2610
|
-
} else {
|
|
2611
|
-
let rowIndex = this.values[uniqueKey].rowIndex;
|
|
2612
|
-
this.SHEET.deleteRow(rowIndex + 2);
|
|
2613
|
-
for (uniqueKey in this.values) {
|
|
2614
|
-
if (this.values[uniqueKey].rowIndex > rowIndex) {
|
|
2615
|
-
this.values[uniqueKey].rowIndex--;
|
|
2616
|
-
}
|
|
2617
|
-
}
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
//----------------------------------------------------------------
|
|
2621
|
-
//---- addHeader ---------------------------------------------------
|
|
2622
|
-
/**
|
|
2623
|
-
* Adding header to sheet
|
|
2624
|
-
* @param target Sheet
|
|
2625
|
-
* @param array column names to be added
|
|
2626
|
-
*/
|
|
2627
|
-
addHeader(columnNames) {
|
|
2628
|
-
columnNames.forEach((columnName, index) => {
|
|
2629
|
-
this.addColumn(columnName, index + 1);
|
|
2630
|
-
});
|
|
2631
|
-
this.SHEET.getRange("1:1").setBackground("#f3f3f3").setHorizontalAlignment("center");
|
|
2632
|
-
this.SHEET.setFrozenRows(1);
|
|
2633
|
-
}
|
|
2634
|
-
//----------------------------------------------------------------
|
|
2635
|
-
//---- addColumn ---------------------------------------------------
|
|
2636
|
-
/**
|
|
2637
|
-
* Adding a column to the sheet
|
|
2638
|
-
* @param columnName (string) column name
|
|
2639
|
-
* @param columnIndex (integer) optional; column index
|
|
2640
|
-
*/
|
|
2641
|
-
addColumn(columnName, columnIndex = 1) {
|
|
2642
|
-
const numColumns = this.SHEET.getMaxColumns();
|
|
2643
|
-
if (columnIndex <= 0 || columnIndex > numColumns + 1) {
|
|
2644
|
-
throw new Error(`Column index ${columnIndex} is out of bounds (1-${numColumns + 1})`);
|
|
2645
|
-
}
|
|
2646
|
-
if (columnIndex <= numColumns) {
|
|
2647
|
-
const headerValue = this.SHEET.getRange(1, columnIndex).getValue();
|
|
2648
|
-
if (headerValue !== "") {
|
|
2649
|
-
const findFirstEmptyColumn = (startIndex) => {
|
|
2650
|
-
let index = startIndex;
|
|
2651
|
-
let foundEmpty = false;
|
|
2652
|
-
while (index <= numColumns) {
|
|
2653
|
-
if (this.SHEET.getRange(1, index).getValue() === "") {
|
|
2654
|
-
foundEmpty = true;
|
|
2655
|
-
break;
|
|
2656
|
-
}
|
|
2657
|
-
index++;
|
|
2658
|
-
}
|
|
2659
|
-
return {
|
|
2660
|
-
columnIndex: index,
|
|
2661
|
-
foundEmpty
|
|
2662
|
-
};
|
|
2663
|
-
};
|
|
2664
|
-
const result = findFirstEmptyColumn(columnIndex);
|
|
2665
|
-
if (!result.foundEmpty) {
|
|
2666
|
-
this.SHEET.insertColumnAfter(numColumns);
|
|
2667
|
-
columnIndex = numColumns + 1;
|
|
2668
|
-
} else {
|
|
2669
|
-
this.SHEET.insertColumnBefore(result.columnIndex);
|
|
2670
|
-
columnIndex = result.columnIndex;
|
|
2671
|
-
}
|
|
2672
|
-
}
|
|
2673
|
-
} else {
|
|
2674
|
-
this.SHEET.insertColumnAfter(numColumns);
|
|
2675
|
-
columnIndex = numColumns + 1;
|
|
2676
|
-
}
|
|
2677
|
-
this.SHEET.getRange(1, columnIndex).setValue(columnName);
|
|
2678
|
-
if (this.schema != null && columnName in this.schema && "GoogleSheetsFormat" in this.schema[columnName]) {
|
|
2679
|
-
let columnLetter = String.fromCharCode(64 + columnIndex);
|
|
2680
|
-
console.log(
|
|
2681
|
-
columnName,
|
|
2682
|
-
this.schema[columnName]["GoogleSheetsFormat"],
|
|
2683
|
-
this.SHEET.getRange(`${columnLetter}:${columnLetter}`).getA1Notation()
|
|
2684
|
-
);
|
|
2685
|
-
this.SHEET.getRange(`${columnLetter}:${columnLetter}`).setNumberFormat(this.schema[columnName]["GoogleSheetsFormat"]);
|
|
2686
|
-
}
|
|
2687
|
-
this.columnNames.push(columnName);
|
|
2688
|
-
}
|
|
2689
|
-
//----------------------------------------------------------------
|
|
2690
|
-
//---- formatColumn ------------------------------------------------
|
|
2691
|
-
/**
|
|
2692
|
-
* Format column as it described in schema
|
|
2693
|
-
* @param columnName (string) column name
|
|
2694
|
-
*/
|
|
2695
|
-
formatColumn(columnName) {
|
|
2696
|
-
if ("type" in this.schema[columnName]) ;
|
|
2697
|
-
}
|
|
2698
|
-
//----------------------------------------------------------------
|
|
2699
|
-
//---- getColumnIndexByName ----------------------------------------
|
|
2700
|
-
/**
|
|
2701
|
-
* @param columnName string column name
|
|
2702
|
-
* @return integer columnIndex
|
|
2703
|
-
*/
|
|
2704
|
-
getColumnIndexByName(columnName) {
|
|
2705
|
-
const columnIndex = this.columnNames.indexOf(columnName);
|
|
2706
|
-
if (columnIndex == -1) {
|
|
2707
|
-
throw new Error(`Column ${columnName} not found in '${this.SHEET.getName()}' sheet`);
|
|
2708
|
-
}
|
|
2709
|
-
return columnIndex + 1;
|
|
2710
|
-
}
|
|
2711
|
-
//----------------------------------------------------------------
|
|
2712
|
-
//---- isEmpty -----------------------------------------------------
|
|
2713
|
-
/**
|
|
2714
|
-
* @return boolean true if sheet is empty, false overwise
|
|
2715
|
-
*/
|
|
2716
|
-
isEmpty() {
|
|
2717
|
-
return this.SHEET.getLastRow() === 0 && this.SHEET.getLastColumn() === 0;
|
|
2718
|
-
}
|
|
2719
|
-
//----------------------------------------------------------------
|
|
2720
|
-
//---- areValuesEqual ----------------------------------------------
|
|
2721
|
-
/**
|
|
2722
|
-
* Comparing to vaariables if they are equal or not
|
|
2723
|
-
* @param value1 (mixed)
|
|
2724
|
-
* @param value2 (mixed)
|
|
2725
|
-
* @return boolean true if equal, falst overwise
|
|
2726
|
-
*/
|
|
2727
|
-
areValuesEqual(value1, value2) {
|
|
2728
|
-
var equal = null;
|
|
2729
|
-
if (typeof value1 !== "undefined" && typeof value2 !== "undefined" && ((value1 == null ? void 0 : value1.constructor.name) == "Date" || (value2 == null ? void 0 : value2.constructor.name) == "Date")) {
|
|
2730
|
-
const normalizeToDate = (value) => {
|
|
2731
|
-
if (value === null || value === "") return null;
|
|
2732
|
-
if (value.constructor.name == "Date") return value;
|
|
2733
|
-
const date = new Date(value);
|
|
2734
|
-
return isNaN(date.getTime()) ? null : date;
|
|
2735
|
-
};
|
|
2736
|
-
const date1 = normalizeToDate(value1);
|
|
2737
|
-
const date2 = normalizeToDate(value2);
|
|
2738
|
-
if (date1 === null || date2 === null) {
|
|
2739
|
-
return value1 === value2;
|
|
2740
|
-
}
|
|
2741
|
-
equal = date1.getTime() === date2.getTime();
|
|
2742
|
-
} else if (typeof value1 == typeof value2) {
|
|
2743
|
-
equal = value1 === value2;
|
|
2744
|
-
} else if (value1 === void 0 && value2 === "" || value2 === void 0 && value1 === "") {
|
|
2745
|
-
equal = true;
|
|
2746
|
-
} else if (value1 === null && value2 === "" || value2 === null && value1 === "") {
|
|
2747
|
-
equal = true;
|
|
2748
|
-
}
|
|
2749
|
-
return equal;
|
|
2750
|
-
}
|
|
2751
|
-
//----------------------------------------------------------------
|
|
2752
|
-
//---- areHeadersNeeded ------------------------------------------
|
|
2753
|
-
/**
|
|
2754
|
-
* Checks if storage is empty and adds headers if needed
|
|
2755
|
-
* if destination sheet is empty than header should be created based on unique key columns list
|
|
2756
|
-
* @return {boolean} true if headers were added, false if they already existed
|
|
2757
|
-
*/
|
|
2758
|
-
areHeadersNeeded() {
|
|
2759
|
-
return this.isEmpty();
|
|
2760
|
-
}
|
|
2761
|
-
//----------------------------------------------------------------
|
|
2762
|
-
};
|
|
2763
|
-
const manifest = {
|
|
2764
|
-
"name": "GoogleSheetsStorage",
|
|
2765
|
-
"description": "Storage for Google Sheets",
|
|
2766
|
-
"title": "Google Sheets",
|
|
2767
|
-
"version": "0.0.0",
|
|
2768
|
-
"author": "OWOX, Inc.",
|
|
2769
|
-
"license": "MIT",
|
|
2770
|
-
"environment": {
|
|
2771
|
-
"node": {
|
|
2772
|
-
"enabled": false
|
|
2773
|
-
},
|
|
2774
|
-
"appscript": {
|
|
2775
|
-
"enabled": true
|
|
2776
|
-
}
|
|
2777
|
-
}
|
|
2778
|
-
};
|
|
2779
|
-
return {
|
|
2780
|
-
GoogleSheetsStorage: GoogleSheetsStorage2,
|
|
2781
|
-
manifest
|
|
2782
|
-
};
|
|
2783
|
-
})();
|
|
2784
1802
|
const GoogleBigQuery = (function() {
|
|
2785
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
1803
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
2786
1804
|
var GoogleBigQueryStorage = class GoogleBigQueryStorage extends AbstractStorage2 {
|
|
2787
1805
|
//---- constructor -------------------------------------------------
|
|
2788
1806
|
/**
|
|
2789
|
-
*
|
|
2790
|
-
*
|
|
2791
|
-
* @param config (object) instance of
|
|
1807
|
+
* Abstract class for Google BigQuery storage operations
|
|
1808
|
+
*
|
|
1809
|
+
* @param config (object) instance of AbstractConfig
|
|
2792
1810
|
* @param uniqueKeyColumns (mixed) a name of column with unique key or array with columns names
|
|
2793
1811
|
* @param schema (object) object with structure like {fieldName: {type: "number", description: "smth" } }
|
|
2794
1812
|
* @param description (string) string with storage description }
|
|
@@ -2834,22 +1852,29 @@ ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}
|
|
|
2834
1852
|
schema,
|
|
2835
1853
|
description
|
|
2836
1854
|
);
|
|
2837
|
-
this.checkIfGoogleBigQueryIsConnected();
|
|
2838
|
-
this.loadTableSchema();
|
|
2839
1855
|
this.updatedRecordsBuffer = {};
|
|
2840
1856
|
this.totalRecordsProcessed = 0;
|
|
2841
1857
|
}
|
|
1858
|
+
//---- init --------------------------------------------------------
|
|
1859
|
+
/**
|
|
1860
|
+
* Initializing storage
|
|
1861
|
+
*/
|
|
1862
|
+
async init() {
|
|
1863
|
+
this.checkIfGoogleBigQueryIsConnected();
|
|
1864
|
+
await this.loadTableSchema();
|
|
1865
|
+
}
|
|
1866
|
+
//----------------------------------------------------------------
|
|
2842
1867
|
//---- loads Google BigQuery Table Schema ---------------------------
|
|
2843
|
-
loadTableSchema() {
|
|
1868
|
+
async loadTableSchema() {
|
|
2844
1869
|
this.existingColumns = this.getAListOfExistingColumns() || {};
|
|
2845
1870
|
if (Object.keys(this.existingColumns).length == 0) {
|
|
2846
|
-
this.createDatasetIfItDoesntExist();
|
|
2847
|
-
this.existingColumns = this.createTableIfItDoesntExist();
|
|
1871
|
+
await this.createDatasetIfItDoesntExist();
|
|
1872
|
+
this.existingColumns = await this.createTableIfItDoesntExist();
|
|
2848
1873
|
} else {
|
|
2849
1874
|
let selectedFields = this.getSelectedFields();
|
|
2850
1875
|
let newFields = selectedFields.filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2851
1876
|
if (newFields.length > 0) {
|
|
2852
|
-
this.addNewColumns(newFields);
|
|
1877
|
+
await this.addNewColumns(newFields);
|
|
2853
1878
|
}
|
|
2854
1879
|
}
|
|
2855
1880
|
}
|
|
@@ -2887,17 +1912,17 @@ ${notificationMessage}${status === EXECUTION_STATUS.ERROR && error ? `: ${error}
|
|
|
2887
1912
|
return columns;
|
|
2888
1913
|
}
|
|
2889
1914
|
//---- createDatasetIfItDoesntExist --------------------------------
|
|
2890
|
-
createDatasetIfItDoesntExist() {
|
|
1915
|
+
async createDatasetIfItDoesntExist() {
|
|
2891
1916
|
let query = `---- Create Dataset if it not exists -----
|
|
2892
1917
|
`;
|
|
2893
1918
|
query += `CREATE SCHEMA IF NOT EXISTS \`${this.config.DestinationProjectID.value}.${this.config.DestinationDatasetName.value}\`
|
|
2894
1919
|
OPTIONS (
|
|
2895
1920
|
location = '${this.config.DestinationLocation.value}'
|
|
2896
1921
|
)`;
|
|
2897
|
-
this.executeQuery(query);
|
|
1922
|
+
await this.executeQuery(query);
|
|
2898
1923
|
}
|
|
2899
1924
|
//---- createTableIfItDoesntExist ----------------------------------
|
|
2900
|
-
createTableIfItDoesntExist() {
|
|
1925
|
+
async createTableIfItDoesntExist() {
|
|
2901
1926
|
let columns = [];
|
|
2902
1927
|
let columnPartitioned = null;
|
|
2903
1928
|
let existingColumns = {};
|
|
@@ -2933,15 +1958,14 @@ PARTITION BY ${columnPartitioned}`;
|
|
|
2933
1958
|
query += `
|
|
2934
1959
|
OPTIONS(description="${this.description}")`;
|
|
2935
1960
|
}
|
|
2936
|
-
this.executeQuery(query);
|
|
1961
|
+
await this.executeQuery(query);
|
|
2937
1962
|
this.config.logMessage(`Table ${this.config.DestinationDatasetID.value}.${this.config.DestinationTableName.value} was created`);
|
|
2938
1963
|
return existingColumns;
|
|
2939
1964
|
}
|
|
2940
1965
|
//---- checkIfGoogleBigQueryIsConnected ---------------------
|
|
2941
1966
|
checkIfGoogleBigQueryIsConnected() {
|
|
2942
1967
|
if (typeof BigQuery == "undefined") {
|
|
2943
|
-
throw new Error(`
|
|
2944
|
-
Extension / Apps Script / Editor / Services / + BigQuery API`);
|
|
1968
|
+
throw new Error(`BigQuery client library is not available. Ensure @google-cloud/bigquery is installed.`);
|
|
2945
1969
|
}
|
|
2946
1970
|
}
|
|
2947
1971
|
//---- addNewColumns -----------------------------------------------
|
|
@@ -2952,7 +1976,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
2952
1976
|
* @param {newColumns} array with a list of new columns
|
|
2953
1977
|
*
|
|
2954
1978
|
*/
|
|
2955
|
-
addNewColumns(newColumns) {
|
|
1979
|
+
async addNewColumns(newColumns) {
|
|
2956
1980
|
let query = "";
|
|
2957
1981
|
let columns = [];
|
|
2958
1982
|
for (var i in newColumns) {
|
|
@@ -2974,7 +1998,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
2974
1998
|
|
|
2975
1999
|
`;
|
|
2976
2000
|
query += columns.join(",\n");
|
|
2977
|
-
this.executeQuery(query);
|
|
2001
|
+
await this.executeQuery(query);
|
|
2978
2002
|
this.config.logMessage(`Columns '${newColumns.join(",")}' were added to ${this.config.DestinationDatasetID.value} dataset`);
|
|
2979
2003
|
}
|
|
2980
2004
|
}
|
|
@@ -2983,17 +2007,17 @@ OPTIONS(description="${this.description}")`;
|
|
|
2983
2007
|
* Saving data to a storage
|
|
2984
2008
|
* @param {data} array of assoc objects with records to save
|
|
2985
2009
|
*/
|
|
2986
|
-
saveData(data) {
|
|
2987
|
-
|
|
2010
|
+
async saveData(data) {
|
|
2011
|
+
for (const row of data) {
|
|
2988
2012
|
let newFields = Object.keys(row).filter((column) => !Object.keys(this.existingColumns).includes(column));
|
|
2989
2013
|
if (newFields.length > 0) {
|
|
2990
2014
|
console.log(newFields);
|
|
2991
|
-
this.addNewColumns(newFields);
|
|
2015
|
+
await this.addNewColumns(newFields);
|
|
2992
2016
|
}
|
|
2993
2017
|
this.addRecordToBuffer(row);
|
|
2994
|
-
this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2995
|
-
}
|
|
2996
|
-
this.saveRecordsAddedToBuffer();
|
|
2018
|
+
await this.saveRecordsAddedToBuffer(this.config.MaxBufferSize.value);
|
|
2019
|
+
}
|
|
2020
|
+
await this.saveRecordsAddedToBuffer();
|
|
2997
2021
|
}
|
|
2998
2022
|
// ------- addReordTuBuffer ---------------------
|
|
2999
2023
|
/**
|
|
@@ -3008,24 +2032,24 @@ OPTIONS(description="${this.description}")`;
|
|
|
3008
2032
|
* Add records from buffer to a sheet
|
|
3009
2033
|
* @param (integer) {maxBufferSize} record will be added only if buffer size if larger than this parameter
|
|
3010
2034
|
*/
|
|
3011
|
-
saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
2035
|
+
async saveRecordsAddedToBuffer(maxBufferSize = 0) {
|
|
3012
2036
|
let bufferSize = Object.keys(this.updatedRecordsBuffer).length;
|
|
3013
2037
|
if (bufferSize && bufferSize >= maxBufferSize) {
|
|
3014
2038
|
console.log(`Starting BigQuery MERGE operation for ${bufferSize} records...`);
|
|
3015
|
-
this.executeQueryWithSizeLimit();
|
|
2039
|
+
await this.executeQueryWithSizeLimit();
|
|
3016
2040
|
}
|
|
3017
2041
|
}
|
|
3018
2042
|
//---- executeQueryWithSizeLimit ----------------------------------
|
|
3019
2043
|
/**
|
|
3020
2044
|
* Executes the MERGE query with automatic size reduction if it exceeds BigQuery limits
|
|
3021
2045
|
*/
|
|
3022
|
-
executeQueryWithSizeLimit() {
|
|
2046
|
+
async executeQueryWithSizeLimit() {
|
|
3023
2047
|
const bufferKeys = Object.keys(this.updatedRecordsBuffer);
|
|
3024
2048
|
const totalRecords = bufferKeys.length;
|
|
3025
2049
|
if (totalRecords === 0) {
|
|
3026
2050
|
return;
|
|
3027
2051
|
}
|
|
3028
|
-
this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
2052
|
+
await this.executeMergeQueryRecursively(bufferKeys, totalRecords);
|
|
3029
2053
|
this.updatedRecordsBuffer = {};
|
|
3030
2054
|
}
|
|
3031
2055
|
//---- executeMergeQueryRecursively --------------------------------
|
|
@@ -3034,7 +2058,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3034
2058
|
* @param {Array} recordKeys - Array of record keys to process
|
|
3035
2059
|
* @param {number} batchSize - Current batch size to attempt
|
|
3036
2060
|
*/
|
|
3037
|
-
executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
2061
|
+
async executeMergeQueryRecursively(recordKeys, batchSize) {
|
|
3038
2062
|
if (recordKeys.length === 0) {
|
|
3039
2063
|
return;
|
|
3040
2064
|
}
|
|
@@ -3048,11 +2072,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
3048
2072
|
const maxQuerySize = 1024 * 1024;
|
|
3049
2073
|
if (querySize > maxQuerySize) {
|
|
3050
2074
|
console.log(`Query size (${Math.round(querySize / 1024)}KB) exceeds BigQuery limit. Reducing batch size from ${batchSize} to ${Math.floor(batchSize / 2)}`);
|
|
3051
|
-
this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
2075
|
+
await this.executeMergeQueryRecursively(recordKeys, Math.floor(batchSize / 2));
|
|
3052
2076
|
return;
|
|
3053
2077
|
}
|
|
3054
2078
|
try {
|
|
3055
|
-
this.executeQuery(query);
|
|
2079
|
+
await this.executeQuery(query);
|
|
3056
2080
|
this.totalRecordsProcessed += currentBatch.length;
|
|
3057
2081
|
console.log(`BigQuery MERGE completed successfully for ${currentBatch.length} records (Total processed: ${this.totalRecordsProcessed})`);
|
|
3058
2082
|
if (remainingRecords.length > 0) {
|
|
@@ -3086,9 +2110,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
3086
2110
|
if (record[columnName] === void 0 || record[columnName] === null) {
|
|
3087
2111
|
columnValue = null;
|
|
3088
2112
|
} else if (columnType.toUpperCase() == "DATE" && record[columnName] instanceof Date) {
|
|
3089
|
-
columnValue =
|
|
2113
|
+
columnValue = DateUtils2.formatDate(record[columnName]);
|
|
3090
2114
|
} else if (columnType.toUpperCase() == "DATETIME" && record[columnName] instanceof Date) {
|
|
3091
|
-
|
|
2115
|
+
const isoString = record[columnName].toISOString();
|
|
2116
|
+
columnValue = isoString.replace("T", " ").substring(0, 19);
|
|
3092
2117
|
} else {
|
|
3093
2118
|
columnValue = this.obfuscateSpecialCharacters(record[columnName]);
|
|
3094
2119
|
}
|
|
@@ -3123,53 +2148,36 @@ OPTIONS(description="${this.description}")`;
|
|
|
3123
2148
|
//---- query -------------------------------------------------------
|
|
3124
2149
|
/**
|
|
3125
2150
|
* Executes Google BigQuery Query and returns a result
|
|
3126
|
-
*
|
|
3127
|
-
* @param {query} string
|
|
3128
|
-
*
|
|
3129
|
-
* @return object
|
|
3130
|
-
*
|
|
2151
|
+
*
|
|
2152
|
+
* @param {query} string
|
|
2153
|
+
*
|
|
2154
|
+
* @return Promise<object>
|
|
2155
|
+
*
|
|
3131
2156
|
*/
|
|
3132
|
-
executeQuery(query) {
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
let error = void 0;
|
|
3142
|
-
let bigqueryClient = null;
|
|
3143
|
-
if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
|
|
3144
|
-
const { JWT } = require("google-auth-library");
|
|
3145
|
-
const credentials = JSON.parse(this.config.ServiceAccountJson.value);
|
|
3146
|
-
const authClient = new JWT({
|
|
3147
|
-
email: credentials.client_email,
|
|
3148
|
-
key: credentials.private_key,
|
|
3149
|
-
scopes: ["https://www.googleapis.com/auth/bigquery"]
|
|
3150
|
-
});
|
|
3151
|
-
bigqueryClient = new BigQuery({
|
|
3152
|
-
projectId: this.config.ProjectID.value || credentials.project_id,
|
|
3153
|
-
authClient
|
|
3154
|
-
});
|
|
3155
|
-
} else {
|
|
3156
|
-
throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
|
|
3157
|
-
}
|
|
3158
|
-
const options = {
|
|
3159
|
-
query,
|
|
3160
|
-
useLegacySql: false
|
|
3161
|
-
};
|
|
3162
|
-
bigqueryClient.createQueryJob(options).then(([job]) => job.getQueryResults()).then(([rows]) => rows).then((value) => {
|
|
3163
|
-
result = value;
|
|
3164
|
-
}).catch((e) => {
|
|
3165
|
-
error = e;
|
|
2157
|
+
async executeQuery(query) {
|
|
2158
|
+
let bigqueryClient = null;
|
|
2159
|
+
if (this.config.ServiceAccountJson && this.config.ServiceAccountJson.value) {
|
|
2160
|
+
const { JWT } = require("google-auth-library");
|
|
2161
|
+
const credentials = JSON.parse(this.config.ServiceAccountJson.value);
|
|
2162
|
+
const authClient = new JWT({
|
|
2163
|
+
email: credentials.client_email,
|
|
2164
|
+
key: credentials.private_key,
|
|
2165
|
+
scopes: ["https://www.googleapis.com/auth/bigquery"]
|
|
3166
2166
|
});
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
}
|
|
3171
|
-
|
|
2167
|
+
bigqueryClient = new BigQuery({
|
|
2168
|
+
projectId: this.config.ProjectID.value || credentials.project_id,
|
|
2169
|
+
authClient
|
|
2170
|
+
});
|
|
2171
|
+
} else {
|
|
2172
|
+
throw new Error("Service account JSON is required to connect to Google BigQuery in Node.js environment");
|
|
3172
2173
|
}
|
|
2174
|
+
const options = {
|
|
2175
|
+
query,
|
|
2176
|
+
useLegacySql: false
|
|
2177
|
+
};
|
|
2178
|
+
const [job] = await bigqueryClient.createQueryJob(options);
|
|
2179
|
+
const [rows] = await job.getQueryResults();
|
|
2180
|
+
return rows;
|
|
3173
2181
|
}
|
|
3174
2182
|
//---- obfuscateSpecialCharacters ----------------------------------
|
|
3175
2183
|
obfuscateSpecialCharacters(inputString) {
|
|
@@ -3265,7 +2273,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3265
2273
|
};
|
|
3266
2274
|
})();
|
|
3267
2275
|
const AwsAthena = (function() {
|
|
3268
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
2276
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
3269
2277
|
var AwsAthenaStorage = class AwsAthenaStorage extends AbstractStorage2 {
|
|
3270
2278
|
//---- constructor -------------------------------------------------
|
|
3271
2279
|
/**
|
|
@@ -3323,9 +2331,21 @@ OPTIONS(description="${this.description}")`;
|
|
|
3323
2331
|
this.initAWS();
|
|
3324
2332
|
this.updatedRecordsBuffer = {};
|
|
3325
2333
|
this.existingColumns = {};
|
|
3326
|
-
this.setupAthenaDatabase();
|
|
3327
2334
|
this.uploadSid = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:.]/g, "") + "_" + Math.random().toString(36).substring(2, 15);
|
|
3328
2335
|
}
|
|
2336
|
+
//---- init --------------------------------------------------------
|
|
2337
|
+
/**
|
|
2338
|
+
* Initializing storage
|
|
2339
|
+
*/
|
|
2340
|
+
async init() {
|
|
2341
|
+
const success = await this.setupAthenaDatabase();
|
|
2342
|
+
if (success) {
|
|
2343
|
+
console.log("Database created or already exists");
|
|
2344
|
+
} else {
|
|
2345
|
+
throw new Error("Failed to create database");
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
//----------------------------------------------------------------
|
|
3329
2349
|
//---- initAWS ----------------------------------------------------
|
|
3330
2350
|
/**
|
|
3331
2351
|
* Initialize AWS SDK clients
|
|
@@ -3357,61 +2377,58 @@ OPTIONS(description="${this.description}")`;
|
|
|
3357
2377
|
/**
|
|
3358
2378
|
* Create Athena database if it doesn't exist
|
|
3359
2379
|
*/
|
|
3360
|
-
createDatabaseIfNotExists() {
|
|
2380
|
+
async createDatabaseIfNotExists() {
|
|
3361
2381
|
const params = {
|
|
3362
2382
|
QueryString: `CREATE SCHEMA IF NOT EXISTS \`${this.config.AthenaDatabaseName.value}\``,
|
|
3363
2383
|
ResultConfiguration: {
|
|
3364
2384
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3365
2385
|
}
|
|
3366
2386
|
};
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
});
|
|
2387
|
+
await this.executeQuery(params, "ddl");
|
|
2388
|
+
this.config.logMessage(`Database ${this.config.AthenaDatabaseName.value} created or already exists`);
|
|
2389
|
+
return true;
|
|
3371
2390
|
}
|
|
3372
2391
|
//---- checkTableExists ------------------------------------------
|
|
3373
2392
|
/**
|
|
3374
2393
|
* Check if the target table exists in Athena
|
|
3375
2394
|
*/
|
|
3376
|
-
checkTableExists() {
|
|
2395
|
+
async checkTableExists() {
|
|
3377
2396
|
const params = {
|
|
3378
2397
|
QueryString: `SHOW TABLES IN \`${this.config.AthenaDatabaseName.value}\` LIKE '${this.config.DestinationTableName.value}'`,
|
|
3379
2398
|
ResultConfiguration: {
|
|
3380
2399
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3381
2400
|
}
|
|
3382
2401
|
};
|
|
3383
|
-
|
|
2402
|
+
try {
|
|
2403
|
+
const results = await this.executeQuery(params, "ddl");
|
|
3384
2404
|
if (results && results.length > 0) {
|
|
3385
|
-
return this.getTableSchema();
|
|
2405
|
+
return await this.getTableSchema();
|
|
3386
2406
|
}
|
|
3387
|
-
return this.createTargetTable();
|
|
3388
|
-
}
|
|
3389
|
-
return this.createTargetTable();
|
|
3390
|
-
}
|
|
2407
|
+
return await this.createTargetTable();
|
|
2408
|
+
} catch {
|
|
2409
|
+
return await this.createTargetTable();
|
|
2410
|
+
}
|
|
3391
2411
|
}
|
|
3392
2412
|
//---- getTableSchema -------------------------------------------
|
|
3393
2413
|
/**
|
|
3394
2414
|
* Get the schema of the existing table
|
|
3395
2415
|
*/
|
|
3396
|
-
getTableSchema() {
|
|
2416
|
+
async getTableSchema() {
|
|
3397
2417
|
const params = {
|
|
3398
2418
|
QueryString: `SHOW COLUMNS IN \`${this.config.AthenaDatabaseName.value}\`.\`${this.config.DestinationTableName.value}\``,
|
|
3399
2419
|
ResultConfiguration: {
|
|
3400
2420
|
OutputLocation: this.config.AthenaOutputLocation.value
|
|
3401
2421
|
}
|
|
3402
2422
|
};
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
}).catch((error) => {
|
|
3413
|
-
return {};
|
|
3414
|
-
});
|
|
2423
|
+
const results = await this.executeQuery(params, "ddl");
|
|
2424
|
+
let columns = {};
|
|
2425
|
+
if (results && results.length > 0) {
|
|
2426
|
+
results.forEach((row) => {
|
|
2427
|
+
columns[row] = this.getColumnType(row);
|
|
2428
|
+
});
|
|
2429
|
+
}
|
|
2430
|
+
this.existingColumns = columns;
|
|
2431
|
+
return columns;
|
|
3415
2432
|
}
|
|
3416
2433
|
//---- createTargetTable ----------------------------------------------
|
|
3417
2434
|
/**
|
|
@@ -3501,45 +2518,42 @@ OPTIONS(description="${this.description}")`;
|
|
|
3501
2518
|
* @param {Array} data - Array of objects with records to save
|
|
3502
2519
|
* @returns {Promise}
|
|
3503
2520
|
*/
|
|
3504
|
-
saveData(data) {
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
});
|
|
3523
|
-
}
|
|
2521
|
+
async saveData(data) {
|
|
2522
|
+
await this.checkTableExists();
|
|
2523
|
+
const allColumns = /* @__PURE__ */ new Set();
|
|
2524
|
+
if (data.length > 0) {
|
|
2525
|
+
data.forEach((row) => {
|
|
2526
|
+
Object.keys(row).forEach((column) => allColumns.add(column));
|
|
2527
|
+
});
|
|
2528
|
+
}
|
|
2529
|
+
if (this.config.Fields.value) {
|
|
2530
|
+
this.getSelectedFields().forEach((columnName) => {
|
|
2531
|
+
if (columnName && !allColumns.has(columnName)) {
|
|
2532
|
+
allColumns.add(columnName);
|
|
2533
|
+
if (data.length > 0) {
|
|
2534
|
+
data.forEach((row) => {
|
|
2535
|
+
if (!row[columnName]) {
|
|
2536
|
+
row[columnName] = "";
|
|
2537
|
+
}
|
|
2538
|
+
});
|
|
3524
2539
|
}
|
|
3525
|
-
}
|
|
3526
|
-
}
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
deasync.loopWhile(() => !done);
|
|
2540
|
+
}
|
|
2541
|
+
});
|
|
2542
|
+
}
|
|
2543
|
+
const existingColumnsSet = new Set(Object.keys(this.existingColumns));
|
|
2544
|
+
const newColumns = Array.from(allColumns).filter((column) => !existingColumnsSet.has(column));
|
|
2545
|
+
if (newColumns.length > 0) {
|
|
2546
|
+
await this.addNewColumns(newColumns);
|
|
2547
|
+
}
|
|
2548
|
+
if (data.length === 0) {
|
|
2549
|
+
return;
|
|
2550
|
+
}
|
|
2551
|
+
this.config.logMessage(`Saving ${data.length} records to Athena`);
|
|
2552
|
+
const tempFolder = `${this.config.S3Prefix.value}_temp/${this.uploadSid}`;
|
|
2553
|
+
await this.uploadDataToS3TempFolder(data, tempFolder);
|
|
2554
|
+
const tempTableName = await this.createTempTable(tempFolder, this.uploadSid);
|
|
2555
|
+
await this.mergeDataFromTempTable(tempTableName, this.uploadSid);
|
|
2556
|
+
await this.cleanupTempResources(tempFolder, tempTableName);
|
|
3543
2557
|
}
|
|
3544
2558
|
//---- uploadDataToS3TempFolder ---------------------------------
|
|
3545
2559
|
/**
|
|
@@ -3728,6 +2742,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3728
2742
|
QueryExecutionId: queryExecutionId
|
|
3729
2743
|
};
|
|
3730
2744
|
return this.athenaClient.send(new GetQueryExecutionCommand(params)).then((data) => {
|
|
2745
|
+
var _a;
|
|
3731
2746
|
const state = data.QueryExecution.Status.State;
|
|
3732
2747
|
if (state === "SUCCEEDED") {
|
|
3733
2748
|
return new Promise((resolve) => {
|
|
@@ -3736,6 +2751,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
3736
2751
|
}, 3e3);
|
|
3737
2752
|
});
|
|
3738
2753
|
} else if (state === "FAILED" || state === "CANCELLED") {
|
|
2754
|
+
this.config.logMessage(`Query ${queryExecutionId} ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}. Error: ${((_a = data.QueryExecution.Status.Error) == null ? void 0 : _a.Message) || ""}`);
|
|
3739
2755
|
throw new Error(`Query ${state}: ${data.QueryExecution.Status.StateChangeReason || ""}`);
|
|
3740
2756
|
} else {
|
|
3741
2757
|
return new Promise((resolve) => {
|
|
@@ -3895,12 +2911,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
3895
2911
|
};
|
|
3896
2912
|
})();
|
|
3897
2913
|
const Storages = {
|
|
3898
|
-
GoogleSheets,
|
|
3899
2914
|
GoogleBigQuery,
|
|
3900
2915
|
AwsAthena
|
|
3901
2916
|
};
|
|
3902
2917
|
const XAds = (function() {
|
|
3903
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
2918
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
3904
2919
|
const XAdsHelper = {
|
|
3905
2920
|
/**
|
|
3906
2921
|
* Parse fields string into a structured object
|
|
@@ -4794,34 +3809,34 @@ OPTIONS(description="${this.description}")`;
|
|
|
4794
3809
|
* @param {string} [opts.end_time]
|
|
4795
3810
|
* @returns {Array<Object>}
|
|
4796
3811
|
*/
|
|
4797
|
-
fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
|
|
4798
|
-
|
|
3812
|
+
async fetchData({ nodeName, accountId, fields = [], start_time, end_time }) {
|
|
3813
|
+
await AsyncUtils2.delay(this.config.AdsApiDelay.value * 1e3);
|
|
4799
3814
|
switch (nodeName) {
|
|
4800
3815
|
case "accounts": {
|
|
4801
|
-
const resp = this._getData(`accounts/${accountId}`, "accounts", fields);
|
|
3816
|
+
const resp = await this._getData(`accounts/${accountId}`, "accounts", fields);
|
|
4802
3817
|
return [resp.data];
|
|
4803
3818
|
}
|
|
4804
3819
|
case "campaigns":
|
|
4805
3820
|
case "line_items":
|
|
4806
3821
|
case "promoted_tweets":
|
|
4807
3822
|
case "tweets":
|
|
4808
|
-
return this._catalogFetch({
|
|
3823
|
+
return await this._catalogFetch({
|
|
4809
3824
|
nodeName,
|
|
4810
3825
|
accountId,
|
|
4811
3826
|
fields,
|
|
4812
3827
|
pageSize: this.config.DataMaxCount.value
|
|
4813
3828
|
});
|
|
4814
3829
|
case "cards":
|
|
4815
|
-
return this._catalogFetch({
|
|
3830
|
+
return await this._catalogFetch({
|
|
4816
3831
|
nodeName,
|
|
4817
3832
|
accountId,
|
|
4818
3833
|
fields,
|
|
4819
3834
|
pageSize: this.config.CardsMaxCountPerRequest.value
|
|
4820
3835
|
});
|
|
4821
3836
|
case "cards_all":
|
|
4822
|
-
return this._fetchAllCards(accountId, fields);
|
|
3837
|
+
return await this._fetchAllCards(accountId, fields);
|
|
4823
3838
|
case "stats":
|
|
4824
|
-
return this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
|
|
3839
|
+
return await this._timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time });
|
|
4825
3840
|
default:
|
|
4826
3841
|
throw new ConfigurationError(`Unknown node: ${nodeName}`);
|
|
4827
3842
|
}
|
|
@@ -4857,7 +3872,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
4857
3872
|
/**
|
|
4858
3873
|
* Shared logic for non-time-series endpoints
|
|
4859
3874
|
*/
|
|
4860
|
-
_catalogFetch({ nodeName, accountId, fields, pageSize }) {
|
|
3875
|
+
async _catalogFetch({ nodeName, accountId, fields, pageSize }) {
|
|
4861
3876
|
const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
|
|
4862
3877
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
4863
3878
|
if (missingKeys.length > 0) {
|
|
@@ -4881,7 +3896,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
4881
3896
|
console.log("deleting cached tweets");
|
|
4882
3897
|
this._tweetsCache.delete(accountId);
|
|
4883
3898
|
}
|
|
4884
|
-
let all = this._fetchPages({
|
|
3899
|
+
let all = await this._fetchPages({
|
|
4885
3900
|
accountId,
|
|
4886
3901
|
nodeName,
|
|
4887
3902
|
fields,
|
|
@@ -4905,7 +3920,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
4905
3920
|
/**
|
|
4906
3921
|
* Shared pagination logic
|
|
4907
3922
|
*/
|
|
4908
|
-
_fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
|
|
3923
|
+
async _fetchPages({ accountId, nodeName, fields, extraParams = {}, pageSize }) {
|
|
4909
3924
|
const all = [];
|
|
4910
3925
|
let cursor = null;
|
|
4911
3926
|
const MAX_PAGES = 100;
|
|
@@ -4916,7 +3931,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
4916
3931
|
...extraParams,
|
|
4917
3932
|
...cursor ? { cursor } : {}
|
|
4918
3933
|
};
|
|
4919
|
-
const resp = this._getData(
|
|
3934
|
+
const resp = await this._getData(
|
|
4920
3935
|
`accounts/${accountId}/${nodeName}`,
|
|
4921
3936
|
nodeName,
|
|
4922
3937
|
fields,
|
|
@@ -4937,15 +3952,15 @@ OPTIONS(description="${this.description}")`;
|
|
|
4937
3952
|
* Fetch all cards by first collecting URIs from tweets,
|
|
4938
3953
|
* then calling the cards/all endpoint in chunks.
|
|
4939
3954
|
*/
|
|
4940
|
-
_fetchAllCards(accountId, fields) {
|
|
4941
|
-
const tweets = this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
|
|
3955
|
+
async _fetchAllCards(accountId, fields) {
|
|
3956
|
+
const tweets = await this.fetchData({ nodeName: "tweets", accountId, fields: ["id", "card_uri"] });
|
|
4942
3957
|
const uris = tweets.map((t) => t.card_uri).filter(Boolean);
|
|
4943
3958
|
if (!uris.length) return [];
|
|
4944
3959
|
const all = [];
|
|
4945
3960
|
const chunkSize = this.config.CardsMaxCountPerRequest.value;
|
|
4946
3961
|
for (let i = 0; i < uris.length; i += chunkSize) {
|
|
4947
3962
|
const chunk = uris.slice(i, i + chunkSize);
|
|
4948
|
-
const resp = this._getData(
|
|
3963
|
+
const resp = await this._getData(
|
|
4949
3964
|
`accounts/${accountId}/cards/all`,
|
|
4950
3965
|
"cards_all",
|
|
4951
3966
|
fields,
|
|
@@ -4962,18 +3977,18 @@ OPTIONS(description="${this.description}")`;
|
|
|
4962
3977
|
/**
|
|
4963
3978
|
* Stats are time-series and need flattening of `metrics`
|
|
4964
3979
|
*/
|
|
4965
|
-
_timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
|
|
3980
|
+
async _timeSeriesFetch({ nodeName, accountId, fields, start_time, end_time }) {
|
|
4966
3981
|
const uniqueKeys = this.fieldsSchema[nodeName].uniqueKeys || [];
|
|
4967
3982
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
4968
3983
|
if (missingKeys.length > 0) {
|
|
4969
3984
|
throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
|
|
4970
3985
|
}
|
|
4971
|
-
const promos = this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
|
|
3986
|
+
const promos = await this.fetchData({ nodeName: "promoted_tweets", accountId, fields: ["id"] });
|
|
4972
3987
|
const ids = promos.map((r) => r.id);
|
|
4973
3988
|
if (!ids.length) return [];
|
|
4974
3989
|
const e = new Date(end_time);
|
|
4975
3990
|
e.setDate(e.getDate() + 1);
|
|
4976
|
-
const endStr =
|
|
3991
|
+
const endStr = DateUtils2.formatDate(e);
|
|
4977
3992
|
const result = [];
|
|
4978
3993
|
for (let i = 0; i < ids.length; i += this.config.StatsMaxEntityIds.value) {
|
|
4979
3994
|
const batch = ids.slice(i, i + this.config.StatsMaxEntityIds.value).join(",");
|
|
@@ -4986,7 +4001,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
4986
4001
|
end_time: endStr
|
|
4987
4002
|
};
|
|
4988
4003
|
for (const placement of ["ALL_ON_TWITTER", "PUBLISHER_NETWORK"]) {
|
|
4989
|
-
const raw = this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
|
|
4004
|
+
const raw = await this._rawFetch(`stats/accounts/${accountId}`, { ...common, placement });
|
|
4990
4005
|
const arr = Array.isArray(raw.data) ? raw.data : [raw.data];
|
|
4991
4006
|
arr.forEach((h) => {
|
|
4992
4007
|
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
|
|
@@ -5022,18 +4037,19 @@ OPTIONS(description="${this.description}")`;
|
|
|
5022
4037
|
/**
|
|
5023
4038
|
* Pull JSON from the Ads API (raw, no field-filter).
|
|
5024
4039
|
*/
|
|
5025
|
-
_rawFetch(path, params = {}) {
|
|
4040
|
+
async _rawFetch(path, params = {}) {
|
|
5026
4041
|
const url = `${this.BASE_URL}${this.config.Version.value}/${path}`;
|
|
5027
4042
|
const qs = Object.keys(params).length ? "?" + Object.entries(params).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&") : "";
|
|
5028
4043
|
const finalUrl = url + qs;
|
|
5029
4044
|
const oauth = this._generateOAuthHeader({ method: "GET", url, params });
|
|
5030
|
-
|
|
5031
|
-
const resp = this.urlFetchWithRetry(finalUrl, {
|
|
4045
|
+
await AsyncUtils2.delay(1e3);
|
|
4046
|
+
const resp = await this.urlFetchWithRetry(finalUrl, {
|
|
5032
4047
|
method: "GET",
|
|
5033
4048
|
headers: { Authorization: oauth, "Content-Type": "application/json" },
|
|
5034
4049
|
muteHttpExceptions: true
|
|
5035
4050
|
});
|
|
5036
|
-
|
|
4051
|
+
const text = await resp.getContentText();
|
|
4052
|
+
return JSON.parse(text);
|
|
5037
4053
|
}
|
|
5038
4054
|
/**
|
|
5039
4055
|
* Determines if a X Ads API error is valid for retry
|
|
@@ -5056,8 +4072,8 @@ OPTIONS(description="${this.description}")`;
|
|
|
5056
4072
|
}
|
|
5057
4073
|
return false;
|
|
5058
4074
|
}
|
|
5059
|
-
_getData(path, nodeName, fields, extraParams = {}) {
|
|
5060
|
-
const json = this._rawFetch(path, extraParams);
|
|
4075
|
+
async _getData(path, nodeName, fields, extraParams = {}) {
|
|
4076
|
+
const json = await this._rawFetch(path, extraParams);
|
|
5061
4077
|
if (!json.data) return json;
|
|
5062
4078
|
const arr = Array.isArray(json.data) ? json.data : [json.data];
|
|
5063
4079
|
const filtered = this._filterBySchema(arr, nodeName, fields);
|
|
@@ -5096,7 +4112,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5096
4112
|
const { ConsumerKey, ConsumerSecret, AccessToken, AccessTokenSecret } = this.config;
|
|
5097
4113
|
const oauth = {
|
|
5098
4114
|
oauth_consumer_key: ConsumerKey.value,
|
|
5099
|
-
oauth_nonce:
|
|
4115
|
+
oauth_nonce: CryptoUtils2.getUuid().replace(/-/g, ""),
|
|
5100
4116
|
oauth_signature_method: "HMAC-SHA1",
|
|
5101
4117
|
oauth_timestamp: Math.floor(Date.now() / 1e3),
|
|
5102
4118
|
oauth_token: AccessToken.value,
|
|
@@ -5111,9 +4127,9 @@ OPTIONS(description="${this.description}")`;
|
|
|
5111
4127
|
)
|
|
5112
4128
|
].join("&");
|
|
5113
4129
|
const signingKey = encodeURIComponent(ConsumerSecret.value) + "&" + encodeURIComponent(AccessTokenSecret.value);
|
|
5114
|
-
oauth.oauth_signature =
|
|
5115
|
-
|
|
5116
|
-
|
|
4130
|
+
oauth.oauth_signature = CryptoUtils2.base64Encode(
|
|
4131
|
+
CryptoUtils2.computeHmacSignature(
|
|
4132
|
+
CryptoUtils2.MacAlgorithm.HMAC_SHA_1,
|
|
5117
4133
|
baseString,
|
|
5118
4134
|
signingKey
|
|
5119
4135
|
)
|
|
@@ -5129,7 +4145,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5129
4145
|
}
|
|
5130
4146
|
};
|
|
5131
4147
|
var XAdsConnector = class XAdsConnector extends AbstractConnector2 {
|
|
5132
|
-
constructor(config, source, storageName = "
|
|
4148
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
5133
4149
|
super(config, source, null, runConfig);
|
|
5134
4150
|
this.storageName = storageName;
|
|
5135
4151
|
}
|
|
@@ -5137,12 +4153,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
5137
4153
|
* Main method - entry point for the import process
|
|
5138
4154
|
* Processes all nodes defined in the fields configuration
|
|
5139
4155
|
*/
|
|
5140
|
-
startImportProcess() {
|
|
4156
|
+
async startImportProcess() {
|
|
5141
4157
|
const fields = XAdsHelper.parseFields(this.config.Fields.value);
|
|
5142
4158
|
const accountIds = XAdsHelper.parseAccountIds(this.config.AccountIDs.value);
|
|
5143
4159
|
for (const accountId of accountIds) {
|
|
5144
4160
|
for (const nodeName in fields) {
|
|
5145
|
-
this.processNode({
|
|
4161
|
+
await this.processNode({
|
|
5146
4162
|
nodeName,
|
|
5147
4163
|
accountId,
|
|
5148
4164
|
fields: fields[nodeName] || []
|
|
@@ -5158,15 +4174,15 @@ OPTIONS(description="${this.description}")`;
|
|
|
5158
4174
|
* @param {string} options.accountId - Account ID
|
|
5159
4175
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5160
4176
|
*/
|
|
5161
|
-
processNode({ nodeName, accountId, fields }) {
|
|
4177
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
5162
4178
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
5163
|
-
this.processTimeSeriesNode({
|
|
4179
|
+
await this.processTimeSeriesNode({
|
|
5164
4180
|
nodeName,
|
|
5165
4181
|
accountId,
|
|
5166
4182
|
fields
|
|
5167
4183
|
});
|
|
5168
4184
|
} else {
|
|
5169
|
-
this.processCatalogNode({
|
|
4185
|
+
await this.processCatalogNode({
|
|
5170
4186
|
nodeName,
|
|
5171
4187
|
accountId,
|
|
5172
4188
|
fields
|
|
@@ -5181,7 +4197,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5181
4197
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5182
4198
|
* @param {Object} options.storage - Storage instance
|
|
5183
4199
|
*/
|
|
5184
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
4200
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
5185
4201
|
var _a;
|
|
5186
4202
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
5187
4203
|
if (daysToFetch <= 0) {
|
|
@@ -5191,12 +4207,13 @@ OPTIONS(description="${this.description}")`;
|
|
|
5191
4207
|
for (let i = 0; i < daysToFetch; i++) {
|
|
5192
4208
|
const currentDate = new Date(startDate);
|
|
5193
4209
|
currentDate.setDate(currentDate.getDate() + i);
|
|
5194
|
-
const formattedDate =
|
|
5195
|
-
const data = this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
|
|
4210
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
4211
|
+
const data = await this.source.fetchData({ nodeName, accountId, start_time: formattedDate, end_time: formattedDate, fields });
|
|
5196
4212
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
|
|
5197
4213
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
5198
4214
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
5199
|
-
this.getStorageByNode(nodeName)
|
|
4215
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
4216
|
+
await storage.saveData(preparedData);
|
|
5200
4217
|
}
|
|
5201
4218
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
5202
4219
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -5211,13 +4228,14 @@ OPTIONS(description="${this.description}")`;
|
|
|
5211
4228
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
5212
4229
|
* @param {Object} options.storage - Storage instance
|
|
5213
4230
|
*/
|
|
5214
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
4231
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
5215
4232
|
var _a;
|
|
5216
|
-
const data = this.source.fetchData({ nodeName, accountId, fields });
|
|
4233
|
+
const data = await this.source.fetchData({ nodeName, accountId, fields });
|
|
5217
4234
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
|
|
5218
4235
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
5219
4236
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
5220
|
-
this.getStorageByNode(nodeName)
|
|
4237
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
4238
|
+
await storage.saveData(preparedData);
|
|
5221
4239
|
}
|
|
5222
4240
|
}
|
|
5223
4241
|
/**
|
|
@@ -5225,7 +4243,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5225
4243
|
* @param {string} nodeName - Name of the node
|
|
5226
4244
|
* @returns {Object} Storage instance
|
|
5227
4245
|
*/
|
|
5228
|
-
getStorageByNode(nodeName) {
|
|
4246
|
+
async getStorageByNode(nodeName) {
|
|
5229
4247
|
if (!("storages" in this)) {
|
|
5230
4248
|
this.storages = {};
|
|
5231
4249
|
}
|
|
@@ -5243,6 +4261,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5243
4261
|
this.source.fieldsSchema[nodeName].fields,
|
|
5244
4262
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
5245
4263
|
);
|
|
4264
|
+
await this.storages[nodeName].init();
|
|
5246
4265
|
}
|
|
5247
4266
|
return this.storages[nodeName];
|
|
5248
4267
|
}
|
|
@@ -5259,7 +4278,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5259
4278
|
};
|
|
5260
4279
|
})();
|
|
5261
4280
|
const TikTokAds = (function() {
|
|
5262
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
4281
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
5263
4282
|
class TiktokMarketingApiProvider {
|
|
5264
4283
|
constructor(appId, accessToken, appSecret, isSandbox = false) {
|
|
5265
4284
|
this.BASE_URL = "https://business-api.tiktok.com/open_api/";
|
|
@@ -5283,7 +4302,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5283
4302
|
getApiVersion() {
|
|
5284
4303
|
return this.API_VERSION;
|
|
5285
4304
|
}
|
|
5286
|
-
makeRequest(options) {
|
|
4305
|
+
async makeRequest(options) {
|
|
5287
4306
|
const { url, method, data } = options;
|
|
5288
4307
|
const headers = {
|
|
5289
4308
|
"Access-Token": this.accessToken,
|
|
@@ -5292,21 +4311,21 @@ OPTIONS(description="${this.description}")`;
|
|
|
5292
4311
|
let backoff = this.INITIAL_BACKOFF;
|
|
5293
4312
|
for (let retries = 0; retries < this.MAX_RETRIES; retries++) {
|
|
5294
4313
|
try {
|
|
5295
|
-
const response =
|
|
4314
|
+
const response = await HttpUtils2.fetch(url, {
|
|
5296
4315
|
method,
|
|
5297
4316
|
headers,
|
|
5298
|
-
body: data ? JSON.stringify(data) : null
|
|
5299
|
-
muteHttpExceptions: true
|
|
4317
|
+
body: data ? JSON.stringify(data) : null
|
|
5300
4318
|
});
|
|
5301
4319
|
const responseCode = response.getResponseCode();
|
|
4320
|
+
const text = await response.getContentText();
|
|
5302
4321
|
if (responseCode !== this.SUCCESS_RESPONSE_CODE) {
|
|
5303
|
-
throw new Error(`TikTok API error: ${
|
|
4322
|
+
throw new Error(`TikTok API error: ${text}`);
|
|
5304
4323
|
}
|
|
5305
|
-
const jsonData = JSON.parse(
|
|
4324
|
+
const jsonData = JSON.parse(text);
|
|
5306
4325
|
if (jsonData.code !== this.SUCCESS_CODE) {
|
|
5307
4326
|
if (jsonData.code === this.RATE_LIMIT_CODE) {
|
|
5308
4327
|
console.error("TikTok Marketing API rate limit exceeded. Retrying...");
|
|
5309
|
-
|
|
4328
|
+
await AsyncUtils2.delay(backoff);
|
|
5310
4329
|
backoff *= 2;
|
|
5311
4330
|
continue;
|
|
5312
4331
|
}
|
|
@@ -5315,7 +4334,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5315
4334
|
return jsonData;
|
|
5316
4335
|
} catch (error) {
|
|
5317
4336
|
if (retries < this.MAX_RETRIES - 1 && error.message.includes("rate limit")) {
|
|
5318
|
-
|
|
4337
|
+
await AsyncUtils2.delay(backoff);
|
|
5319
4338
|
backoff *= 2;
|
|
5320
4339
|
} else {
|
|
5321
4340
|
throw error;
|
|
@@ -5323,7 +4342,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5323
4342
|
}
|
|
5324
4343
|
}
|
|
5325
4344
|
}
|
|
5326
|
-
handlePagination(endpoint, params = {}) {
|
|
4345
|
+
async handlePagination(endpoint, params = {}) {
|
|
5327
4346
|
let allData = [];
|
|
5328
4347
|
let page = 1;
|
|
5329
4348
|
let hasMorePages = true;
|
|
@@ -5331,7 +4350,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5331
4350
|
while (hasMorePages) {
|
|
5332
4351
|
const paginatedParams = { ...params, page, page_size: pageSize };
|
|
5333
4352
|
const url = this.buildUrl(endpoint, paginatedParams);
|
|
5334
|
-
const response = this.makeRequest({ url, method: "GET" });
|
|
4353
|
+
const response = await this.makeRequest({ url, method: "GET" });
|
|
5335
4354
|
const pageData = response.data.list || [];
|
|
5336
4355
|
allData = allData.concat(pageData);
|
|
5337
4356
|
const total = response.data.page_info ? response.data.page_info.total_number : 0;
|
|
@@ -5339,7 +4358,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
5339
4358
|
hasMorePages = currentCount < total && pageData.length > 0;
|
|
5340
4359
|
page++;
|
|
5341
4360
|
if (hasMorePages) {
|
|
5342
|
-
|
|
4361
|
+
await AsyncUtils2.delay(100);
|
|
5343
4362
|
}
|
|
5344
4363
|
}
|
|
5345
4364
|
return allData;
|
|
@@ -6422,7 +5441,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6422
5441
|
* @param {Date} endDate - End date for time-series data (optional)
|
|
6423
5442
|
* @return {array} - Array of data objects
|
|
6424
5443
|
*/
|
|
6425
|
-
fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
|
|
5444
|
+
async fetchData(nodeName, advertiserId, fields, startDate = null, endDate = null) {
|
|
6426
5445
|
if (!this.fieldsSchema[nodeName]) {
|
|
6427
5446
|
throw new Error(`Unknown node type: ${nodeName}`);
|
|
6428
5447
|
}
|
|
@@ -6443,8 +5462,8 @@ OPTIONS(description="${this.description}")`;
|
|
|
6443
5462
|
let formattedStartDate = null;
|
|
6444
5463
|
let formattedEndDate = null;
|
|
6445
5464
|
if (startDate) {
|
|
6446
|
-
formattedStartDate =
|
|
6447
|
-
formattedEndDate = endDate ?
|
|
5465
|
+
formattedStartDate = DateUtils2.formatDate(startDate);
|
|
5466
|
+
formattedEndDate = endDate ? DateUtils2.formatDate(endDate) : formattedStartDate;
|
|
6448
5467
|
}
|
|
6449
5468
|
let filtering = null;
|
|
6450
5469
|
if (this.config.IncludeDeleted && this.config.IncludeDeleted.value) {
|
|
@@ -6666,11 +5685,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
6666
5685
|
}
|
|
6667
5686
|
};
|
|
6668
5687
|
var TikTokAdsConnector = class TikTokAdsConnector extends AbstractConnector2 {
|
|
6669
|
-
constructor(config, source, storageName = "
|
|
5688
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
6670
5689
|
super(config, source, null, runConfig);
|
|
6671
5690
|
this.storageName = storageName;
|
|
6672
5691
|
}
|
|
6673
|
-
startImportProcess() {
|
|
5692
|
+
async startImportProcess() {
|
|
6674
5693
|
try {
|
|
6675
5694
|
let advertiserIds = TikTokAdsHelper.parseAdvertiserIds(this.config.AdvertiserIDs.value || "");
|
|
6676
5695
|
if (!advertiserIds || advertiserIds.length === 0) {
|
|
@@ -6694,7 +5713,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6694
5713
|
}
|
|
6695
5714
|
}
|
|
6696
5715
|
if (Object.keys(catalogNodes).length > 0) {
|
|
6697
|
-
this.importCatalogData(catalogNodes, advertiserIds);
|
|
5716
|
+
await this.importCatalogData(catalogNodes, advertiserIds);
|
|
6698
5717
|
}
|
|
6699
5718
|
if (Object.keys(timeSeriesNodes).length > 0) {
|
|
6700
5719
|
try {
|
|
@@ -6703,7 +5722,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6703
5722
|
this.config.logMessage("There is nothing to import in this data range");
|
|
6704
5723
|
return;
|
|
6705
5724
|
}
|
|
6706
|
-
this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
|
|
5725
|
+
await this.startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch);
|
|
6707
5726
|
} catch (error) {
|
|
6708
5727
|
this.config.logMessage(`Error determining date range: ${error.message}`);
|
|
6709
5728
|
console.error(error.stack);
|
|
@@ -6722,35 +5741,36 @@ OPTIONS(description="${this.description}")`;
|
|
|
6722
5741
|
}
|
|
6723
5742
|
/**
|
|
6724
5743
|
* Imports all catalog (non-time-series) data types
|
|
6725
|
-
*
|
|
5744
|
+
*
|
|
6726
5745
|
* @param {object} catalogNodes - Object with node names as keys and field arrays as values
|
|
6727
5746
|
* @param {array} advertiserIds - List of advertiser IDs to fetch data for
|
|
6728
5747
|
*/
|
|
6729
|
-
importCatalogData(catalogNodes, advertiserIds) {
|
|
5748
|
+
async importCatalogData(catalogNodes, advertiserIds) {
|
|
6730
5749
|
for (var nodeName in catalogNodes) {
|
|
6731
5750
|
this.config.logMessage(`Starting import for ${nodeName} data...`);
|
|
6732
|
-
this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
|
|
5751
|
+
await this.startImportProcessOfCatalogData(nodeName, advertiserIds, catalogNodes[nodeName]);
|
|
6733
5752
|
}
|
|
6734
5753
|
}
|
|
6735
5754
|
/**
|
|
6736
5755
|
* Imports catalog (not time series) data
|
|
6737
|
-
*
|
|
5756
|
+
*
|
|
6738
5757
|
* @param {string} nodeName - Node name
|
|
6739
5758
|
* @param {array} advertiserIds - List of advertiser IDs
|
|
6740
5759
|
* @param {array} fields - List of fields
|
|
6741
5760
|
*/
|
|
6742
|
-
startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
|
|
5761
|
+
async startImportProcessOfCatalogData(nodeName, advertiserIds, fields) {
|
|
6743
5762
|
var _a;
|
|
6744
5763
|
this.config.logMessage(`Fetching all available fields for ${nodeName}`);
|
|
6745
5764
|
for (var i in advertiserIds) {
|
|
6746
5765
|
let advertiserId = advertiserIds[i];
|
|
6747
5766
|
try {
|
|
6748
|
-
let data = this.source.fetchData(nodeName, advertiserId, fields);
|
|
5767
|
+
let data = await this.source.fetchData(nodeName, advertiserId, fields);
|
|
6749
5768
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for advertiser ${advertiserId}` : `No records have been fetched`);
|
|
6750
5769
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
6751
5770
|
try {
|
|
6752
5771
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
6753
|
-
this.getStorageByNode(nodeName)
|
|
5772
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
5773
|
+
await storage.saveData(preparedData);
|
|
6754
5774
|
} catch (storageError) {
|
|
6755
5775
|
this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
|
|
6756
5776
|
console.error(`Error details: ${storageError.stack}`);
|
|
@@ -6764,29 +5784,30 @@ OPTIONS(description="${this.description}")`;
|
|
|
6764
5784
|
}
|
|
6765
5785
|
/**
|
|
6766
5786
|
* Imports time series data
|
|
6767
|
-
*
|
|
5787
|
+
*
|
|
6768
5788
|
* @param {array} advertiserIds - List of advertiser IDs
|
|
6769
5789
|
* @param {object} timeSeriesNodes - Object of properties, each is array of fields
|
|
6770
5790
|
* @param {Date} startDate - Start date
|
|
6771
5791
|
* @param {number} daysToFetch - Number of days to fetch
|
|
6772
5792
|
*/
|
|
6773
|
-
startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
|
|
5793
|
+
async startImportProcessOfTimeSeriesData(advertiserIds, timeSeriesNodes, startDate, daysToFetch) {
|
|
6774
5794
|
var _a;
|
|
6775
5795
|
for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
6776
5796
|
const currentDate = new Date(startDate);
|
|
6777
5797
|
currentDate.setDate(currentDate.getDate() + daysShift);
|
|
6778
|
-
const formattedDate =
|
|
5798
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
6779
5799
|
this.config.logMessage(`Processing data for date: ${formattedDate}`);
|
|
6780
5800
|
for (let advertiserId of advertiserIds) {
|
|
6781
5801
|
for (var nodeName in timeSeriesNodes) {
|
|
6782
5802
|
try {
|
|
6783
5803
|
this.config.logMessage(`Start importing data for ${formattedDate}: ${advertiserId}/${nodeName}`);
|
|
6784
|
-
let data = this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
|
|
5804
|
+
let data = await this.source.fetchData(nodeName, advertiserId, timeSeriesNodes[nodeName], currentDate);
|
|
6785
5805
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
6786
5806
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
6787
5807
|
try {
|
|
6788
5808
|
const preparedData = data.length ? this.addMissingFieldsToData(data, timeSeriesNodes[nodeName]) : data;
|
|
6789
|
-
this.getStorageByNode(nodeName)
|
|
5809
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
5810
|
+
await storage.saveData(preparedData);
|
|
6790
5811
|
} catch (storageError) {
|
|
6791
5812
|
this.config.logMessage(`Error saving data to storage: ${storageError.message}`);
|
|
6792
5813
|
console.error(`Error details: ${storageError.stack}`);
|
|
@@ -6810,7 +5831,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6810
5831
|
* @param {array} requestedFields - List of requested fields
|
|
6811
5832
|
* @return {AbstractStorage} - Storage instance
|
|
6812
5833
|
*/
|
|
6813
|
-
getStorageByNode(nodeName) {
|
|
5834
|
+
async getStorageByNode(nodeName) {
|
|
6814
5835
|
if (!("storages" in this)) {
|
|
6815
5836
|
this.storages = {};
|
|
6816
5837
|
}
|
|
@@ -6832,6 +5853,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6832
5853
|
this.source.fieldsSchema[nodeName]["fields"] || {},
|
|
6833
5854
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
6834
5855
|
);
|
|
5856
|
+
await this.storages[nodeName].init();
|
|
6835
5857
|
}
|
|
6836
5858
|
return this.storages[nodeName];
|
|
6837
5859
|
}
|
|
@@ -6849,35 +5871,9 @@ OPTIONS(description="${this.description}")`;
|
|
|
6849
5871
|
this.config.logMessage(`Cleaning up data older than ${keepDays} days...`);
|
|
6850
5872
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
6851
5873
|
cutoffDate.setDate(cutoffDate.getDate() - keepDays);
|
|
6852
|
-
|
|
5874
|
+
DateUtils2.formatDate(cutoffDate);
|
|
6853
5875
|
for (var nodeName in this.source.fieldsSchema) {
|
|
6854
|
-
if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"]))
|
|
6855
|
-
try {
|
|
6856
|
-
const storage = this.getStorageByNode(nodeName);
|
|
6857
|
-
if (storage instanceof GoogleSheetsStorage) {
|
|
6858
|
-
const dateField = this.source.fieldsSchema[nodeName]["fields"]["date_start"] ? "date_start" : "stat_time_day";
|
|
6859
|
-
let keysToDelete = [];
|
|
6860
|
-
for (const uniqueKey in storage.values) {
|
|
6861
|
-
const record = storage.values[uniqueKey];
|
|
6862
|
-
const rowDate = record[dateField];
|
|
6863
|
-
if (rowDate && rowDate < cutoffDate) {
|
|
6864
|
-
keysToDelete.push(uniqueKey);
|
|
6865
|
-
}
|
|
6866
|
-
}
|
|
6867
|
-
let deletedCount = 0;
|
|
6868
|
-
for (const key of keysToDelete) {
|
|
6869
|
-
storage.deleteRecord(key);
|
|
6870
|
-
deletedCount++;
|
|
6871
|
-
}
|
|
6872
|
-
if (deletedCount > 0) {
|
|
6873
|
-
this.config.logMessage(`Deleted ${deletedCount} rows from ${nodeName} that were older than ${formattedCutoffDate}`);
|
|
6874
|
-
}
|
|
6875
|
-
}
|
|
6876
|
-
} catch (error) {
|
|
6877
|
-
this.config.logMessage(`Error cleaning up old data from ${nodeName}: ${error.message}`);
|
|
6878
|
-
console.error(`Error details: ${error.stack}`);
|
|
6879
|
-
}
|
|
6880
|
-
}
|
|
5876
|
+
if ("fields" in this.source.fieldsSchema[nodeName] && ("date_start" in this.source.fieldsSchema[nodeName]["fields"] || "stat_time_day" in this.source.fieldsSchema[nodeName]["fields"])) ;
|
|
6881
5877
|
}
|
|
6882
5878
|
}
|
|
6883
5879
|
};
|
|
@@ -6894,7 +5890,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
6894
5890
|
};
|
|
6895
5891
|
})();
|
|
6896
5892
|
const RedditAds = (function() {
|
|
6897
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
5893
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
6898
5894
|
const RedditAdsHelper = {
|
|
6899
5895
|
/**
|
|
6900
5896
|
* Parse fields string into a structured object
|
|
@@ -9930,7 +8926,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
9930
8926
|
* @param {Date|null} startDate - The start date for data fetching.
|
|
9931
8927
|
* @returns {Array} An array of data records.
|
|
9932
8928
|
*/
|
|
9933
|
-
fetchData(nodeName, accountId, fields, startDate = null) {
|
|
8929
|
+
async fetchData(nodeName, accountId, fields, startDate = null) {
|
|
9934
8930
|
var _a;
|
|
9935
8931
|
console.log(`Fetching data from ${nodeName}/${accountId} for ${startDate}`);
|
|
9936
8932
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -9938,7 +8934,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
9938
8934
|
if (missingKeys.length > 0) {
|
|
9939
8935
|
throw new Error(`Missing required unique fields for endpoint '${nodeName}'. Missing fields: ${missingKeys.join(", ")}`);
|
|
9940
8936
|
}
|
|
9941
|
-
const tokenResponse = this.getRedditAccessToken(
|
|
8937
|
+
const tokenResponse = await this.getRedditAccessToken(
|
|
9942
8938
|
this.config.ClientId.value,
|
|
9943
8939
|
this.config.ClientSecret.value,
|
|
9944
8940
|
this.config.RedirectUri.value,
|
|
@@ -9948,7 +8944,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
9948
8944
|
this.config.AccessToken.value = tokenResponse.accessToken;
|
|
9949
8945
|
}
|
|
9950
8946
|
const baseUrl = "https://ads-api.reddit.com/api/v3/";
|
|
9951
|
-
let formattedDate = startDate ?
|
|
8947
|
+
let formattedDate = startDate ? DateUtils2.formatDate(startDate) : null;
|
|
9952
8948
|
let headers = {
|
|
9953
8949
|
"Accept": "application/json",
|
|
9954
8950
|
"User-Agent": this.config.UserAgent.value,
|
|
@@ -9974,8 +8970,9 @@ OPTIONS(description="${this.description}")`;
|
|
|
9974
8970
|
let nextPageURL = finalUrl;
|
|
9975
8971
|
while (nextPageURL) {
|
|
9976
8972
|
try {
|
|
9977
|
-
const response = this.urlFetchWithRetry(nextPageURL, options);
|
|
9978
|
-
const
|
|
8973
|
+
const response = await this.urlFetchWithRetry(nextPageURL, options);
|
|
8974
|
+
const text = await response.getContentText();
|
|
8975
|
+
const jsonData = JSON.parse(text);
|
|
9979
8976
|
if ("data" in jsonData) {
|
|
9980
8977
|
nextPageURL = jsonData.pagination ? jsonData.pagination.next_url : null;
|
|
9981
8978
|
if (jsonData && jsonData.data && jsonData.data.metrics) {
|
|
@@ -9992,7 +8989,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
9992
8989
|
}
|
|
9993
8990
|
} catch (error) {
|
|
9994
8991
|
if (error.statusCode === HTTP_STATUS2.UNAUTHORIZED) {
|
|
9995
|
-
const newTokenResponse = this.getRedditAccessToken(
|
|
8992
|
+
const newTokenResponse = await this.getRedditAccessToken(
|
|
9996
8993
|
this.config.ClientId.value,
|
|
9997
8994
|
this.config.ClientSecret.value,
|
|
9998
8995
|
this.config.RedirectUri.value,
|
|
@@ -10019,12 +9016,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
10019
9016
|
* @param {string} refreshToken - The refresh token.
|
|
10020
9017
|
* @returns {Object} An object with a success flag and either the access token or an error message.
|
|
10021
9018
|
*/
|
|
10022
|
-
getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
|
|
9019
|
+
async getRedditAccessToken(clientId, clientSecret, redirectUri, refreshToken) {
|
|
10023
9020
|
const url = "https://www.reddit.com/api/v1/access_token";
|
|
10024
9021
|
const headers = {
|
|
10025
9022
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
10026
9023
|
"User-Agent": this.config.UserAgent.value,
|
|
10027
|
-
"Authorization": "Basic " +
|
|
9024
|
+
"Authorization": "Basic " + CryptoUtils2.base64Encode(clientId + ":" + clientSecret)
|
|
10028
9025
|
};
|
|
10029
9026
|
const payload = {
|
|
10030
9027
|
"grant_type": "refresh_token",
|
|
@@ -10040,8 +9037,8 @@ OPTIONS(description="${this.description}")`;
|
|
|
10040
9037
|
muteHttpExceptions: true
|
|
10041
9038
|
};
|
|
10042
9039
|
try {
|
|
10043
|
-
const response =
|
|
10044
|
-
const result = response.getContentText();
|
|
9040
|
+
const response = await HttpUtils2.fetch(url, options);
|
|
9041
|
+
const result = await response.getContentText();
|
|
10045
9042
|
const json = JSON.parse(result);
|
|
10046
9043
|
if (json.error) {
|
|
10047
9044
|
return { success: false, message: json.error };
|
|
@@ -10426,7 +9423,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10426
9423
|
}
|
|
10427
9424
|
};
|
|
10428
9425
|
var RedditAdsConnector = class RedditAdsConnector extends AbstractConnector2 {
|
|
10429
|
-
constructor(config, source, storageName = "
|
|
9426
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10430
9427
|
super(config, source, null, runConfig);
|
|
10431
9428
|
this.storageName = storageName;
|
|
10432
9429
|
}
|
|
@@ -10434,12 +9431,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
10434
9431
|
* Main method - entry point for the import process
|
|
10435
9432
|
* Processes all nodes defined in the fields configuration
|
|
10436
9433
|
*/
|
|
10437
|
-
startImportProcess() {
|
|
9434
|
+
async startImportProcess() {
|
|
10438
9435
|
const fields = RedditAdsHelper.parseFields(this.config.Fields.value);
|
|
10439
9436
|
const accountIds = RedditAdsHelper.parseAccountIds(this.config.AccountIDs.value);
|
|
10440
9437
|
for (const accountId of accountIds) {
|
|
10441
9438
|
for (const nodeName in fields) {
|
|
10442
|
-
this.processNode({
|
|
9439
|
+
await this.processNode({
|
|
10443
9440
|
nodeName,
|
|
10444
9441
|
accountId,
|
|
10445
9442
|
fields: fields[nodeName] || []
|
|
@@ -10454,15 +9451,15 @@ OPTIONS(description="${this.description}")`;
|
|
|
10454
9451
|
* @param {string} options.accountId - Account ID
|
|
10455
9452
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10456
9453
|
*/
|
|
10457
|
-
processNode({ nodeName, accountId, fields }) {
|
|
9454
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
10458
9455
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
10459
|
-
this.processTimeSeriesNode({
|
|
9456
|
+
await this.processTimeSeriesNode({
|
|
10460
9457
|
nodeName,
|
|
10461
9458
|
accountId,
|
|
10462
9459
|
fields
|
|
10463
9460
|
});
|
|
10464
9461
|
} else {
|
|
10465
|
-
this.processCatalogNode({
|
|
9462
|
+
await this.processCatalogNode({
|
|
10466
9463
|
nodeName,
|
|
10467
9464
|
accountId,
|
|
10468
9465
|
fields
|
|
@@ -10477,7 +9474,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10477
9474
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10478
9475
|
* @param {Object} options.storage - Storage instance
|
|
10479
9476
|
*/
|
|
10480
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
9477
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
10481
9478
|
var _a;
|
|
10482
9479
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
10483
9480
|
if (daysToFetch <= 0) {
|
|
@@ -10487,13 +9484,14 @@ OPTIONS(description="${this.description}")`;
|
|
|
10487
9484
|
for (let i = 0; i < daysToFetch; i++) {
|
|
10488
9485
|
const currentDate = new Date(startDate);
|
|
10489
9486
|
currentDate.setDate(currentDate.getDate() + i);
|
|
10490
|
-
const formattedDate =
|
|
9487
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
10491
9488
|
this.config.logMessage(`Start importing data for ${formattedDate}: ${accountId}/${nodeName}`);
|
|
10492
|
-
const data = this.source.fetchData(nodeName, accountId, fields, currentDate);
|
|
9489
|
+
const data = await this.source.fetchData(nodeName, accountId, fields, currentDate);
|
|
10493
9490
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
10494
9491
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10495
9492
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10496
|
-
this.getStorageByNode(nodeName)
|
|
9493
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
9494
|
+
await storage.saveData(preparedData);
|
|
10497
9495
|
}
|
|
10498
9496
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
10499
9497
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -10508,13 +9506,14 @@ OPTIONS(description="${this.description}")`;
|
|
|
10508
9506
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10509
9507
|
* @param {Object} options.storage - Storage instance
|
|
10510
9508
|
*/
|
|
10511
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
9509
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
10512
9510
|
var _a;
|
|
10513
|
-
const data = this.source.fetchData(nodeName, accountId, fields);
|
|
9511
|
+
const data = await this.source.fetchData(nodeName, accountId, fields);
|
|
10514
9512
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for account ${accountId}` : `No records have been fetched`);
|
|
10515
9513
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10516
9514
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10517
|
-
this.getStorageByNode(nodeName)
|
|
9515
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
9516
|
+
await storage.saveData(preparedData);
|
|
10518
9517
|
}
|
|
10519
9518
|
}
|
|
10520
9519
|
/**
|
|
@@ -10522,7 +9521,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10522
9521
|
* @param {string} nodeName - Name of the node
|
|
10523
9522
|
* @returns {Object} Storage instance
|
|
10524
9523
|
*/
|
|
10525
|
-
getStorageByNode(nodeName) {
|
|
9524
|
+
async getStorageByNode(nodeName) {
|
|
10526
9525
|
if (!("storages" in this)) {
|
|
10527
9526
|
this.storages = {};
|
|
10528
9527
|
}
|
|
@@ -10540,6 +9539,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10540
9539
|
this.source.fieldsSchema[nodeName].fields,
|
|
10541
9540
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
10542
9541
|
);
|
|
9542
|
+
await this.storages[nodeName].init();
|
|
10543
9543
|
}
|
|
10544
9544
|
return this.storages[nodeName];
|
|
10545
9545
|
}
|
|
@@ -10556,7 +9556,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10556
9556
|
};
|
|
10557
9557
|
})();
|
|
10558
9558
|
const OpenHolidays = (function() {
|
|
10559
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
9559
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
10560
9560
|
var publicHolidaysFields = {
|
|
10561
9561
|
id: {
|
|
10562
9562
|
type: "string",
|
|
@@ -10664,10 +9664,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
10664
9664
|
* @param {string} [opts.end_time]
|
|
10665
9665
|
* @returns {Array<Object>}
|
|
10666
9666
|
*/
|
|
10667
|
-
fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
9667
|
+
async fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
10668
9668
|
switch (nodeName) {
|
|
10669
9669
|
case "publicHolidays":
|
|
10670
|
-
return this._fetchPublicHolidays({ fields, start_time, end_time });
|
|
9670
|
+
return await this._fetchPublicHolidays({ fields, start_time, end_time });
|
|
10671
9671
|
default:
|
|
10672
9672
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
10673
9673
|
}
|
|
@@ -10680,10 +9680,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
10680
9680
|
* @param {string} options.end_time - End date for data fetch (YYYY-MM-DD format)
|
|
10681
9681
|
* @returns {Array} Array of holiday data
|
|
10682
9682
|
*/
|
|
10683
|
-
_fetchPublicHolidays({ fields, start_time, end_time }) {
|
|
9683
|
+
async _fetchPublicHolidays({ fields, start_time, end_time }) {
|
|
10684
9684
|
let countryIsoCode = this.config.countryIsoCode.value;
|
|
10685
9685
|
let languageIsoCode = this.config.languageIsoCode.value;
|
|
10686
|
-
const holidays = this.makeRequest({
|
|
9686
|
+
const holidays = await this.makeRequest({
|
|
10687
9687
|
endpoint: `PublicHolidays?countryIsoCode=${countryIsoCode}&languageIsoCode=${languageIsoCode}&validFrom=${start_time}&validTo=${end_time}`
|
|
10688
9688
|
});
|
|
10689
9689
|
if (!holidays || !holidays.length) {
|
|
@@ -10710,12 +9710,13 @@ OPTIONS(description="${this.description}")`;
|
|
|
10710
9710
|
* @param {string} options.endpoint - API endpoint path (e.g., "PublicHolidays?countryIsoCode=CH&...")
|
|
10711
9711
|
* @returns {Object} - API response parsed from JSON
|
|
10712
9712
|
*/
|
|
10713
|
-
makeRequest({ endpoint }) {
|
|
9713
|
+
async makeRequest({ endpoint }) {
|
|
10714
9714
|
const baseUrl = "https://openholidaysapi.org/";
|
|
10715
9715
|
const url = `${baseUrl}${endpoint}`;
|
|
10716
9716
|
console.log(`OpenHolidays API Request URL:`, url);
|
|
10717
|
-
const response =
|
|
10718
|
-
const
|
|
9717
|
+
const response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
9718
|
+
const text = await response.getContentText();
|
|
9719
|
+
const result = JSON.parse(text);
|
|
10719
9720
|
return result;
|
|
10720
9721
|
}
|
|
10721
9722
|
/**
|
|
@@ -10741,7 +9742,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10741
9742
|
}
|
|
10742
9743
|
};
|
|
10743
9744
|
var OpenHolidaysConnector = class OpenHolidaysConnector extends AbstractConnector2 {
|
|
10744
|
-
constructor(config, source, storageName = "
|
|
9745
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10745
9746
|
super(config, source, null, runConfig);
|
|
10746
9747
|
this.storageName = storageName;
|
|
10747
9748
|
}
|
|
@@ -10749,10 +9750,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
10749
9750
|
* Main method - entry point for the import process
|
|
10750
9751
|
* Processes all nodes defined in the fields configuration
|
|
10751
9752
|
*/
|
|
10752
|
-
startImportProcess() {
|
|
9753
|
+
async startImportProcess() {
|
|
10753
9754
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
10754
9755
|
for (const nodeName in fields) {
|
|
10755
|
-
this.processNode({
|
|
9756
|
+
await this.processNode({
|
|
10756
9757
|
nodeName,
|
|
10757
9758
|
fields: fields[nodeName] || []
|
|
10758
9759
|
});
|
|
@@ -10764,12 +9765,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
10764
9765
|
* @param {string} options.nodeName - Name of the node to process
|
|
10765
9766
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10766
9767
|
*/
|
|
10767
|
-
processNode({ nodeName, fields }) {
|
|
9768
|
+
async processNode({ nodeName, fields }) {
|
|
10768
9769
|
const storage = this.getStorageByNode(nodeName);
|
|
10769
9770
|
if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
|
|
10770
|
-
this.processTimeSeriesNode({ nodeName, fields, storage });
|
|
9771
|
+
await this.processTimeSeriesNode({ nodeName, fields, storage });
|
|
10771
9772
|
} else {
|
|
10772
|
-
this.processCatalogNode({ nodeName, fields, storage });
|
|
9773
|
+
await this.processCatalogNode({ nodeName, fields, storage });
|
|
10773
9774
|
}
|
|
10774
9775
|
}
|
|
10775
9776
|
/**
|
|
@@ -10779,14 +9780,14 @@ OPTIONS(description="${this.description}")`;
|
|
|
10779
9780
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10780
9781
|
* @param {Object} options.storage - Storage instance
|
|
10781
9782
|
*/
|
|
10782
|
-
processTimeSeriesNode({ nodeName, fields, storage }) {
|
|
9783
|
+
async processTimeSeriesNode({ nodeName, fields, storage }) {
|
|
10783
9784
|
var _a;
|
|
10784
9785
|
const dateRange = this.prepareDateRange();
|
|
10785
9786
|
if (!dateRange) {
|
|
10786
9787
|
console.log("No date range available for time series data");
|
|
10787
9788
|
return;
|
|
10788
9789
|
}
|
|
10789
|
-
const data = this.source.fetchData({
|
|
9790
|
+
const data = await this.source.fetchData({
|
|
10790
9791
|
nodeName,
|
|
10791
9792
|
start_time: dateRange.startDate,
|
|
10792
9793
|
end_time: dateRange.endDate,
|
|
@@ -10795,7 +9796,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10795
9796
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
|
|
10796
9797
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
10797
9798
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
10798
|
-
storage.saveData(preparedData);
|
|
9799
|
+
await storage.saveData(preparedData);
|
|
10799
9800
|
}
|
|
10800
9801
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
10801
9802
|
this.config.updateLastRequstedDate(new Date(dateRange.endDate));
|
|
@@ -10808,7 +9809,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10808
9809
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
10809
9810
|
* @param {Object} options.storage - Storage instance
|
|
10810
9811
|
*/
|
|
10811
|
-
processCatalogNode({ nodeName, fields, storage }) {
|
|
9812
|
+
async processCatalogNode({ nodeName, fields, storage }) {
|
|
10812
9813
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
10813
9814
|
}
|
|
10814
9815
|
/**
|
|
@@ -10816,7 +9817,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10816
9817
|
* @param {string} nodeName - Name of the node
|
|
10817
9818
|
* @returns {Object} Storage instance
|
|
10818
9819
|
*/
|
|
10819
|
-
getStorageByNode(nodeName) {
|
|
9820
|
+
async getStorageByNode(nodeName) {
|
|
10820
9821
|
if (!("storages" in this)) {
|
|
10821
9822
|
this.storages = {};
|
|
10822
9823
|
}
|
|
@@ -10834,6 +9835,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10834
9835
|
this.source.fieldsSchema[nodeName].fields,
|
|
10835
9836
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
10836
9837
|
);
|
|
9838
|
+
await this.storages[nodeName].init();
|
|
10837
9839
|
}
|
|
10838
9840
|
return this.storages[nodeName];
|
|
10839
9841
|
}
|
|
@@ -10849,8 +9851,8 @@ OPTIONS(description="${this.description}")`;
|
|
|
10849
9851
|
const endDate = new Date(startDate);
|
|
10850
9852
|
endDate.setDate(endDate.getDate() + daysToFetch - 1);
|
|
10851
9853
|
return {
|
|
10852
|
-
startDate:
|
|
10853
|
-
endDate:
|
|
9854
|
+
startDate: DateUtils2.formatDate(startDate),
|
|
9855
|
+
endDate: DateUtils2.formatDate(endDate)
|
|
10854
9856
|
};
|
|
10855
9857
|
}
|
|
10856
9858
|
};
|
|
@@ -10866,7 +9868,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10866
9868
|
};
|
|
10867
9869
|
})();
|
|
10868
9870
|
const OpenExchangeRates = (function() {
|
|
10869
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
9871
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
10870
9872
|
var historicalFields = {
|
|
10871
9873
|
date: {
|
|
10872
9874
|
type: "DATE",
|
|
@@ -10954,12 +9956,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
10954
9956
|
this.fieldsSchema = OpenExchangeRatesFieldsSchema;
|
|
10955
9957
|
}
|
|
10956
9958
|
/*
|
|
10957
|
-
|
|
9959
|
+
@param date The requested date as a Date object
|
|
10958
9960
|
|
|
10959
|
-
|
|
9961
|
+
@return data array
|
|
10960
9962
|
|
|
10961
|
-
|
|
10962
|
-
fetchData(date) {
|
|
9963
|
+
*/
|
|
9964
|
+
async fetchData(date) {
|
|
10963
9965
|
let data = [];
|
|
10964
9966
|
let base = this.config.base.value;
|
|
10965
9967
|
let app_id = this.config.AppId.value;
|
|
@@ -10967,13 +9969,14 @@ OPTIONS(description="${this.description}")`;
|
|
|
10967
9969
|
if (this.config.Symbols.value) {
|
|
10968
9970
|
symbols = "&symbols=" + String(this.config.Symbols.value).replace(/[^A-Z,]/g, "");
|
|
10969
9971
|
}
|
|
10970
|
-
var date =
|
|
9972
|
+
var date = DateUtils2.formatDate(date);
|
|
10971
9973
|
const urlWithoutKey = `https://openexchangerates.org/api/historical/${date}.json?base=${base}${symbols}`;
|
|
10972
9974
|
console.log(`OpenExchangeRates API URL:`, urlWithoutKey);
|
|
10973
9975
|
const url = `${urlWithoutKey}&app_id=${app_id}`;
|
|
10974
9976
|
this.config.logMessage(`Fetching rates for ${date}`);
|
|
10975
|
-
var response =
|
|
10976
|
-
var
|
|
9977
|
+
var response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
9978
|
+
var text = await response.getContentText();
|
|
9979
|
+
var historical = JSON.parse(text);
|
|
10977
9980
|
for (var currency in historical["rates"]) {
|
|
10978
9981
|
data.push({
|
|
10979
9982
|
date: new Date(date),
|
|
@@ -10986,7 +9989,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
10986
9989
|
}
|
|
10987
9990
|
};
|
|
10988
9991
|
var OpenExchangeRatesConnector = class OpenExchangeRatesConnector extends AbstractConnector2 {
|
|
10989
|
-
constructor(config, source, storageName = "
|
|
9992
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
10990
9993
|
super(config, source, null, runConfig);
|
|
10991
9994
|
this.storageName = storageName;
|
|
10992
9995
|
}
|
|
@@ -10999,11 +10002,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
10999
10002
|
* Main method - entry point for the import process
|
|
11000
10003
|
* Processes all nodes defined in the fields configuration
|
|
11001
10004
|
*/
|
|
11002
|
-
startImportProcess() {
|
|
10005
|
+
async startImportProcess() {
|
|
11003
10006
|
var _a;
|
|
11004
10007
|
const fields = ConnectorUtils.parseFields((_a = this.config.Fields) == null ? void 0 : _a.value);
|
|
11005
10008
|
for (const nodeName in fields) {
|
|
11006
|
-
this.processNode({
|
|
10009
|
+
await this.processNode({
|
|
11007
10010
|
nodeName,
|
|
11008
10011
|
fields: fields[nodeName] || []
|
|
11009
10012
|
});
|
|
@@ -11015,11 +10018,11 @@ OPTIONS(description="${this.description}")`;
|
|
|
11015
10018
|
* @param {string} options.nodeName - Name of the node to process
|
|
11016
10019
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11017
10020
|
*/
|
|
11018
|
-
processNode({ nodeName, fields }) {
|
|
10021
|
+
async processNode({ nodeName, fields }) {
|
|
11019
10022
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
11020
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
10023
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
11021
10024
|
} else {
|
|
11022
|
-
this.processCatalogNode({ nodeName, fields });
|
|
10025
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
11023
10026
|
}
|
|
11024
10027
|
}
|
|
11025
10028
|
/**
|
|
@@ -11028,7 +10031,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11028
10031
|
* @param {string} options.nodeName - Name of the node
|
|
11029
10032
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11030
10033
|
*/
|
|
11031
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
10034
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
11032
10035
|
var _a;
|
|
11033
10036
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
11034
10037
|
if (daysToFetch <= 0) {
|
|
@@ -11038,11 +10041,12 @@ OPTIONS(description="${this.description}")`;
|
|
|
11038
10041
|
for (let daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
11039
10042
|
const currentDate = new Date(startDate);
|
|
11040
10043
|
currentDate.setDate(currentDate.getDate() + daysShift);
|
|
11041
|
-
let data = this.source.fetchData(currentDate);
|
|
10044
|
+
let data = await this.source.fetchData(currentDate);
|
|
11042
10045
|
this.config.logMessage(data.length ? `${data.length} rows were fetched` : `No records have been fetched`);
|
|
11043
10046
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
11044
10047
|
const preparedData = data.length ? data : [];
|
|
11045
|
-
this.getStorageByNode(nodeName)
|
|
10048
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
10049
|
+
await storage.saveData(preparedData);
|
|
11046
10050
|
}
|
|
11047
10051
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
11048
10052
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -11055,7 +10059,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11055
10059
|
* @param {string} options.nodeName - Name of the node
|
|
11056
10060
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
11057
10061
|
*/
|
|
11058
|
-
processCatalogNode({ nodeName, fields }) {
|
|
10062
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
11059
10063
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
11060
10064
|
}
|
|
11061
10065
|
//---- getStorageName -------------------------------------------------
|
|
@@ -11067,7 +10071,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11067
10071
|
* @return AbstractStorage
|
|
11068
10072
|
*
|
|
11069
10073
|
*/
|
|
11070
|
-
getStorageByNode(nodeName) {
|
|
10074
|
+
async getStorageByNode(nodeName) {
|
|
11071
10075
|
if (!("storages" in this)) {
|
|
11072
10076
|
this.storages = {};
|
|
11073
10077
|
}
|
|
@@ -11085,6 +10089,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11085
10089
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
11086
10090
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
11087
10091
|
);
|
|
10092
|
+
await this.storages[nodeName].init();
|
|
11088
10093
|
}
|
|
11089
10094
|
return this.storages[nodeName];
|
|
11090
10095
|
}
|
|
@@ -11101,7 +10106,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11101
10106
|
};
|
|
11102
10107
|
})();
|
|
11103
10108
|
const MicrosoftAds = (function() {
|
|
11104
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
10109
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
11105
10110
|
const MicrosoftAdsHelper = {
|
|
11106
10111
|
/**
|
|
11107
10112
|
* Parse fields string into a structured object
|
|
@@ -11124,7 +10129,7 @@ OPTIONS(description="${this.description}")`;
|
|
|
11124
10129
|
* @param {number} [opts.interval=5000]
|
|
11125
10130
|
* @returns {Object}
|
|
11126
10131
|
*/
|
|
11127
|
-
pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
|
|
10132
|
+
async pollUntilStatus({ url, options, isDone, interval = 5e3 }) {
|
|
11128
10133
|
const startTime = Date.now();
|
|
11129
10134
|
const timeout = 15 * 60 * 1e3;
|
|
11130
10135
|
let statusResult;
|
|
@@ -11133,9 +10138,10 @@ OPTIONS(description="${this.description}")`;
|
|
|
11133
10138
|
if (Date.now() - startTime > timeout) {
|
|
11134
10139
|
throw new Error("Polling timed out after 15 minutes");
|
|
11135
10140
|
}
|
|
11136
|
-
|
|
11137
|
-
const response =
|
|
11138
|
-
|
|
10141
|
+
await AsyncUtils2.delay(interval);
|
|
10142
|
+
const response = await HttpUtils2.fetch(url, options);
|
|
10143
|
+
const text = await response.getContentText();
|
|
10144
|
+
statusResult = JSON.parse(text);
|
|
11139
10145
|
} while (!isDone(statusResult));
|
|
11140
10146
|
return statusResult;
|
|
11141
10147
|
} catch (error) {
|
|
@@ -11151,13 +10157,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
11151
10157
|
* @param {string} url
|
|
11152
10158
|
* @returns {Array<Array<string>>}
|
|
11153
10159
|
*/
|
|
11154
|
-
downloadCsvRows(url) {
|
|
11155
|
-
const response =
|
|
11156
|
-
const
|
|
10160
|
+
async downloadCsvRows(url) {
|
|
10161
|
+
const response = await HttpUtils2.fetch(url);
|
|
10162
|
+
const blob = await response.getBlob();
|
|
10163
|
+
const files = FileUtils2.unzip(blob);
|
|
11157
10164
|
const allRows = [];
|
|
11158
10165
|
files.forEach((file) => {
|
|
11159
10166
|
const csvText = file.getDataAsString();
|
|
11160
|
-
const rows =
|
|
10167
|
+
const rows = FileUtils2.parseCsv(csvText);
|
|
11161
10168
|
allRows.push(...rows);
|
|
11162
10169
|
});
|
|
11163
10170
|
return allRows;
|
|
@@ -12284,7 +11291,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12284
11291
|
/**
|
|
12285
11292
|
* Retrieve and store an OAuth access token using the refresh token
|
|
12286
11293
|
*/
|
|
12287
|
-
getAccessToken() {
|
|
11294
|
+
async getAccessToken() {
|
|
12288
11295
|
var _a, _b;
|
|
12289
11296
|
const tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
|
|
12290
11297
|
const scopes = [
|
|
@@ -12310,8 +11317,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12310
11317
|
body: Object.entries(form).map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&")
|
|
12311
11318
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
12312
11319
|
};
|
|
12313
|
-
const resp =
|
|
12314
|
-
const
|
|
11320
|
+
const resp = await HttpUtils2.fetch(tokenUrl, options);
|
|
11321
|
+
const text = await resp.getContentText();
|
|
11322
|
+
const json = JSON.parse(text);
|
|
12315
11323
|
if (json.error) {
|
|
12316
11324
|
throw new Error(`Token error: ${json.error} - ${json.error_description}`);
|
|
12317
11325
|
}
|
|
@@ -12337,7 +11345,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12337
11345
|
* @param {Function} [opts.onBatchReady] - Optional callback for batch processing
|
|
12338
11346
|
* @returns {Array<Object>}
|
|
12339
11347
|
*/
|
|
12340
|
-
fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
|
|
11348
|
+
async fetchData({ nodeName, accountId, fields = [], start_time, end_time, onBatchReady }) {
|
|
12341
11349
|
const schema = this.fieldsSchema[nodeName];
|
|
12342
11350
|
if (schema.uniqueKeys) {
|
|
12343
11351
|
const missingKeys = schema.uniqueKeys.filter((key) => !fields.includes(key));
|
|
@@ -12347,12 +11355,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12347
11355
|
}
|
|
12348
11356
|
switch (nodeName) {
|
|
12349
11357
|
case "campaigns":
|
|
12350
|
-
this._fetchCampaignData({ accountId, fields, onBatchReady });
|
|
11358
|
+
await this._fetchCampaignData({ accountId, fields, onBatchReady });
|
|
12351
11359
|
return [];
|
|
12352
11360
|
case "ad_performance_report":
|
|
12353
|
-
return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
11361
|
+
return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
12354
11362
|
case "user_location_performance_report":
|
|
12355
|
-
return this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
11363
|
+
return await this._fetchReportData({ accountId, fields, start_time, end_time, nodeName });
|
|
12356
11364
|
default:
|
|
12357
11365
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
12358
11366
|
}
|
|
@@ -12366,14 +11374,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12366
11374
|
* @returns {void}
|
|
12367
11375
|
* @private
|
|
12368
11376
|
*/
|
|
12369
|
-
_fetchCampaignData({ accountId, fields, onBatchReady }) {
|
|
12370
|
-
this.getAccessToken();
|
|
11377
|
+
async _fetchCampaignData({ accountId, fields, onBatchReady }) {
|
|
11378
|
+
await this.getAccessToken();
|
|
12371
11379
|
this.config.logMessage(`Fetching Campaigns, AssetGroups and AdGroups for account ${accountId}...`);
|
|
12372
11380
|
const entityTypes = ["Campaigns", "AssetGroups", "AdGroups"];
|
|
12373
11381
|
const allRecords = [];
|
|
12374
11382
|
let campaignRecords = [];
|
|
12375
11383
|
for (const entityType of entityTypes) {
|
|
12376
|
-
const records = this._downloadEntity({
|
|
11384
|
+
const records = await this._downloadEntity({
|
|
12377
11385
|
submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByAccountIds",
|
|
12378
11386
|
submitOpts: {
|
|
12379
11387
|
method: "post",
|
|
@@ -12411,21 +11419,21 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12411
11419
|
}
|
|
12412
11420
|
const filteredMainData = MicrosoftAdsHelper.filterByFields(allRecords, fields);
|
|
12413
11421
|
if (filteredMainData.length > 0) {
|
|
12414
|
-
onBatchReady(filteredMainData);
|
|
11422
|
+
await onBatchReady(filteredMainData);
|
|
12415
11423
|
}
|
|
12416
11424
|
this.config.logMessage(`Fetching Keywords for account ${accountId} (processing by campaigns to avoid size limits)...`);
|
|
12417
11425
|
const campaignIds = MicrosoftAdsHelper.extractCampaignIds(campaignRecords);
|
|
12418
11426
|
this.config.logMessage(`Found ${campaignIds.length} campaigns, fetching Keywords in batches`);
|
|
12419
11427
|
this.config.logMessage(`Campaign IDs: ${campaignIds.slice(0, 10).join(", ")}${campaignIds.length > 10 ? "..." : ""}`);
|
|
12420
11428
|
let totalFetched = 0;
|
|
12421
|
-
this._fetchEntityByCampaigns({
|
|
11429
|
+
await this._fetchEntityByCampaigns({
|
|
12422
11430
|
accountId,
|
|
12423
11431
|
entityType: "Keywords",
|
|
12424
11432
|
campaignIds,
|
|
12425
|
-
onBatchReady: (batchRecords) => {
|
|
11433
|
+
onBatchReady: async (batchRecords) => {
|
|
12426
11434
|
totalFetched += batchRecords.length;
|
|
12427
11435
|
const filteredBatch = MicrosoftAdsHelper.filterByFields(batchRecords, fields);
|
|
12428
|
-
onBatchReady(filteredBatch);
|
|
11436
|
+
await onBatchReady(filteredBatch);
|
|
12429
11437
|
}
|
|
12430
11438
|
});
|
|
12431
11439
|
this.config.logMessage(`${totalFetched} rows of Keywords were fetched for account ${accountId}`);
|
|
@@ -12439,15 +11447,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12439
11447
|
* @returns {Array<Object>}
|
|
12440
11448
|
* @private
|
|
12441
11449
|
*/
|
|
12442
|
-
_downloadEntity({ submitUrl, submitOpts }) {
|
|
12443
|
-
const submitResp =
|
|
12444
|
-
const
|
|
11450
|
+
async _downloadEntity({ submitUrl, submitOpts }) {
|
|
11451
|
+
const submitResp = await HttpUtils2.fetch(submitUrl, submitOpts);
|
|
11452
|
+
const text = await submitResp.getContentText();
|
|
11453
|
+
const requestId = JSON.parse(text).DownloadRequestId;
|
|
12445
11454
|
const pollUrl = "https://bulk.api.bingads.microsoft.com/Bulk/v13/BulkDownloadStatus/Query";
|
|
12446
11455
|
const pollOpts = Object.assign({}, submitOpts, {
|
|
12447
11456
|
payload: JSON.stringify({ RequestId: requestId }),
|
|
12448
11457
|
body: JSON.stringify({ RequestId: requestId })
|
|
12449
11458
|
});
|
|
12450
|
-
const pollResult = MicrosoftAdsHelper.pollUntilStatus({
|
|
11459
|
+
const pollResult = await MicrosoftAdsHelper.pollUntilStatus({
|
|
12451
11460
|
url: pollUrl,
|
|
12452
11461
|
options: pollOpts,
|
|
12453
11462
|
isDone: (status) => {
|
|
@@ -12457,7 +11466,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12457
11466
|
return status.RequestStatus === "Completed";
|
|
12458
11467
|
}
|
|
12459
11468
|
});
|
|
12460
|
-
const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
|
|
11469
|
+
const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ResultFileUrl);
|
|
12461
11470
|
const result = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
|
|
12462
11471
|
return result;
|
|
12463
11472
|
}
|
|
@@ -12471,7 +11480,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12471
11480
|
* @returns {Array<Object>} - Returns empty array if onBatchReady callback is provided, otherwise returns all records
|
|
12472
11481
|
* @private
|
|
12473
11482
|
*/
|
|
12474
|
-
_fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
|
|
11483
|
+
async _fetchEntityByCampaigns({ accountId, entityType, campaignIds, onBatchReady }) {
|
|
12475
11484
|
if (campaignIds.length === 0) {
|
|
12476
11485
|
this.config.logMessage(`No active campaigns found for account ${accountId}, skipping ${entityType} fetch`);
|
|
12477
11486
|
return [];
|
|
@@ -12481,9 +11490,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12481
11490
|
const campaignBatch = campaignIds.slice(i, i + batchSize);
|
|
12482
11491
|
this.config.logMessage(`Fetching ${entityType} for campaigns batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(campaignIds.length / batchSize)} (${campaignBatch.length} campaigns)`);
|
|
12483
11492
|
try {
|
|
12484
|
-
const batchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch });
|
|
11493
|
+
const batchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch });
|
|
12485
11494
|
this.config.logMessage(`Fetched ${batchRecords.length} ${entityType.toLowerCase()} from current batch`);
|
|
12486
|
-
onBatchReady(batchRecords);
|
|
11495
|
+
await onBatchReady(batchRecords);
|
|
12487
11496
|
} catch (error) {
|
|
12488
11497
|
if (error.message && error.message.includes("100MB")) {
|
|
12489
11498
|
const newBatchSize = Math.max(1, Math.floor(batchSize / 2));
|
|
@@ -12491,9 +11500,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12491
11500
|
for (let j = i; j < Math.min(i + batchSize, campaignIds.length); j += newBatchSize) {
|
|
12492
11501
|
const smallerBatch = campaignIds.slice(j, j + newBatchSize);
|
|
12493
11502
|
try {
|
|
12494
|
-
const smallerBatchRecords = this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
|
|
11503
|
+
const smallerBatchRecords = await this._downloadEntityBatch({ accountId, entityType, campaignBatch: smallerBatch });
|
|
12495
11504
|
this.config.logMessage(`Fetched ${smallerBatchRecords.length} ${entityType.toLowerCase()} from smaller batch (${smallerBatch.length} campaigns)`);
|
|
12496
|
-
onBatchReady(smallerBatchRecords);
|
|
11505
|
+
await onBatchReady(smallerBatchRecords);
|
|
12497
11506
|
} catch (smallerError) {
|
|
12498
11507
|
if (smallerError.message && smallerError.message.includes("100MB")) {
|
|
12499
11508
|
throw new Error(`Failed to fetch ${entityType}: batch size of ${smallerBatch.length} campaigns still exceeds 100MB limit`);
|
|
@@ -12520,7 +11529,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12520
11529
|
* @returns {Array<Object>}
|
|
12521
11530
|
* @private
|
|
12522
11531
|
*/
|
|
12523
|
-
_downloadEntityBatch({ accountId, entityType, campaignBatch }) {
|
|
11532
|
+
async _downloadEntityBatch({ accountId, entityType, campaignBatch }) {
|
|
12524
11533
|
const downloadBody = {
|
|
12525
11534
|
Campaigns: campaignBatch.map((id) => ({
|
|
12526
11535
|
CampaignId: Number(id),
|
|
@@ -12545,7 +11554,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12545
11554
|
payload: JSON.stringify(downloadBody),
|
|
12546
11555
|
body: JSON.stringify(downloadBody)
|
|
12547
11556
|
};
|
|
12548
|
-
return this._downloadEntity({
|
|
11557
|
+
return await this._downloadEntity({
|
|
12549
11558
|
submitUrl: "https://bulk.api.bingads.microsoft.com/Bulk/v13/Campaigns/DownloadByCampaignIds",
|
|
12550
11559
|
submitOpts
|
|
12551
11560
|
});
|
|
@@ -12561,22 +11570,22 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12561
11570
|
* @returns {Array<Object>}
|
|
12562
11571
|
* @private
|
|
12563
11572
|
*/
|
|
12564
|
-
_fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
|
|
12565
|
-
this.getAccessToken();
|
|
11573
|
+
async _fetchReportData({ accountId, fields, start_time, end_time, nodeName }) {
|
|
11574
|
+
await this.getAccessToken();
|
|
12566
11575
|
const schema = this.fieldsSchema[nodeName];
|
|
12567
|
-
const submitResponse = this._submitReportRequest({
|
|
11576
|
+
const submitResponse = await this._submitReportRequest({
|
|
12568
11577
|
accountId,
|
|
12569
11578
|
fields,
|
|
12570
11579
|
start_time,
|
|
12571
11580
|
end_time,
|
|
12572
11581
|
schema
|
|
12573
11582
|
});
|
|
12574
|
-
const pollResult = this._pollReportStatus({ submitResponse });
|
|
11583
|
+
const pollResult = await this._pollReportStatus({ submitResponse });
|
|
12575
11584
|
if (!pollResult.ReportRequestStatus.ReportDownloadUrl) {
|
|
12576
11585
|
this.config.logMessage(`No data available for the specified time period (${start_time} to ${end_time}). Report status: ${JSON.stringify(pollResult.ReportRequestStatus)}`);
|
|
12577
11586
|
return [];
|
|
12578
11587
|
}
|
|
12579
|
-
const csvRows = MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
|
|
11588
|
+
const csvRows = await MicrosoftAdsHelper.downloadCsvRows(pollResult.ReportRequestStatus.ReportDownloadUrl);
|
|
12580
11589
|
const records = MicrosoftAdsHelper.csvRowsToObjects(csvRows);
|
|
12581
11590
|
return MicrosoftAdsHelper.filterByFields(records, fields);
|
|
12582
11591
|
}
|
|
@@ -12591,7 +11600,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12591
11600
|
* @returns {Object} - Submit response
|
|
12592
11601
|
* @private
|
|
12593
11602
|
*/
|
|
12594
|
-
_submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
|
|
11603
|
+
async _submitReportRequest({ accountId, fields, start_time, end_time, schema }) {
|
|
12595
11604
|
const dateRange = {
|
|
12596
11605
|
CustomDateRangeStart: { Day: new Date(start_time).getDate(), Month: new Date(start_time).getMonth() + 1, Year: new Date(start_time).getFullYear() },
|
|
12597
11606
|
CustomDateRangeEnd: { Day: new Date(end_time).getDate(), Month: new Date(end_time).getMonth() + 1, Year: new Date(end_time).getFullYear() },
|
|
@@ -12624,8 +11633,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12624
11633
|
body: JSON.stringify({ ReportRequest: requestBody })
|
|
12625
11634
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
12626
11635
|
};
|
|
12627
|
-
const submitResp =
|
|
12628
|
-
const submitResponseText = submitResp.getContentText();
|
|
11636
|
+
const submitResp = await HttpUtils2.fetch(submitUrl, submitOpts);
|
|
11637
|
+
const submitResponseText = await submitResp.getContentText();
|
|
12629
11638
|
try {
|
|
12630
11639
|
const submitResponse = JSON.parse(submitResponseText);
|
|
12631
11640
|
if (submitResponse.OperationErrors && submitResponse.OperationErrors.length > 0) {
|
|
@@ -12649,7 +11658,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12649
11658
|
* @returns {Object} - Poll result with report status
|
|
12650
11659
|
* @private
|
|
12651
11660
|
*/
|
|
12652
|
-
_pollReportStatus({ submitResponse }) {
|
|
11661
|
+
async _pollReportStatus({ submitResponse }) {
|
|
12653
11662
|
const pollUrl = "https://reporting.api.bingads.microsoft.com/Reporting/v13/GenerateReport/Poll";
|
|
12654
11663
|
const submitResponseText = JSON.stringify(submitResponse);
|
|
12655
11664
|
const pollOpts = {
|
|
@@ -12665,7 +11674,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12665
11674
|
payload: submitResponseText,
|
|
12666
11675
|
body: submitResponseText
|
|
12667
11676
|
};
|
|
12668
|
-
return MicrosoftAdsHelper.pollUntilStatus({
|
|
11677
|
+
return await MicrosoftAdsHelper.pollUntilStatus({
|
|
12669
11678
|
url: pollUrl,
|
|
12670
11679
|
options: pollOpts,
|
|
12671
11680
|
isDone: (status) => {
|
|
@@ -12678,7 +11687,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12678
11687
|
}
|
|
12679
11688
|
};
|
|
12680
11689
|
var MicrosoftAdsConnector = class MicrosoftAdsConnector extends AbstractConnector2 {
|
|
12681
|
-
constructor(config, source, storageName = "
|
|
11690
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
12682
11691
|
super(config, source, null, runConfig);
|
|
12683
11692
|
this.storageName = storageName;
|
|
12684
11693
|
}
|
|
@@ -12686,10 +11695,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12686
11695
|
* Main method - entry point for the import process
|
|
12687
11696
|
* Processes all nodes defined in the fields configuration
|
|
12688
11697
|
*/
|
|
12689
|
-
startImportProcess() {
|
|
11698
|
+
async startImportProcess() {
|
|
12690
11699
|
const fields = MicrosoftAdsHelper.parseFields(this.config.Fields.value);
|
|
12691
11700
|
for (const nodeName in fields) {
|
|
12692
|
-
this.processNode({
|
|
11701
|
+
await this.processNode({
|
|
12693
11702
|
nodeName,
|
|
12694
11703
|
accountId: this.config.AccountID.value,
|
|
12695
11704
|
fields: fields[nodeName] || []
|
|
@@ -12703,15 +11712,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12703
11712
|
* @param {string} options.accountId - Account ID
|
|
12704
11713
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12705
11714
|
*/
|
|
12706
|
-
processNode({ nodeName, accountId, fields }) {
|
|
11715
|
+
async processNode({ nodeName, accountId, fields }) {
|
|
12707
11716
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
12708
|
-
this.processTimeSeriesNode({
|
|
11717
|
+
await this.processTimeSeriesNode({
|
|
12709
11718
|
nodeName,
|
|
12710
11719
|
accountId,
|
|
12711
11720
|
fields
|
|
12712
11721
|
});
|
|
12713
11722
|
} else {
|
|
12714
|
-
this.processCatalogNode({
|
|
11723
|
+
await this.processCatalogNode({
|
|
12715
11724
|
nodeName,
|
|
12716
11725
|
accountId,
|
|
12717
11726
|
fields
|
|
@@ -12726,7 +11735,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12726
11735
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12727
11736
|
* @param {Object} options.storage - Storage instance
|
|
12728
11737
|
*/
|
|
12729
|
-
processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
11738
|
+
async processTimeSeriesNode({ nodeName, accountId, fields }) {
|
|
12730
11739
|
var _a;
|
|
12731
11740
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
12732
11741
|
if (daysToFetch <= 0) {
|
|
@@ -12736,9 +11745,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12736
11745
|
for (let dayOffset = 0; dayOffset < daysToFetch; dayOffset++) {
|
|
12737
11746
|
const currentDate = new Date(startDate);
|
|
12738
11747
|
currentDate.setDate(currentDate.getDate() + dayOffset);
|
|
12739
|
-
const formattedDate =
|
|
11748
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
12740
11749
|
this.config.logMessage(`Processing ${nodeName} for ${accountId} on ${formattedDate} (day ${dayOffset + 1} of ${daysToFetch})`);
|
|
12741
|
-
const data = this.source.fetchData({
|
|
11750
|
+
const data = await this.source.fetchData({
|
|
12742
11751
|
nodeName,
|
|
12743
11752
|
accountId,
|
|
12744
11753
|
start_time: formattedDate,
|
|
@@ -12748,7 +11757,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12748
11757
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId} on ${formattedDate}` : `No records have been fetched`);
|
|
12749
11758
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
12750
11759
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
12751
|
-
this.getStorageByNode(nodeName)
|
|
11760
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11761
|
+
await storage.saveData(preparedData);
|
|
12752
11762
|
data.length && this.config.logMessage(`Successfully saved ${data.length} rows for ${formattedDate}`);
|
|
12753
11763
|
}
|
|
12754
11764
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
@@ -12764,22 +11774,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12764
11774
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
12765
11775
|
* @param {Object} options.storage - Storage instance
|
|
12766
11776
|
*/
|
|
12767
|
-
processCatalogNode({ nodeName, accountId, fields }) {
|
|
11777
|
+
async processCatalogNode({ nodeName, accountId, fields }) {
|
|
12768
11778
|
var _a;
|
|
12769
|
-
const data = this.source.fetchData({
|
|
11779
|
+
const data = await this.source.fetchData({
|
|
12770
11780
|
nodeName,
|
|
12771
11781
|
accountId,
|
|
12772
11782
|
fields,
|
|
12773
|
-
onBatchReady: (batchData) => {
|
|
11783
|
+
onBatchReady: async (batchData) => {
|
|
12774
11784
|
this.config.logMessage(`Saving batch of ${batchData.length} records to storage`);
|
|
12775
11785
|
const preparedData = this.addMissingFieldsToData(batchData, fields);
|
|
12776
|
-
this.getStorageByNode(nodeName)
|
|
11786
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11787
|
+
await storage.saveData(preparedData);
|
|
12777
11788
|
}
|
|
12778
11789
|
});
|
|
12779
11790
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${accountId}` : `No records have been fetched`);
|
|
12780
11791
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
12781
11792
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
12782
|
-
this.getStorageByNode(nodeName)
|
|
11793
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
11794
|
+
await storage.saveData(preparedData);
|
|
12783
11795
|
}
|
|
12784
11796
|
}
|
|
12785
11797
|
/**
|
|
@@ -12787,7 +11799,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12787
11799
|
* @param {string} nodeName - Name of the node
|
|
12788
11800
|
* @returns {Object} Storage instance
|
|
12789
11801
|
*/
|
|
12790
|
-
getStorageByNode(nodeName) {
|
|
11802
|
+
async getStorageByNode(nodeName) {
|
|
12791
11803
|
if (!("storages" in this)) {
|
|
12792
11804
|
this.storages = {};
|
|
12793
11805
|
}
|
|
@@ -12805,6 +11817,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12805
11817
|
this.source.fieldsSchema[nodeName].fields,
|
|
12806
11818
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
12807
11819
|
);
|
|
11820
|
+
await this.storages[nodeName].init();
|
|
12808
11821
|
}
|
|
12809
11822
|
return this.storages[nodeName];
|
|
12810
11823
|
}
|
|
@@ -12821,7 +11834,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
12821
11834
|
};
|
|
12822
11835
|
})();
|
|
12823
11836
|
const LinkedInPages = (function() {
|
|
12824
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
11837
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
12825
11838
|
var followerStatisticsTimeBoundFields = {
|
|
12826
11839
|
"organization_urn": {
|
|
12827
11840
|
"description": "Organization URN",
|
|
@@ -13003,7 +12016,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13003
12016
|
* @param {Object} params - Additional parameters for the request
|
|
13004
12017
|
* @returns {Array} - Array of processed data objects
|
|
13005
12018
|
*/
|
|
13006
|
-
fetchData(nodeName, urn, params = {}) {
|
|
12019
|
+
async fetchData(nodeName, urn, params = {}) {
|
|
13007
12020
|
var _a;
|
|
13008
12021
|
const fields = params.fields || [];
|
|
13009
12022
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -13013,7 +12026,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13013
12026
|
}
|
|
13014
12027
|
switch (nodeName) {
|
|
13015
12028
|
case "follower_statistics_time_bound":
|
|
13016
|
-
return this.fetchOrganizationStats({
|
|
12029
|
+
return await this.fetchOrganizationStats({
|
|
13017
12030
|
urn,
|
|
13018
12031
|
nodeName,
|
|
13019
12032
|
endpoint: "organizationalEntityFollowerStatistics",
|
|
@@ -13022,7 +12035,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13022
12035
|
params
|
|
13023
12036
|
});
|
|
13024
12037
|
case "follower_statistics":
|
|
13025
|
-
return this.fetchOrganizationStats({
|
|
12038
|
+
return await this.fetchOrganizationStats({
|
|
13026
12039
|
urn,
|
|
13027
12040
|
nodeName,
|
|
13028
12041
|
endpoint: "organizationalEntityFollowerStatistics",
|
|
@@ -13047,7 +12060,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13047
12060
|
* @param {Array} [options.params.fields] - Additional parameters including fields
|
|
13048
12061
|
* @returns {Array} - Processed statistics data
|
|
13049
12062
|
*/
|
|
13050
|
-
fetchOrganizationStats(options) {
|
|
12063
|
+
async fetchOrganizationStats(options) {
|
|
13051
12064
|
const { urn, nodeName, endpoint, entityParam, formatter, params } = options;
|
|
13052
12065
|
const orgUrn = `urn:li:organization:${urn}`;
|
|
13053
12066
|
const encodedUrn = encodeURIComponent(orgUrn);
|
|
@@ -13058,7 +12071,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13058
12071
|
const endTimestamp = new Date(params.endDate).getTime();
|
|
13059
12072
|
url += `&timeIntervals=(timeRange:(start:${startTimestamp},end:${endTimestamp}),timeGranularityType:DAY)`;
|
|
13060
12073
|
}
|
|
13061
|
-
const response = this.makeRequest(url);
|
|
12074
|
+
const response = await this.makeRequest(url);
|
|
13062
12075
|
const elements = response.elements || [];
|
|
13063
12076
|
if (elements.length === 0) {
|
|
13064
12077
|
return [];
|
|
@@ -13075,9 +12088,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13075
12088
|
* @param {Object} headers - Optional additional headers
|
|
13076
12089
|
* @returns {Object} - API response parsed from JSON
|
|
13077
12090
|
*/
|
|
13078
|
-
makeRequest(url) {
|
|
12091
|
+
async makeRequest(url) {
|
|
13079
12092
|
console.log(`LinkedIn Pages API URL:`, url);
|
|
13080
|
-
OAuthUtils.getAccessToken({
|
|
12093
|
+
await OAuthUtils.getAccessToken({
|
|
13081
12094
|
config: this.config,
|
|
13082
12095
|
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
|
|
13083
12096
|
formData: {
|
|
@@ -13092,12 +12105,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13092
12105
|
"X-RestLi-Protocol-Version": "2.0.0"
|
|
13093
12106
|
};
|
|
13094
12107
|
const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
|
|
13095
|
-
const response =
|
|
13096
|
-
const result =
|
|
13097
|
-
|
|
13098
|
-
|
|
12108
|
+
const response = await HttpUtils2.fetch(authUrl, { headers });
|
|
12109
|
+
const result = await response.getContentText();
|
|
12110
|
+
const parsedResult = JSON.parse(result);
|
|
12111
|
+
if (parsedResult.status && parsedResult.status >= HTTP_STATUS2.BAD_REQUEST) {
|
|
12112
|
+
throw new Error(`LinkedIn API Error: ${parsedResult.message || "Unknown error"} (Status: ${parsedResult.status})`);
|
|
13099
12113
|
}
|
|
13100
|
-
return
|
|
12114
|
+
return parsedResult;
|
|
13101
12115
|
}
|
|
13102
12116
|
/**
|
|
13103
12117
|
* Process time-bound statistics data
|
|
@@ -13191,7 +12205,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13191
12205
|
}
|
|
13192
12206
|
};
|
|
13193
12207
|
var LinkedInPagesConnector = class LinkedInPagesConnector extends AbstractConnector2 {
|
|
13194
|
-
constructor(config, source, storageName = "
|
|
12208
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
13195
12209
|
super(config, source, null, runConfig);
|
|
13196
12210
|
this.storageName = storageName;
|
|
13197
12211
|
}
|
|
@@ -13199,11 +12213,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13199
12213
|
* Main method - entry point for the import process
|
|
13200
12214
|
* Processes all nodes defined in the fields configuration
|
|
13201
12215
|
*/
|
|
13202
|
-
startImportProcess() {
|
|
12216
|
+
async startImportProcess() {
|
|
13203
12217
|
const urns = FormatUtils.parseIds(this.config.OrganizationURNs.value, { prefix: "urn:li:organization:" });
|
|
13204
12218
|
const dataSources = FormatUtils.parseFields(this.config.Fields.value);
|
|
13205
12219
|
for (const nodeName in dataSources) {
|
|
13206
|
-
this.processNode({
|
|
12220
|
+
await this.processNode({
|
|
13207
12221
|
nodeName,
|
|
13208
12222
|
urns,
|
|
13209
12223
|
fields: dataSources[nodeName] || []
|
|
@@ -13217,13 +12231,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13217
12231
|
* @param {Array} options.urns - URNs to process
|
|
13218
12232
|
* @param {Array} options.fields - Fields to fetch
|
|
13219
12233
|
*/
|
|
13220
|
-
processNode({ nodeName, urns, fields }) {
|
|
12234
|
+
async processNode({ nodeName, urns, fields }) {
|
|
13221
12235
|
const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
|
|
13222
12236
|
const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
|
|
13223
12237
|
if (isTimeSeriesNode && !dateInfo) {
|
|
13224
12238
|
return;
|
|
13225
12239
|
}
|
|
13226
|
-
this.fetchAndSaveData({
|
|
12240
|
+
await this.fetchAndSaveData({
|
|
13227
12241
|
nodeName,
|
|
13228
12242
|
urns,
|
|
13229
12243
|
fields,
|
|
@@ -13244,7 +12258,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13244
12258
|
* @param {string} [options.startDate] - Start date for time series data
|
|
13245
12259
|
* @param {string} [options.endDate] - End date for time series data
|
|
13246
12260
|
*/
|
|
13247
|
-
fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
12261
|
+
async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
13248
12262
|
var _a;
|
|
13249
12263
|
for (const urn of urns) {
|
|
13250
12264
|
console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
|
|
@@ -13252,11 +12266,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13252
12266
|
console.log(`End date is +1 day due to LinkedIn Pages API requirements (to include actual end date in results)`);
|
|
13253
12267
|
}
|
|
13254
12268
|
const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
|
|
13255
|
-
const data = this.source.fetchData(nodeName, urn, params);
|
|
12269
|
+
const data = await this.source.fetchData(nodeName, urn, params);
|
|
13256
12270
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
|
|
13257
12271
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
13258
12272
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
13259
|
-
this.getStorageByNode(nodeName)
|
|
12273
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
12274
|
+
await storage.saveData(preparedData);
|
|
13260
12275
|
}
|
|
13261
12276
|
}
|
|
13262
12277
|
}
|
|
@@ -13265,7 +12280,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13265
12280
|
* @param {string} nodeName - Name of the node
|
|
13266
12281
|
* @returns {Object} - Storage instance
|
|
13267
12282
|
*/
|
|
13268
|
-
getStorageByNode(nodeName) {
|
|
12283
|
+
async getStorageByNode(nodeName) {
|
|
13269
12284
|
if (!("storages" in this)) {
|
|
13270
12285
|
this.storages = {};
|
|
13271
12286
|
}
|
|
@@ -13283,6 +12298,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13283
12298
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
13284
12299
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
13285
12300
|
);
|
|
12301
|
+
await this.storages[nodeName].init();
|
|
13286
12302
|
}
|
|
13287
12303
|
return this.storages[nodeName];
|
|
13288
12304
|
}
|
|
@@ -13320,7 +12336,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
13320
12336
|
};
|
|
13321
12337
|
})();
|
|
13322
12338
|
const LinkedInAds = (function() {
|
|
13323
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
12339
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
13324
12340
|
var creativesFields = {
|
|
13325
12341
|
"account": {
|
|
13326
12342
|
"description": "URN identifying the advertising account associated with the creative. This field is read-only.",
|
|
@@ -14276,7 +13292,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14276
13292
|
* @param {Object} params - Additional parameters for the request
|
|
14277
13293
|
* @returns {Array} - Array of fetched data objects
|
|
14278
13294
|
*/
|
|
14279
|
-
fetchData(nodeName, urn, params = {}) {
|
|
13295
|
+
async fetchData(nodeName, urn, params = {}) {
|
|
14280
13296
|
var _a;
|
|
14281
13297
|
const fields = params.fields || [];
|
|
14282
13298
|
const uniqueKeys = ((_a = this.fieldsSchema[nodeName]) == null ? void 0 : _a.uniqueKeys) || [];
|
|
@@ -14286,15 +13302,15 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14286
13302
|
}
|
|
14287
13303
|
switch (nodeName) {
|
|
14288
13304
|
case "adAccounts":
|
|
14289
|
-
return this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
|
|
13305
|
+
return await this.fetchSingleResource({ urn, resourceType: "adAccounts", params });
|
|
14290
13306
|
case "adCampaignGroups":
|
|
14291
|
-
return this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
|
|
13307
|
+
return await this.fetchAdResource({ urn, resourceType: "adCampaignGroups", params });
|
|
14292
13308
|
case "adCampaigns":
|
|
14293
|
-
return this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
|
|
13309
|
+
return await this.fetchAdResource({ urn, resourceType: "adCampaigns", params });
|
|
14294
13310
|
case "creatives":
|
|
14295
|
-
return this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
|
|
13311
|
+
return await this.fetchAdResource({ urn, resourceType: "creatives", params, queryType: "criteria" });
|
|
14296
13312
|
case "adAnalytics":
|
|
14297
|
-
return this.fetchAdAnalytics(urn, params);
|
|
13313
|
+
return await this.fetchAdAnalytics(urn, params);
|
|
14298
13314
|
default:
|
|
14299
13315
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
14300
13316
|
}
|
|
@@ -14307,10 +13323,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14307
13323
|
* @param {Object} options.params - Additional parameters for the request
|
|
14308
13324
|
* @returns {Array} - Array containing the single resource
|
|
14309
13325
|
*/
|
|
14310
|
-
fetchSingleResource({ urn, resourceType, params }) {
|
|
13326
|
+
async fetchSingleResource({ urn, resourceType, params }) {
|
|
14311
13327
|
let url = `${this.BASE_URL}${resourceType}/${encodeURIComponent(urn)}`;
|
|
14312
13328
|
url += `?fields=${this.formatFields(params.fields)}`;
|
|
14313
|
-
const result = this.makeRequest(url);
|
|
13329
|
+
const result = await this.makeRequest(url);
|
|
14314
13330
|
return [result];
|
|
14315
13331
|
}
|
|
14316
13332
|
/**
|
|
@@ -14322,10 +13338,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14322
13338
|
* @param {string} [options.queryType='search'] - Query type parameter
|
|
14323
13339
|
* @returns {Array} - Array of fetched resources
|
|
14324
13340
|
*/
|
|
14325
|
-
fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
|
|
13341
|
+
async fetchAdResource({ urn, resourceType, params, queryType = "search" }) {
|
|
14326
13342
|
let url = `${this.BASE_URL}adAccounts/${encodeURIComponent(urn)}/${resourceType}?q=${queryType}&pageSize=100`;
|
|
14327
13343
|
url += `&fields=${this.formatFields(params.fields)}`;
|
|
14328
|
-
return this.fetchWithPagination(url);
|
|
13344
|
+
return await this.fetchWithPagination(url);
|
|
14329
13345
|
}
|
|
14330
13346
|
/**
|
|
14331
13347
|
* Fetch analytics data, handling field limits and data merging
|
|
@@ -14336,7 +13352,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14336
13352
|
* @param {Array} params.fields - Fields to fetch
|
|
14337
13353
|
* @returns {Array} - Combined array of analytics data
|
|
14338
13354
|
*/
|
|
14339
|
-
fetchAdAnalytics(urn, params) {
|
|
13355
|
+
async fetchAdAnalytics(urn, params) {
|
|
14340
13356
|
const startDate = new Date(params.startDate);
|
|
14341
13357
|
const endDate = new Date(params.endDate);
|
|
14342
13358
|
const accountUrn = `urn:li:sponsoredAccount:${urn}`;
|
|
@@ -14351,7 +13367,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14351
13367
|
encodedUrn,
|
|
14352
13368
|
fields: fieldChunk
|
|
14353
13369
|
});
|
|
14354
|
-
const res = this.makeRequest(url);
|
|
13370
|
+
const res = await this.makeRequest(url);
|
|
14355
13371
|
const elements = res.elements || [];
|
|
14356
13372
|
allResults = this.mergeAnalyticsResults(allResults, elements);
|
|
14357
13373
|
}
|
|
@@ -14480,9 +13496,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14480
13496
|
* @param {Object} headers - Optional additional headers
|
|
14481
13497
|
* @returns {Object} - API response parsed from JSON
|
|
14482
13498
|
*/
|
|
14483
|
-
makeRequest(url) {
|
|
13499
|
+
async makeRequest(url) {
|
|
14484
13500
|
console.log(`LinkedIn Ads API Request URL:`, url);
|
|
14485
|
-
OAuthUtils.getAccessToken({
|
|
13501
|
+
await OAuthUtils.getAccessToken({
|
|
14486
13502
|
config: this.config,
|
|
14487
13503
|
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
|
|
14488
13504
|
formData: {
|
|
@@ -14497,8 +13513,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14497
13513
|
"X-RestLi-Protocol-Version": "2.0.0"
|
|
14498
13514
|
};
|
|
14499
13515
|
const authUrl = `${url}${url.includes("?") ? "&" : "?"}oauth2_access_token=${this.config.AccessToken.value}`;
|
|
14500
|
-
const response =
|
|
14501
|
-
const
|
|
13516
|
+
const response = await HttpUtils2.fetch(authUrl, { headers });
|
|
13517
|
+
const text = await response.getContentText();
|
|
13518
|
+
const result = JSON.parse(text);
|
|
14502
13519
|
return result;
|
|
14503
13520
|
}
|
|
14504
13521
|
/**
|
|
@@ -14507,7 +13524,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14507
13524
|
* @param {Object} headers - Optional additional headers
|
|
14508
13525
|
* @returns {Array} - Combined array of results from all pages
|
|
14509
13526
|
*/
|
|
14510
|
-
fetchWithPagination(baseUrl) {
|
|
13527
|
+
async fetchWithPagination(baseUrl) {
|
|
14511
13528
|
let allResults = [];
|
|
14512
13529
|
let pageToken = null;
|
|
14513
13530
|
do {
|
|
@@ -14515,7 +13532,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14515
13532
|
if (pageToken) {
|
|
14516
13533
|
pageUrl += `${pageUrl.includes("?") ? "&" : "?"}pageToken=${encodeURIComponent(pageToken)}`;
|
|
14517
13534
|
}
|
|
14518
|
-
const res = this.makeRequest(pageUrl);
|
|
13535
|
+
const res = await this.makeRequest(pageUrl);
|
|
14519
13536
|
const elements = res.elements || [];
|
|
14520
13537
|
allResults = allResults.concat(elements);
|
|
14521
13538
|
const metadata = res.metadata || {};
|
|
@@ -14525,7 +13542,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14525
13542
|
}
|
|
14526
13543
|
};
|
|
14527
13544
|
var LinkedInAdsConnector = class LinkedInAdsConnector extends AbstractConnector2 {
|
|
14528
|
-
constructor(config, source, storageName = "
|
|
13545
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
14529
13546
|
super(config, source, null, runConfig);
|
|
14530
13547
|
this.storageName = storageName;
|
|
14531
13548
|
}
|
|
@@ -14533,11 +13550,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14533
13550
|
* Main method - entry point for the import process
|
|
14534
13551
|
* Processes all nodes defined in the fields configuration
|
|
14535
13552
|
*/
|
|
14536
|
-
startImportProcess() {
|
|
13553
|
+
async startImportProcess() {
|
|
14537
13554
|
const urns = FormatUtils.parseIds(this.config.AccountURNs.value, { prefix: "urn:li:sponsoredAccount:" });
|
|
14538
13555
|
const dataSources = FormatUtils.parseFields(this.config.Fields.value);
|
|
14539
13556
|
for (const nodeName in dataSources) {
|
|
14540
|
-
this.processNode({
|
|
13557
|
+
await this.processNode({
|
|
14541
13558
|
nodeName,
|
|
14542
13559
|
urns,
|
|
14543
13560
|
fields: dataSources[nodeName] || []
|
|
@@ -14551,13 +13568,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14551
13568
|
* @param {Array} options.urns - URNs to process
|
|
14552
13569
|
* @param {Array} options.fields - Fields to fetch
|
|
14553
13570
|
*/
|
|
14554
|
-
processNode({ nodeName, urns, fields }) {
|
|
13571
|
+
async processNode({ nodeName, urns, fields }) {
|
|
14555
13572
|
const isTimeSeriesNode = ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName]);
|
|
14556
13573
|
const dateInfo = this.prepareDateRangeIfNeeded(nodeName, isTimeSeriesNode);
|
|
14557
13574
|
if (isTimeSeriesNode && !dateInfo) {
|
|
14558
13575
|
return;
|
|
14559
13576
|
}
|
|
14560
|
-
this.fetchAndSaveData({
|
|
13577
|
+
await this.fetchAndSaveData({
|
|
14561
13578
|
nodeName,
|
|
14562
13579
|
urns,
|
|
14563
13580
|
fields,
|
|
@@ -14578,16 +13595,17 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14578
13595
|
* @param {string} [options.startDate] - Start date for time series data
|
|
14579
13596
|
* @param {string} [options.endDate] - End date for time series data
|
|
14580
13597
|
*/
|
|
14581
|
-
fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
13598
|
+
async fetchAndSaveData({ nodeName, urns, fields, isTimeSeriesNode, startDate, endDate }) {
|
|
14582
13599
|
var _a;
|
|
14583
13600
|
for (const urn of urns) {
|
|
14584
13601
|
console.log(`Processing ${nodeName} for ${urn}${isTimeSeriesNode ? ` from ${startDate} to ${endDate}` : ""}`);
|
|
14585
13602
|
const params = { fields, ...isTimeSeriesNode && { startDate, endDate } };
|
|
14586
|
-
const data = this.source.fetchData(nodeName, urn, params);
|
|
13603
|
+
const data = await this.source.fetchData(nodeName, urn, params);
|
|
14587
13604
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${urn}${endDate ? ` from ${startDate} to ${endDate}` : ""}` : `No records have been fetched`);
|
|
14588
13605
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
14589
13606
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
14590
|
-
this.getStorageByNode(nodeName)
|
|
13607
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
13608
|
+
await storage.saveData(preparedData);
|
|
14591
13609
|
}
|
|
14592
13610
|
}
|
|
14593
13611
|
}
|
|
@@ -14596,7 +13614,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14596
13614
|
* @param {string} nodeName - Name of the node
|
|
14597
13615
|
* @returns {Object} - Storage instance
|
|
14598
13616
|
*/
|
|
14599
|
-
getStorageByNode(nodeName) {
|
|
13617
|
+
async getStorageByNode(nodeName) {
|
|
14600
13618
|
if (!("storages" in this)) {
|
|
14601
13619
|
this.storages = {};
|
|
14602
13620
|
}
|
|
@@ -14614,6 +13632,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14614
13632
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
14615
13633
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
14616
13634
|
);
|
|
13635
|
+
await this.storages[nodeName].init();
|
|
14617
13636
|
}
|
|
14618
13637
|
return this.storages[nodeName];
|
|
14619
13638
|
}
|
|
@@ -14650,7 +13669,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
14650
13669
|
};
|
|
14651
13670
|
})();
|
|
14652
13671
|
const GoogleAds = (function() {
|
|
14653
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
13672
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
14654
13673
|
var keywordStatsFields = {
|
|
14655
13674
|
"keyword_id": {
|
|
14656
13675
|
"description": "Keyword Criterion ID",
|
|
@@ -15551,7 +14570,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15551
14570
|
* Get access token based on authentication type
|
|
15552
14571
|
* Supports OAuth2 and Service Account authentication
|
|
15553
14572
|
*/
|
|
15554
|
-
getAccessToken() {
|
|
14573
|
+
async getAccessToken() {
|
|
15555
14574
|
var _a;
|
|
15556
14575
|
if (this.accessToken && this.tokenExpiryTime && Date.now() < this.tokenExpiryTime) {
|
|
15557
14576
|
return this.accessToken;
|
|
@@ -15564,7 +14583,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15564
14583
|
let accessToken;
|
|
15565
14584
|
try {
|
|
15566
14585
|
if (authType === "oauth2") {
|
|
15567
|
-
accessToken = OAuthUtils.getAccessToken({
|
|
14586
|
+
accessToken = await OAuthUtils.getAccessToken({
|
|
15568
14587
|
config: this.config,
|
|
15569
14588
|
tokenUrl: "https://oauth2.googleapis.com/token",
|
|
15570
14589
|
formData: {
|
|
@@ -15575,7 +14594,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15575
14594
|
}
|
|
15576
14595
|
});
|
|
15577
14596
|
} else if (authType === "service_account") {
|
|
15578
|
-
accessToken = OAuthUtils.getServiceAccountToken({
|
|
14597
|
+
accessToken = await OAuthUtils.getServiceAccountToken({
|
|
15579
14598
|
config: this.config,
|
|
15580
14599
|
tokenUrl: "https://oauth2.googleapis.com/token",
|
|
15581
14600
|
serviceAccountKeyJson: authConfig.ServiceAccountKey.value,
|
|
@@ -15602,11 +14621,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15602
14621
|
* @param {Date} [options.startDate] - Start date for time series data
|
|
15603
14622
|
* @returns {Array<Object>} - Fetched data
|
|
15604
14623
|
*/
|
|
15605
|
-
fetchData(nodeName, customerId, options) {
|
|
14624
|
+
async fetchData(nodeName, customerId, options) {
|
|
15606
14625
|
console.log("Fetching data from Google Ads API for customer:", customerId);
|
|
15607
14626
|
const { fields, startDate } = options;
|
|
15608
14627
|
const query = this._buildQuery({ nodeName, fields, startDate });
|
|
15609
|
-
|
|
14628
|
+
const response = await this.makeRequest({ customerId, query, nodeName, fields });
|
|
14629
|
+
return await response;
|
|
15610
14630
|
}
|
|
15611
14631
|
/**
|
|
15612
14632
|
* Convert field names to API field names
|
|
@@ -15656,7 +14676,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15656
14676
|
const resourceName = this._getResourceName(nodeName);
|
|
15657
14677
|
let query = `SELECT ${apiFields.join(", ")} FROM ${resourceName}`;
|
|
15658
14678
|
if (startDate && this.fieldsSchema[nodeName].isTimeSeries) {
|
|
15659
|
-
const formattedDate =
|
|
14679
|
+
const formattedDate = DateUtils2.formatDate(startDate);
|
|
15660
14680
|
query += ` WHERE segments.date = '${formattedDate}'`;
|
|
15661
14681
|
}
|
|
15662
14682
|
return query;
|
|
@@ -15670,9 +14690,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15670
14690
|
* @param {Array<string>} options.fields - Fields that were requested
|
|
15671
14691
|
* @returns {Array<Object>} - API response data
|
|
15672
14692
|
*/
|
|
15673
|
-
makeRequest({ customerId, query, nodeName, fields }) {
|
|
14693
|
+
async makeRequest({ customerId, query, nodeName, fields }) {
|
|
15674
14694
|
var _a, _b;
|
|
15675
|
-
const accessToken = this.getAccessToken();
|
|
14695
|
+
const accessToken = await this.getAccessToken();
|
|
15676
14696
|
const url = `https://googleads.googleapis.com/v21/customers/${customerId}/googleAds:search`;
|
|
15677
14697
|
console.log(`Google Ads API Request URL: ${url}`);
|
|
15678
14698
|
console.log(`GAQL Query: ${query}`);
|
|
@@ -15696,8 +14716,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15696
14716
|
body: JSON.stringify(requestBody),
|
|
15697
14717
|
muteHttpExceptions: true
|
|
15698
14718
|
};
|
|
15699
|
-
const response = this.urlFetchWithRetry(url, options);
|
|
15700
|
-
const
|
|
14719
|
+
const response = await this.urlFetchWithRetry(url, options);
|
|
14720
|
+
const text = await response.getContentText();
|
|
14721
|
+
const jsonData = JSON.parse(text);
|
|
15701
14722
|
if (jsonData.error) {
|
|
15702
14723
|
throw new Error(`Google Ads API error: ${jsonData.error.message}`);
|
|
15703
14724
|
}
|
|
@@ -15752,7 +14773,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15752
14773
|
}
|
|
15753
14774
|
};
|
|
15754
14775
|
var GoogleAdsConnector = class GoogleAdsConnector extends AbstractConnector2 {
|
|
15755
|
-
constructor(config, source, storageName = "
|
|
14776
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
15756
14777
|
super(config, source, null, runConfig);
|
|
15757
14778
|
this.storageName = storageName;
|
|
15758
14779
|
}
|
|
@@ -15760,11 +14781,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15760
14781
|
* Main method - entry point for the import process
|
|
15761
14782
|
* Processes all nodes defined in the fields configuration
|
|
15762
14783
|
*/
|
|
15763
|
-
startImportProcess() {
|
|
14784
|
+
async startImportProcess() {
|
|
15764
14785
|
const customerIds = FormatUtils.parseIds(this.config.CustomerId.value, { stripCharacters: "-" });
|
|
15765
14786
|
const fields = FormatUtils.parseFields(this.config.Fields.value);
|
|
15766
14787
|
for (const nodeName in fields) {
|
|
15767
|
-
this.processNode({
|
|
14788
|
+
await this.processNode({
|
|
15768
14789
|
nodeName,
|
|
15769
14790
|
customerIds,
|
|
15770
14791
|
fields: fields[nodeName] || []
|
|
@@ -15778,16 +14799,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15778
14799
|
* @param {Array<string>} options.customerIds - Array of customer IDs to process
|
|
15779
14800
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15780
14801
|
*/
|
|
15781
|
-
processNode({ nodeName, customerIds, fields }) {
|
|
14802
|
+
async processNode({ nodeName, customerIds, fields }) {
|
|
15782
14803
|
for (const customerId of customerIds) {
|
|
15783
14804
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
15784
|
-
this.processTimeSeriesNode({
|
|
14805
|
+
await this.processTimeSeriesNode({
|
|
15785
14806
|
nodeName,
|
|
15786
14807
|
customerId,
|
|
15787
14808
|
fields
|
|
15788
14809
|
});
|
|
15789
14810
|
} else {
|
|
15790
|
-
this.processCatalogNode({
|
|
14811
|
+
await this.processCatalogNode({
|
|
15791
14812
|
nodeName,
|
|
15792
14813
|
customerId,
|
|
15793
14814
|
fields
|
|
@@ -15802,7 +14823,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15802
14823
|
* @param {string} options.customerId - Customer ID
|
|
15803
14824
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15804
14825
|
*/
|
|
15805
|
-
processTimeSeriesNode({ nodeName, customerId, fields }) {
|
|
14826
|
+
async processTimeSeriesNode({ nodeName, customerId, fields }) {
|
|
15806
14827
|
var _a;
|
|
15807
14828
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
15808
14829
|
if (daysToFetch <= 0) {
|
|
@@ -15812,12 +14833,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15812
14833
|
for (let i = 0; i < daysToFetch; i++) {
|
|
15813
14834
|
const currentDate = new Date(startDate);
|
|
15814
14835
|
currentDate.setDate(currentDate.getDate() + i);
|
|
15815
|
-
const formattedDate =
|
|
15816
|
-
const data = this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
|
|
14836
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
14837
|
+
const data = await this.source.fetchData(nodeName, customerId, { fields, startDate: currentDate });
|
|
15817
14838
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId} on ${formattedDate}` : `ℹ️ No records have been fetched`);
|
|
15818
14839
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
15819
14840
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
15820
|
-
this.getStorageByNode(nodeName)
|
|
14841
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
14842
|
+
await storage.saveData(preparedData);
|
|
15821
14843
|
}
|
|
15822
14844
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
15823
14845
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -15831,13 +14853,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15831
14853
|
* @param {string} options.customerId - Customer ID
|
|
15832
14854
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
15833
14855
|
*/
|
|
15834
|
-
processCatalogNode({ nodeName, customerId, fields }) {
|
|
14856
|
+
async processCatalogNode({ nodeName, customerId, fields }) {
|
|
15835
14857
|
var _a;
|
|
15836
|
-
const data = this.source.fetchData(nodeName, customerId, { fields });
|
|
14858
|
+
const data = await this.source.fetchData(nodeName, customerId, { fields });
|
|
15837
14859
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for customer ${customerId}` : `ℹ️ No records have been fetched`);
|
|
15838
14860
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
15839
14861
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
15840
|
-
this.getStorageByNode(nodeName)
|
|
14862
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
14863
|
+
await storage.saveData(preparedData);
|
|
15841
14864
|
}
|
|
15842
14865
|
}
|
|
15843
14866
|
/**
|
|
@@ -15845,7 +14868,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15845
14868
|
* @param {string} nodeName - Name of the node
|
|
15846
14869
|
* @returns {Object} - Storage instance
|
|
15847
14870
|
*/
|
|
15848
|
-
getStorageByNode(nodeName) {
|
|
14871
|
+
async getStorageByNode(nodeName) {
|
|
15849
14872
|
if (!("storages" in this)) {
|
|
15850
14873
|
this.storages = {};
|
|
15851
14874
|
}
|
|
@@ -15863,6 +14886,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15863
14886
|
this.source.fieldsSchema[nodeName].fields,
|
|
15864
14887
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
15865
14888
|
);
|
|
14889
|
+
await this.storages[nodeName].init();
|
|
15866
14890
|
}
|
|
15867
14891
|
return this.storages[nodeName];
|
|
15868
14892
|
}
|
|
@@ -15879,7 +14903,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
15879
14903
|
};
|
|
15880
14904
|
})();
|
|
15881
14905
|
const GitHub = (function() {
|
|
15882
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
14906
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
15883
14907
|
var repositoryStatsFields = {
|
|
15884
14908
|
date: {
|
|
15885
14909
|
type: "date",
|
|
@@ -16192,16 +15216,16 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16192
15216
|
* @param {Array<string>} opts.fields
|
|
16193
15217
|
* @returns {Array<Object>}
|
|
16194
15218
|
*/
|
|
16195
|
-
fetchData({ nodeName, fields = [] }) {
|
|
15219
|
+
async fetchData({ nodeName, fields = [] }) {
|
|
16196
15220
|
switch (nodeName) {
|
|
16197
15221
|
case "repository":
|
|
16198
|
-
const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
15222
|
+
const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
16199
15223
|
return this._filterBySchema({ items: [repoData], nodeName, fields });
|
|
16200
15224
|
case "contributors":
|
|
16201
|
-
const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
15225
|
+
const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
16202
15226
|
return this._filterBySchema({ items: contribData, nodeName, fields });
|
|
16203
15227
|
case "repositoryStats":
|
|
16204
|
-
return this._fetchRepositoryStats({ nodeName, fields });
|
|
15228
|
+
return await this._fetchRepositoryStats({ nodeName, fields });
|
|
16205
15229
|
default:
|
|
16206
15230
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
16207
15231
|
}
|
|
@@ -16213,9 +15237,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16213
15237
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16214
15238
|
* @returns {Array} Array of repository statistics data
|
|
16215
15239
|
*/
|
|
16216
|
-
_fetchRepositoryStats({ nodeName, fields }) {
|
|
16217
|
-
const repoData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
16218
|
-
const contribData = this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
15240
|
+
async _fetchRepositoryStats({ nodeName, fields }) {
|
|
15241
|
+
const repoData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}` });
|
|
15242
|
+
const contribData = await this.makeRequest({ endpoint: `repos/${this.config.RepositoryName.value}/contributors?per_page=1000` });
|
|
16219
15243
|
return this._filterBySchema({
|
|
16220
15244
|
items: [{
|
|
16221
15245
|
"date": new Date((/* @__PURE__ */ new Date()).setHours(0, 0, 0, 0)),
|
|
@@ -16232,30 +15256,32 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16232
15256
|
* @param {string} options.endpoint - API endpoint path (e.g., "repos/owner/repo")
|
|
16233
15257
|
* @returns {Object} - API response parsed from JSON
|
|
16234
15258
|
*/
|
|
16235
|
-
makeRequest({ endpoint }) {
|
|
16236
|
-
|
|
16237
|
-
|
|
16238
|
-
|
|
16239
|
-
|
|
16240
|
-
|
|
16241
|
-
|
|
16242
|
-
"
|
|
16243
|
-
|
|
16244
|
-
|
|
15259
|
+
async makeRequest({ endpoint }) {
|
|
15260
|
+
try {
|
|
15261
|
+
const baseUrl = "https://api.github.com/";
|
|
15262
|
+
const url = `${baseUrl}${endpoint}`;
|
|
15263
|
+
const response = await HttpUtils2.fetch(url, {
|
|
15264
|
+
"method": "get",
|
|
15265
|
+
"muteHttpExceptions": true,
|
|
15266
|
+
"headers": {
|
|
15267
|
+
"Accept": "application/vnd.github+json",
|
|
15268
|
+
"Authorization": `Bearer ${this.config.AccessToken.value}`,
|
|
15269
|
+
"User-Agent": "owox"
|
|
15270
|
+
}
|
|
15271
|
+
});
|
|
15272
|
+
const text = await response.getContentText();
|
|
15273
|
+
const result = JSON.parse(text);
|
|
15274
|
+
if (result && result.message === "Not Found") {
|
|
15275
|
+
throw new Error(
|
|
15276
|
+
"The repository was not found. The repository name should be in the format: owner/repo"
|
|
15277
|
+
);
|
|
16245
15278
|
}
|
|
16246
|
-
|
|
16247
|
-
|
|
16248
|
-
|
|
16249
|
-
|
|
16250
|
-
|
|
16251
|
-
);
|
|
15279
|
+
return result;
|
|
15280
|
+
} catch (error) {
|
|
15281
|
+
this.config.logMessage(`Error: ${error.message}`);
|
|
15282
|
+
console.error(error.stack);
|
|
15283
|
+
throw error;
|
|
16252
15284
|
}
|
|
16253
|
-
return result;
|
|
16254
|
-
}
|
|
16255
|
-
catch(error) {
|
|
16256
|
-
this.config.logMessage(`Error: ${error.message}`);
|
|
16257
|
-
console.error(error.stack);
|
|
16258
|
-
throw error;
|
|
16259
15285
|
}
|
|
16260
15286
|
/**
|
|
16261
15287
|
* Keep only requestedFields plus any schema-required keys.
|
|
@@ -16281,7 +15307,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16281
15307
|
}
|
|
16282
15308
|
};
|
|
16283
15309
|
var GitHubConnector = class GitHubConnector extends AbstractConnector2 {
|
|
16284
|
-
constructor(config, source, storageName = "
|
|
15310
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
16285
15311
|
super(config, source, null, runConfig);
|
|
16286
15312
|
this.storageName = storageName;
|
|
16287
15313
|
}
|
|
@@ -16289,10 +15315,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16289
15315
|
* Main method - entry point for the import process
|
|
16290
15316
|
* Processes all nodes defined in the fields configuration
|
|
16291
15317
|
*/
|
|
16292
|
-
startImportProcess() {
|
|
15318
|
+
async startImportProcess() {
|
|
16293
15319
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
16294
15320
|
for (const nodeName in fields) {
|
|
16295
|
-
this.processNode({
|
|
15321
|
+
await this.processNode({
|
|
16296
15322
|
nodeName,
|
|
16297
15323
|
fields: fields[nodeName] || []
|
|
16298
15324
|
});
|
|
@@ -16304,11 +15330,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16304
15330
|
* @param {string} options.nodeName - Name of the node to process
|
|
16305
15331
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16306
15332
|
*/
|
|
16307
|
-
processNode({ nodeName, fields }) {
|
|
15333
|
+
async processNode({ nodeName, fields }) {
|
|
16308
15334
|
if (ConnectorUtils.isTimeSeriesNode(this.source.fieldsSchema[nodeName])) {
|
|
16309
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
15335
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
16310
15336
|
} else {
|
|
16311
|
-
this.processCatalogNode({ nodeName, fields });
|
|
15337
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
16312
15338
|
}
|
|
16313
15339
|
}
|
|
16314
15340
|
/**
|
|
@@ -16318,7 +15344,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16318
15344
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16319
15345
|
* @param {Object} options.storage - Storage instance
|
|
16320
15346
|
*/
|
|
16321
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
15347
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
16322
15348
|
console.log(`Time series node processing not implemented for ${nodeName}`);
|
|
16323
15349
|
}
|
|
16324
15350
|
/**
|
|
@@ -16328,13 +15354,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16328
15354
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
16329
15355
|
* @param {Object} options.storage - Storage instance
|
|
16330
15356
|
*/
|
|
16331
|
-
processCatalogNode({ nodeName, fields }) {
|
|
15357
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
16332
15358
|
var _a;
|
|
16333
|
-
const data = this.source.fetchData({ nodeName, fields });
|
|
15359
|
+
const data = await this.source.fetchData({ nodeName, fields });
|
|
16334
15360
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched` : `No records have been fetched`);
|
|
16335
15361
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
16336
15362
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
16337
|
-
this.getStorageByNode(nodeName)
|
|
15363
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
15364
|
+
await storage.saveData(preparedData);
|
|
16338
15365
|
}
|
|
16339
15366
|
}
|
|
16340
15367
|
/**
|
|
@@ -16342,7 +15369,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16342
15369
|
* @param {string} nodeName - Name of the node
|
|
16343
15370
|
* @returns {Object} Storage instance
|
|
16344
15371
|
*/
|
|
16345
|
-
getStorageByNode(nodeName) {
|
|
15372
|
+
async getStorageByNode(nodeName) {
|
|
16346
15373
|
if (!("storages" in this)) {
|
|
16347
15374
|
this.storages = {};
|
|
16348
15375
|
}
|
|
@@ -16360,6 +15387,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16360
15387
|
this.source.fieldsSchema[nodeName].fields,
|
|
16361
15388
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
16362
15389
|
);
|
|
15390
|
+
await this.storages[nodeName].init();
|
|
16363
15391
|
}
|
|
16364
15392
|
return this.storages[nodeName];
|
|
16365
15393
|
}
|
|
@@ -16376,7 +15404,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
16376
15404
|
};
|
|
16377
15405
|
})();
|
|
16378
15406
|
const FacebookMarketing = (function() {
|
|
16379
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
15407
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
16380
15408
|
var adGroupFields = {
|
|
16381
15409
|
"id": {
|
|
16382
15410
|
"description": "The ID of this ad.",
|
|
@@ -20881,12 +19909,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
20881
19909
|
@return data array
|
|
20882
19910
|
|
|
20883
19911
|
*/
|
|
20884
|
-
fetchData(nodeName, accountId, fields, startDate = null) {
|
|
19912
|
+
async fetchData(nodeName, accountId, fields, startDate = null) {
|
|
20885
19913
|
let url = "https://graph.facebook.com/v23.0/";
|
|
20886
19914
|
let formattedDate = null;
|
|
20887
19915
|
let timeRange = null;
|
|
20888
19916
|
if (startDate) {
|
|
20889
|
-
formattedDate =
|
|
19917
|
+
formattedDate = DateUtils2.formatDate(startDate);
|
|
20890
19918
|
timeRange = encodeURIComponent(JSON.stringify({ since: formattedDate, until: formattedDate }));
|
|
20891
19919
|
}
|
|
20892
19920
|
switch (nodeName) {
|
|
@@ -20910,7 +19938,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
20910
19938
|
case "ad-account/insights-by-region":
|
|
20911
19939
|
case "ad-account/insights-by-product-id":
|
|
20912
19940
|
case "ad-account/insights-by-age-and-gender":
|
|
20913
|
-
return this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
|
|
19941
|
+
return await this._fetchInsightsData({ nodeName, accountId, fields, timeRange, url });
|
|
20914
19942
|
case "ad-group":
|
|
20915
19943
|
url += `act_${accountId}/ads?fields=${this._buildFieldsString({ nodeName, fields })}&limit=${this.fieldsSchema[nodeName].limit}`;
|
|
20916
19944
|
break;
|
|
@@ -20919,7 +19947,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
20919
19947
|
}
|
|
20920
19948
|
console.log(`Facebook API URL:`, url);
|
|
20921
19949
|
url += `&access_token=${this.config.AccessToken.value}`;
|
|
20922
|
-
return this._fetchPaginatedData(url, nodeName, fields);
|
|
19950
|
+
return await this._fetchPaginatedData(url, nodeName, fields);
|
|
20923
19951
|
}
|
|
20924
19952
|
//---- castRecordFields -------------------------------------------------
|
|
20925
19953
|
/**
|
|
@@ -20981,7 +20009,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
20981
20009
|
* @return {Array} Processed insights data
|
|
20982
20010
|
* @private
|
|
20983
20011
|
*/
|
|
20984
|
-
_fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
|
|
20012
|
+
async _fetchInsightsData({ nodeName, accountId, fields, timeRange, url }) {
|
|
20985
20013
|
const breakdowns = this.fieldsSchema[nodeName].breakdowns || [];
|
|
20986
20014
|
const regularFields = this._prepareFields({ nodeName, fields, breakdowns });
|
|
20987
20015
|
const requestUrl = this._buildInsightsUrl({
|
|
@@ -20992,7 +20020,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
20992
20020
|
nodeName,
|
|
20993
20021
|
url
|
|
20994
20022
|
});
|
|
20995
|
-
const allData = this._fetchPaginatedData(requestUrl, nodeName, fields);
|
|
20023
|
+
const allData = await this._fetchPaginatedData(requestUrl, nodeName, fields);
|
|
20996
20024
|
if (this.config.ProcessShortLinks.value && allData.length > 0 && allData.some((record) => record.link_url_asset)) {
|
|
20997
20025
|
return processShortLinks(allData, {
|
|
20998
20026
|
shortLinkField: "link_url_asset",
|
|
@@ -21133,12 +20161,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21133
20161
|
* @return {Array} All fetched data
|
|
21134
20162
|
* @private
|
|
21135
20163
|
*/
|
|
21136
|
-
_fetchPaginatedData(initialUrl, nodeName, fields) {
|
|
20164
|
+
async _fetchPaginatedData(initialUrl, nodeName, fields) {
|
|
21137
20165
|
var allData = [];
|
|
21138
20166
|
var nextPageURL = initialUrl;
|
|
21139
20167
|
while (nextPageURL) {
|
|
21140
|
-
var response = this.urlFetchWithRetry(nextPageURL);
|
|
21141
|
-
var
|
|
20168
|
+
var response = await this.urlFetchWithRetry(nextPageURL);
|
|
20169
|
+
var text = await response.getContentText();
|
|
20170
|
+
var jsonData = JSON.parse(text);
|
|
21142
20171
|
if ("data" in jsonData) {
|
|
21143
20172
|
nextPageURL = jsonData.paging ? jsonData.paging.next : null;
|
|
21144
20173
|
jsonData.data.forEach((record, index) => {
|
|
@@ -21161,12 +20190,12 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21161
20190
|
};
|
|
21162
20191
|
var FacebookMarketingConnector = class FacebookMarketingConnector extends AbstractConnector2 {
|
|
21163
20192
|
// ---- constructor ------------------------------------
|
|
21164
|
-
constructor(config, source, storageName = "
|
|
20193
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
21165
20194
|
super(config, source, null, runConfig);
|
|
21166
20195
|
this.storageName = storageName;
|
|
21167
20196
|
}
|
|
21168
20197
|
//---- startImportProcess -------------------------------------------------
|
|
21169
|
-
startImportProcess() {
|
|
20198
|
+
async startImportProcess() {
|
|
21170
20199
|
let accountsIds = String(this.config.AccoundIDs.value).split(/[,;]\s*/);
|
|
21171
20200
|
let fields = this.config.Fields.value.split(", ").reduce((acc, pair) => {
|
|
21172
20201
|
let [key, value] = pair.split(" ");
|
|
@@ -21178,7 +20207,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21178
20207
|
if (nodeName in this.source.fieldsSchema && this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
21179
20208
|
timeSeriesNodes[nodeName] = fields[nodeName];
|
|
21180
20209
|
} else {
|
|
21181
|
-
this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
|
|
20210
|
+
await this.startImportProcessOfCatalogData(nodeName, accountsIds, fields[nodeName]);
|
|
21182
20211
|
}
|
|
21183
20212
|
}
|
|
21184
20213
|
if (Object.keys(timeSeriesNodes).length > 0) {
|
|
@@ -21186,27 +20215,28 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21186
20215
|
let daysToFetch = null;
|
|
21187
20216
|
[startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
21188
20217
|
if (daysToFetch > 0) {
|
|
21189
|
-
this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
|
|
20218
|
+
await this.startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch);
|
|
21190
20219
|
}
|
|
21191
20220
|
}
|
|
21192
20221
|
}
|
|
21193
20222
|
//---- startImportProcessOfCatalogData -------------------------------------------------
|
|
21194
20223
|
/*
|
|
21195
20224
|
|
|
21196
|
-
Imports catalog (not time seriesed) data
|
|
20225
|
+
Imports catalog (not time seriesed) data
|
|
21197
20226
|
|
|
21198
20227
|
@param nodeName string Node name
|
|
21199
20228
|
@param accountsIds array list of account ids
|
|
21200
|
-
@param fields array list of fields
|
|
20229
|
+
@param fields array list of fields
|
|
21201
20230
|
|
|
21202
20231
|
*/
|
|
21203
|
-
startImportProcessOfCatalogData(nodeName, accountIds, fields) {
|
|
20232
|
+
async startImportProcessOfCatalogData(nodeName, accountIds, fields) {
|
|
21204
20233
|
var _a;
|
|
21205
20234
|
for (var i in accountIds) {
|
|
21206
20235
|
let accountId = accountIds[i];
|
|
21207
|
-
let data = this.source.fetchData(nodeName, accountId, fields);
|
|
20236
|
+
let data = await this.source.fetchData(nodeName, accountId, fields);
|
|
21208
20237
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
21209
|
-
this.getStorageByNode(nodeName)
|
|
20238
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
20239
|
+
await storage.saveData(data);
|
|
21210
20240
|
}
|
|
21211
20241
|
data.length && this.config.logMessage(`${data.length} rows of ${nodeName} were fetched for account ${accountId}`);
|
|
21212
20242
|
}
|
|
@@ -21214,23 +20244,24 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21214
20244
|
//---- startImportProcessOfTimeSeriesData -------------------------------------------------
|
|
21215
20245
|
/*
|
|
21216
20246
|
|
|
21217
|
-
Imports time series (not catalog) data
|
|
20247
|
+
Imports time series (not catalog) data
|
|
21218
20248
|
|
|
21219
20249
|
@param accountsIds (array) list of account ids
|
|
21220
20250
|
@param timeSeriesNodes (object) of properties, each is array of fields
|
|
21221
|
-
@param startDate (Data) start date
|
|
20251
|
+
@param startDate (Data) start date
|
|
21222
20252
|
@param daysToFetch (integer) days to import
|
|
21223
20253
|
|
|
21224
20254
|
*/
|
|
21225
|
-
startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
|
|
20255
|
+
async startImportProcessOfTimeSeriesData(accountsIds, timeSeriesNodes, startDate, daysToFetch = 1) {
|
|
21226
20256
|
var _a;
|
|
21227
20257
|
for (var daysShift = 0; daysShift < daysToFetch; daysShift++) {
|
|
21228
20258
|
for (let accountId of accountsIds) {
|
|
21229
20259
|
for (var nodeName in timeSeriesNodes) {
|
|
21230
|
-
this.config.logMessage(`Start importing data for ${
|
|
21231
|
-
let data = this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
|
|
20260
|
+
this.config.logMessage(`Start importing data for ${DateUtils2.formatDate(startDate)}: ${accountId}/${nodeName}`);
|
|
20261
|
+
let data = await this.source.fetchData(nodeName, accountId, timeSeriesNodes[nodeName], startDate);
|
|
21232
20262
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
21233
|
-
this.getStorageByNode(nodeName)
|
|
20263
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
20264
|
+
await storage.saveData(data);
|
|
21234
20265
|
}
|
|
21235
20266
|
this.config.logMessage(data.length ? `${data.length} records were fetched` : `No records have been fetched`);
|
|
21236
20267
|
}
|
|
@@ -21250,7 +20281,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21250
20281
|
* @return AbstractStorage
|
|
21251
20282
|
*
|
|
21252
20283
|
*/
|
|
21253
|
-
getStorageByNode(nodeName) {
|
|
20284
|
+
async getStorageByNode(nodeName) {
|
|
21254
20285
|
if (!("storages" in this)) {
|
|
21255
20286
|
this.storages = {};
|
|
21256
20287
|
}
|
|
@@ -21268,6 +20299,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21268
20299
|
this.source.fieldsSchema[nodeName]["fields"],
|
|
21269
20300
|
`${this.source.fieldsSchema[nodeName]["description"]} ${this.source.fieldsSchema[nodeName]["documentation"]}`
|
|
21270
20301
|
);
|
|
20302
|
+
await this.storages[nodeName].init();
|
|
21271
20303
|
}
|
|
21272
20304
|
return this.storages[nodeName];
|
|
21273
20305
|
}
|
|
@@ -21284,7 +20316,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
21284
20316
|
};
|
|
21285
20317
|
})();
|
|
21286
20318
|
const CriteoAds = (function() {
|
|
21287
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
20319
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
21288
20320
|
var CriteoAdsHelper = {
|
|
21289
20321
|
/**
|
|
21290
20322
|
* Parse fields string into a structured object
|
|
@@ -22131,10 +21163,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22131
21163
|
* @param {Date} opts.date
|
|
22132
21164
|
* @returns {Array<Object>}
|
|
22133
21165
|
*/
|
|
22134
|
-
fetchData({ nodeName, accountId, fields = [], date }) {
|
|
21166
|
+
async fetchData({ nodeName, accountId, fields = [], date }) {
|
|
22135
21167
|
switch (nodeName) {
|
|
22136
21168
|
case "statistics":
|
|
22137
|
-
return this._fetchStatistics({ accountId, fields, date });
|
|
21169
|
+
return await this._fetchStatistics({ accountId, fields, date });
|
|
22138
21170
|
default:
|
|
22139
21171
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
22140
21172
|
}
|
|
@@ -22148,16 +21180,17 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22148
21180
|
* @returns {Array<Object>} - Parsed and enriched data
|
|
22149
21181
|
* @private
|
|
22150
21182
|
*/
|
|
22151
|
-
_fetchStatistics({ accountId, fields, date }) {
|
|
21183
|
+
async _fetchStatistics({ accountId, fields, date }) {
|
|
22152
21184
|
const uniqueKeys = this.fieldsSchema.statistics.uniqueKeys || [];
|
|
22153
21185
|
const missingKeys = uniqueKeys.filter((key) => !fields.includes(key));
|
|
22154
21186
|
if (missingKeys.length > 0) {
|
|
22155
21187
|
throw new Error(`Missing required unique fields for endpoint 'statistics'. Missing fields: ${missingKeys.join(", ")}`);
|
|
22156
21188
|
}
|
|
22157
|
-
this.getAccessToken();
|
|
21189
|
+
await this.getAccessToken();
|
|
22158
21190
|
const requestBody = this._buildStatisticsRequestBody({ accountId, fields, date });
|
|
22159
|
-
const response = this._makeApiRequest(requestBody);
|
|
22160
|
-
const
|
|
21191
|
+
const response = await this._makeApiRequest(requestBody);
|
|
21192
|
+
const text = await response.getContentText();
|
|
21193
|
+
const jsonObject = JSON.parse(text);
|
|
22161
21194
|
return this.parseApiResponse({ apiResponse: jsonObject, date, accountId, fields });
|
|
22162
21195
|
}
|
|
22163
21196
|
/**
|
|
@@ -22195,7 +21228,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22195
21228
|
* @returns {Object} - HTTP response
|
|
22196
21229
|
* @private
|
|
22197
21230
|
*/
|
|
22198
|
-
_makeApiRequest(requestBody) {
|
|
21231
|
+
async _makeApiRequest(requestBody) {
|
|
22199
21232
|
const apiVersion = "2025-07";
|
|
22200
21233
|
const apiUrl = `https://api.criteo.com/${apiVersion}/statistics/report`;
|
|
22201
21234
|
const options = {
|
|
@@ -22209,19 +21242,20 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22209
21242
|
body: JSON.stringify(requestBody)
|
|
22210
21243
|
// TODO: body is for Node.js; refactor to centralize JSON option creation
|
|
22211
21244
|
};
|
|
22212
|
-
const response = this.urlFetchWithRetry(apiUrl, options);
|
|
21245
|
+
const response = await this.urlFetchWithRetry(apiUrl, options);
|
|
22213
21246
|
const responseCode = response.getResponseCode();
|
|
22214
21247
|
if (responseCode === HTTP_STATUS2.OK) {
|
|
22215
21248
|
return response;
|
|
22216
21249
|
} else {
|
|
22217
|
-
|
|
21250
|
+
const text = await response.getContentText();
|
|
21251
|
+
throw new Error(`API Error (${responseCode}): ${text}`);
|
|
22218
21252
|
}
|
|
22219
21253
|
}
|
|
22220
21254
|
/**
|
|
22221
21255
|
* Get access token from API
|
|
22222
21256
|
* Docs: https://developers.criteo.com/marketing-solutions/docs/authorization-code-setup
|
|
22223
21257
|
*/
|
|
22224
|
-
getAccessToken() {
|
|
21258
|
+
async getAccessToken() {
|
|
22225
21259
|
var _a;
|
|
22226
21260
|
if ((_a = this.config.AccessToken) == null ? void 0 : _a.value) {
|
|
22227
21261
|
return;
|
|
@@ -22244,8 +21278,9 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22244
21278
|
muteHttpExceptions: true
|
|
22245
21279
|
};
|
|
22246
21280
|
try {
|
|
22247
|
-
const response = this.urlFetchWithRetry(tokenUrl, options);
|
|
22248
|
-
const
|
|
21281
|
+
const response = await this.urlFetchWithRetry(tokenUrl, options);
|
|
21282
|
+
const text = await response.getContentText();
|
|
21283
|
+
const responseData = JSON.parse(text);
|
|
22249
21284
|
const accessToken = responseData["access_token"];
|
|
22250
21285
|
this.config.AccessToken = {
|
|
22251
21286
|
value: accessToken
|
|
@@ -22293,20 +21328,20 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22293
21328
|
}
|
|
22294
21329
|
};
|
|
22295
21330
|
var CriteoAdsConnector = class CriteoAdsConnector extends AbstractConnector2 {
|
|
22296
|
-
constructor(config, source, storageName = "
|
|
21331
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
22297
21332
|
super(config, source, null, runConfig);
|
|
22298
21333
|
this.storageName = storageName;
|
|
22299
21334
|
}
|
|
22300
21335
|
/**
|
|
22301
21336
|
* Main method - entry point for the import process
|
|
22302
21337
|
*/
|
|
22303
|
-
startImportProcess() {
|
|
21338
|
+
async startImportProcess() {
|
|
22304
21339
|
var _a, _b;
|
|
22305
21340
|
const fields = CriteoAdsHelper.parseFields(((_a = this.config.Fields) == null ? void 0 : _a.value) || "");
|
|
22306
21341
|
const advertiserIds = CriteoAdsHelper.parseAdvertiserIds(((_b = this.config.AdvertiserIDs) == null ? void 0 : _b.value) || "");
|
|
22307
21342
|
for (const advertiserId of advertiserIds) {
|
|
22308
21343
|
for (const nodeName in fields) {
|
|
22309
|
-
this.processNode({
|
|
21344
|
+
await this.processNode({
|
|
22310
21345
|
nodeName,
|
|
22311
21346
|
advertiserId,
|
|
22312
21347
|
fields: fields[nodeName] || []
|
|
@@ -22321,8 +21356,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22321
21356
|
* @param {string} options.advertiserId - Advertiser ID
|
|
22322
21357
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22323
21358
|
*/
|
|
22324
|
-
processNode({ nodeName, advertiserId, fields }) {
|
|
22325
|
-
this.processTimeSeriesNode({
|
|
21359
|
+
async processNode({ nodeName, advertiserId, fields }) {
|
|
21360
|
+
await this.processTimeSeriesNode({
|
|
22326
21361
|
nodeName,
|
|
22327
21362
|
advertiserId,
|
|
22328
21363
|
fields
|
|
@@ -22336,7 +21371,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22336
21371
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22337
21372
|
* @param {Object} options.storage - Storage instance
|
|
22338
21373
|
*/
|
|
22339
|
-
processTimeSeriesNode({ nodeName, advertiserId, fields }) {
|
|
21374
|
+
async processTimeSeriesNode({ nodeName, advertiserId, fields }) {
|
|
22340
21375
|
var _a;
|
|
22341
21376
|
const [startDate, daysToFetch] = this.getStartDateAndDaysToFetch();
|
|
22342
21377
|
if (daysToFetch <= 0) {
|
|
@@ -22346,8 +21381,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22346
21381
|
for (let i = 0; i < daysToFetch; i++) {
|
|
22347
21382
|
const currentDate = new Date(startDate);
|
|
22348
21383
|
currentDate.setDate(currentDate.getDate() + i);
|
|
22349
|
-
const formattedDate =
|
|
22350
|
-
const data = this.source.fetchData({
|
|
21384
|
+
const formattedDate = DateUtils2.formatDate(currentDate);
|
|
21385
|
+
const data = await this.source.fetchData({
|
|
22351
21386
|
nodeName,
|
|
22352
21387
|
accountId: advertiserId,
|
|
22353
21388
|
date: currentDate,
|
|
@@ -22356,7 +21391,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22356
21391
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched for ${advertiserId} on ${formattedDate}` : `No records have been fetched`);
|
|
22357
21392
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
22358
21393
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
22359
|
-
this.getStorageByNode(nodeName)
|
|
21394
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
21395
|
+
await storage.saveData(preparedData);
|
|
22360
21396
|
}
|
|
22361
21397
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
22362
21398
|
this.config.updateLastRequstedDate(currentDate);
|
|
@@ -22368,7 +21404,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22368
21404
|
* @param {string} nodeName - Name of the node
|
|
22369
21405
|
* @returns {Object} Storage instance
|
|
22370
21406
|
*/
|
|
22371
|
-
getStorageByNode(nodeName) {
|
|
21407
|
+
async getStorageByNode(nodeName) {
|
|
22372
21408
|
if (!("storages" in this)) {
|
|
22373
21409
|
this.storages = {};
|
|
22374
21410
|
}
|
|
@@ -22386,6 +21422,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22386
21422
|
this.source.fieldsSchema[nodeName].fields,
|
|
22387
21423
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
22388
21424
|
);
|
|
21425
|
+
await this.storages[nodeName].init();
|
|
22389
21426
|
}
|
|
22390
21427
|
return this.storages[nodeName];
|
|
22391
21428
|
}
|
|
@@ -22402,7 +21439,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22402
21439
|
};
|
|
22403
21440
|
})();
|
|
22404
21441
|
const BankOfCanada = (function() {
|
|
22405
|
-
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2,
|
|
21442
|
+
const { AbstractException: AbstractException2, HttpRequestException: HttpRequestException2, UnsupportedEnvironmentException: UnsupportedEnvironmentException2, AbstractStorage: AbstractStorage2, AbstractSource: AbstractSource2, AbstractRunConfig: AbstractRunConfig2, AbstractConnector: AbstractConnector2, AbstractConfig: AbstractConfig2, HttpUtils: HttpUtils2, FileUtils: FileUtils2, DateUtils: DateUtils2, CryptoUtils: CryptoUtils2, AsyncUtils: AsyncUtils2, RunConfigDto: RunConfigDto2, SourceConfigDto: SourceConfigDto2, StorageConfigDto: StorageConfigDto2, ConfigDto: ConfigDto2, HTTP_STATUS: HTTP_STATUS2, EXECUTION_STATUS: EXECUTION_STATUS2, RUN_CONFIG_TYPE: RUN_CONFIG_TYPE2, CONFIG_ATTRIBUTES: CONFIG_ATTRIBUTES2 } = Core;
|
|
22406
21443
|
var observationsFields = {
|
|
22407
21444
|
"date": {
|
|
22408
21445
|
"description": "The date for which the exchange rate was recorded.",
|
|
@@ -22480,10 +21517,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22480
21517
|
* @param {string} [opts.end_time]
|
|
22481
21518
|
* @returns {Array<Object>}
|
|
22482
21519
|
*/
|
|
22483
|
-
fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
21520
|
+
async fetchData({ nodeName, fields = [], start_time, end_time }) {
|
|
22484
21521
|
switch (nodeName) {
|
|
22485
21522
|
case "observations/group":
|
|
22486
|
-
return this._fetchObservations({ fields, start_time, end_time });
|
|
21523
|
+
return await this._fetchObservations({ fields, start_time, end_time });
|
|
22487
21524
|
default:
|
|
22488
21525
|
throw new Error(`Unknown node: ${nodeName}`);
|
|
22489
21526
|
}
|
|
@@ -22496,8 +21533,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22496
21533
|
* @param {string} opts.end_time
|
|
22497
21534
|
* @returns {Array<Object>}
|
|
22498
21535
|
*/
|
|
22499
|
-
_fetchObservations({ fields, start_time, end_time }) {
|
|
22500
|
-
const rates = this.makeRequest({
|
|
21536
|
+
async _fetchObservations({ fields, start_time, end_time }) {
|
|
21537
|
+
const rates = await this.makeRequest({
|
|
22501
21538
|
endpoint: `observations/group/FX_RATES_DAILY/json?start_date=${start_time}&end_date=${end_time}`
|
|
22502
21539
|
});
|
|
22503
21540
|
let data = [];
|
|
@@ -22519,13 +21556,13 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22519
21556
|
* @param {string} options.endpoint - API endpoint path (e.g., "observations/group/FX_RATES_DAILY/json")
|
|
22520
21557
|
* @returns {Object} - API response parsed from JSON
|
|
22521
21558
|
*/
|
|
22522
|
-
makeRequest({ endpoint }) {
|
|
21559
|
+
async makeRequest({ endpoint }) {
|
|
22523
21560
|
const baseUrl = "https://www.bankofcanada.ca/valet/";
|
|
22524
21561
|
const url = `${baseUrl}${endpoint}`;
|
|
22525
21562
|
console.log(`Bank of Canada API Request URL:`, url);
|
|
22526
|
-
const response =
|
|
22527
|
-
const result =
|
|
22528
|
-
return result;
|
|
21563
|
+
const response = await HttpUtils2.fetch(url, { "method": "get", "muteHttpExceptions": true });
|
|
21564
|
+
const result = await response.getContentText();
|
|
21565
|
+
return JSON.parse(result);
|
|
22529
21566
|
}
|
|
22530
21567
|
/**
|
|
22531
21568
|
* Keep only requestedFields plus any schema-required keys.
|
|
@@ -22550,7 +21587,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22550
21587
|
}
|
|
22551
21588
|
};
|
|
22552
21589
|
var BankOfCanadaConnector = class BankOfCanadaConnector extends AbstractConnector2 {
|
|
22553
|
-
constructor(config, source, storageName = "
|
|
21590
|
+
constructor(config, source, storageName = "GoogleBigQueryStorage", runConfig = null) {
|
|
22554
21591
|
super(config, source, null, runConfig);
|
|
22555
21592
|
this.storageName = storageName;
|
|
22556
21593
|
}
|
|
@@ -22558,10 +21595,10 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22558
21595
|
* Main method - entry point for the import process
|
|
22559
21596
|
* Processes all nodes defined in the fields configuration
|
|
22560
21597
|
*/
|
|
22561
|
-
startImportProcess() {
|
|
21598
|
+
async startImportProcess() {
|
|
22562
21599
|
const fields = ConnectorUtils.parseFields(this.config.Fields.value);
|
|
22563
21600
|
for (const nodeName in fields) {
|
|
22564
|
-
this.processNode({
|
|
21601
|
+
await this.processNode({
|
|
22565
21602
|
nodeName,
|
|
22566
21603
|
fields: fields[nodeName] || []
|
|
22567
21604
|
});
|
|
@@ -22573,11 +21610,11 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22573
21610
|
* @param {string} options.nodeName - Name of the node to process
|
|
22574
21611
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22575
21612
|
*/
|
|
22576
|
-
processNode({ nodeName, fields }) {
|
|
21613
|
+
async processNode({ nodeName, fields }) {
|
|
22577
21614
|
if (this.source.fieldsSchema[nodeName].isTimeSeries) {
|
|
22578
|
-
this.processTimeSeriesNode({ nodeName, fields });
|
|
21615
|
+
await this.processTimeSeriesNode({ nodeName, fields });
|
|
22579
21616
|
} else {
|
|
22580
|
-
this.processCatalogNode({ nodeName, fields });
|
|
21617
|
+
await this.processCatalogNode({ nodeName, fields });
|
|
22581
21618
|
}
|
|
22582
21619
|
}
|
|
22583
21620
|
/**
|
|
@@ -22587,14 +21624,14 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22587
21624
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22588
21625
|
* @param {Object} options.storage - Storage instance
|
|
22589
21626
|
*/
|
|
22590
|
-
processTimeSeriesNode({ nodeName, fields }) {
|
|
21627
|
+
async processTimeSeriesNode({ nodeName, fields }) {
|
|
22591
21628
|
var _a;
|
|
22592
21629
|
const dateRange = this.prepareDateRange();
|
|
22593
21630
|
if (!dateRange) {
|
|
22594
21631
|
console.log("No days to fetch for time series data");
|
|
22595
21632
|
return;
|
|
22596
21633
|
}
|
|
22597
|
-
const data = this.source.fetchData({
|
|
21634
|
+
const data = await this.source.fetchData({
|
|
22598
21635
|
nodeName,
|
|
22599
21636
|
start_time: dateRange.startDate,
|
|
22600
21637
|
end_time: dateRange.endDate,
|
|
@@ -22603,7 +21640,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22603
21640
|
this.config.logMessage(data.length ? `${data.length} rows of ${nodeName} were fetched from ${dateRange.startDate} to ${dateRange.endDate}` : `No records have been fetched`);
|
|
22604
21641
|
if (data.length || ((_a = this.config.CreateEmptyTables) == null ? void 0 : _a.value)) {
|
|
22605
21642
|
const preparedData = data.length ? this.addMissingFieldsToData(data, fields) : data;
|
|
22606
|
-
this.getStorageByNode(nodeName)
|
|
21643
|
+
const storage = await this.getStorageByNode(nodeName);
|
|
21644
|
+
await storage.saveData(preparedData);
|
|
22607
21645
|
}
|
|
22608
21646
|
if (this.runConfig.type === RUN_CONFIG_TYPE2.INCREMENTAL) {
|
|
22609
21647
|
this.config.updateLastRequstedDate(new Date(dateRange.endDate));
|
|
@@ -22616,7 +21654,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22616
21654
|
* @param {Array<string>} options.fields - Array of fields to fetch
|
|
22617
21655
|
* @param {Object} options.storage - Storage instance
|
|
22618
21656
|
*/
|
|
22619
|
-
processCatalogNode({ nodeName, fields }) {
|
|
21657
|
+
async processCatalogNode({ nodeName, fields }) {
|
|
22620
21658
|
console.log(`Catalog node processing not implemented for ${nodeName}`);
|
|
22621
21659
|
}
|
|
22622
21660
|
/**
|
|
@@ -22624,7 +21662,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22624
21662
|
* @param {string} nodeName - Name of the node
|
|
22625
21663
|
* @returns {Object} Storage instance
|
|
22626
21664
|
*/
|
|
22627
|
-
getStorageByNode(nodeName) {
|
|
21665
|
+
async getStorageByNode(nodeName) {
|
|
22628
21666
|
if (!("storages" in this)) {
|
|
22629
21667
|
this.storages = {};
|
|
22630
21668
|
}
|
|
@@ -22642,6 +21680,7 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22642
21680
|
this.source.fieldsSchema[nodeName].fields,
|
|
22643
21681
|
`${this.source.fieldsSchema[nodeName].description} ${this.source.fieldsSchema[nodeName].documentation}`
|
|
22644
21682
|
);
|
|
21683
|
+
await this.storages[nodeName].init();
|
|
22645
21684
|
}
|
|
22646
21685
|
return this.storages[nodeName];
|
|
22647
21686
|
}
|
|
@@ -22657,8 +21696,8 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22657
21696
|
const endDate = new Date(startDate);
|
|
22658
21697
|
endDate.setDate(endDate.getDate() + daysToFetch - 1);
|
|
22659
21698
|
return {
|
|
22660
|
-
startDate:
|
|
22661
|
-
endDate:
|
|
21699
|
+
startDate: DateUtils2.formatDate(startDate),
|
|
21700
|
+
endDate: DateUtils2.formatDate(endDate)
|
|
22662
21701
|
};
|
|
22663
21702
|
}
|
|
22664
21703
|
};
|
|
@@ -22704,7 +21743,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22704
21743
|
"BankOfCanada"
|
|
22705
21744
|
];
|
|
22706
21745
|
const AvailableStorages = [
|
|
22707
|
-
"GoogleSheets",
|
|
22708
21746
|
"GoogleBigQuery",
|
|
22709
21747
|
"AwsAthena"
|
|
22710
21748
|
];
|
|
@@ -22729,7 +21767,6 @@ API Response: ${JSON.stringify(statusResult, null, 2)}`);
|
|
|
22729
21767
|
CriteoAds,
|
|
22730
21768
|
BankOfCanada,
|
|
22731
21769
|
// Individual storages
|
|
22732
|
-
GoogleSheets,
|
|
22733
21770
|
GoogleBigQuery,
|
|
22734
21771
|
AwsAthena
|
|
22735
21772
|
};
|