@transcend-io/cli 6.26.7 → 6.26.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* @param options - Options
|
|
5
5
|
* @returns Number of items marked as completed
|
|
6
6
|
*/
|
|
7
|
-
export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency, transcendUrl, }: {
|
|
7
|
+
export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency, transcendUrl, sleepSeconds, }: {
|
|
8
8
|
/** CSV file path */
|
|
9
9
|
file: string;
|
|
10
10
|
/** Transcend API key authentication */
|
|
@@ -17,5 +17,7 @@ export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, som
|
|
|
17
17
|
transcendUrl?: string;
|
|
18
18
|
/** Sombra API key authentication */
|
|
19
19
|
sombraAuth?: string;
|
|
20
|
+
/** Sleep time in seconds between chunks of concurrent calls */
|
|
21
|
+
sleepSeconds?: number;
|
|
20
22
|
}): Promise<number>;
|
|
21
23
|
//# sourceMappingURL=pushCronIdentifiersFromCsv.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pushCronIdentifiersFromCsv.d.ts","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pushCronIdentifiersFromCsv.d.ts","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":"AAaA;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAiB,EACjB,YAAoC,EACpC,YAAiB,GAClB,EAAE;IACD,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0GlB"}
|
|
@@ -12,13 +12,14 @@ const cli_progress_1 = __importDefault(require("cli-progress"));
|
|
|
12
12
|
const logger_1 = require("../logger");
|
|
13
13
|
const requests_1 = require("../requests");
|
|
14
14
|
const constants_1 = require("../constants");
|
|
15
|
+
const chunk_1 = __importDefault(require("lodash/chunk"));
|
|
15
16
|
/**
|
|
16
17
|
* Given a CSV of cron job outputs, mark all requests as completed in Transcend
|
|
17
18
|
*
|
|
18
19
|
* @param options - Options
|
|
19
20
|
* @returns Number of items marked as completed
|
|
20
21
|
*/
|
|
21
|
-
async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
|
|
22
|
+
async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, sleepSeconds = 10, }) {
|
|
22
23
|
// Create sombra instance to communicate with
|
|
23
24
|
const sombra = await (0, graphql_1.createSombraGotInstance)(transcendUrl, auth, sombraAuth);
|
|
24
25
|
// Read from CSV
|
|
@@ -34,22 +35,38 @@ async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth,
|
|
|
34
35
|
let failureCount = 0;
|
|
35
36
|
let errorCount = 0;
|
|
36
37
|
progressBar.start(activeResults.length, 0);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
// Process in chunks with sleep intervals
|
|
39
|
+
const chunks = (0, chunk_1.default)(activeResults, concurrency);
|
|
40
|
+
const totalChunks = chunks.length;
|
|
41
|
+
const processChunk = async (items, chunkIndex) => {
|
|
42
|
+
logger_1.logger.info(colors_1.default.blue(`Processing chunk ${chunkIndex + 1}/${totalChunks} (${chunk_1.default.length} items)`));
|
|
43
|
+
// Process the items of the chunk concurrently
|
|
44
|
+
await (0, bluebird_1.map)(items, async (identifier) => {
|
|
45
|
+
try {
|
|
46
|
+
const success = await (0, markCronIdentifierCompleted_1.markCronIdentifierCompleted)(sombra, identifier);
|
|
47
|
+
if (success) {
|
|
48
|
+
successCount += 1;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
failureCount += 1;
|
|
52
|
+
}
|
|
42
53
|
}
|
|
43
|
-
|
|
44
|
-
|
|
54
|
+
catch (e) {
|
|
55
|
+
logger_1.logger.error(colors_1.default.red(`Error notifying Transcend for identifier "${identifier.identifier}" - ${e === null || e === void 0 ? void 0 : e.message}`));
|
|
56
|
+
errorCount += 1;
|
|
45
57
|
}
|
|
58
|
+
progressBar.update(successCount + failureCount);
|
|
59
|
+
});
|
|
60
|
+
// Sleep between chunks (except for the last chunk)
|
|
61
|
+
if (sleepSeconds > 0 && chunkIndex < totalChunks - 1) {
|
|
62
|
+
logger_1.logger.info(colors_1.default.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`));
|
|
63
|
+
await new Promise((resolve) => {
|
|
64
|
+
setTimeout(resolve, sleepSeconds * 1000);
|
|
65
|
+
});
|
|
46
66
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
51
|
-
progressBar.update(successCount + failureCount);
|
|
52
|
-
}, { concurrency });
|
|
67
|
+
};
|
|
68
|
+
// Process all chunks sequentially
|
|
69
|
+
await (0, bluebird_1.mapSeries)(chunks, processChunk);
|
|
53
70
|
progressBar.stop();
|
|
54
71
|
const t1 = new Date().getTime();
|
|
55
72
|
const totalTime = t1 - t0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pushCronIdentifiersFromCsv.js","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":";;;;;;AAAA,
|
|
1
|
+
{"version":3,"file":"pushCronIdentifiersFromCsv.js","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA0C;AAC1C,wCAAqD;AACrD,oDAA4B;AAC5B,+EAGuC;AACvC,gEAAuC;AACvC,sCAAmC;AACnC,0CAAsC;AACtC,4CAAqD;AACrD,yDAAiC;AAEjC;;;;;GAKG;AACI,KAAK,UAAU,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,EACpC,YAAY,GAAG,EAAE,GAgBlB;IACC,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,gDAAkB,CAAC,CAAC;IAExD,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,sCAAsC,UAAU,cAAc,aAAa,CAAC,MAAM,6BAA6B,CAChH,CACF,CAAC;IAEF,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAE3C,yCAAyC;IACzC,MAAM,MAAM,GAAG,IAAA,eAAK,EAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,MAAM,YAAY,GAAG,KAAK,EACxB,KAA2B,EAC3B,UAAkB,EACH,EAAE;QACjB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,IAAI,CACT,oBAAoB,UAAU,GAAG,CAAC,IAAI,WAAW,KAC/C,eAAK,CAAC,MACR,SAAS,CACV,CACF,CAAC;QAEF,8CAA8C;QAC9C,MAAM,IAAA,cAAG,EAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;YACpC,IAAI;gBACF,MAAM,OAAO,GAAG,MAAM,IAAA,yDAA2B,EAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACtE,IAAI,OAAO,EAAE;oBACX,YAAY,IAAI,CAAC,CAAC;iBACnB;qBAAM;oBACL,YAAY,IAAI,CAAC,CAAC;iBACnB;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6CAA6C,UAAU,CAAC,UAAU,OAAO,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,OAAO,EAAE,CACtF,CACF,CAAC;gBACF,UAAU,IAAI,CAAC,CAAC;aACjB;YACD,WAAW,CAAC,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,mDAAmD;QACnD,IAAI,YAAY,GAAG,CAAC,IAAI,UAAU,GAAG,WAAW,GAAG,CAAC,EAAE;YACpD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CAAC,gBAAgB,YAAY,wBAAwB,CAAC,CACpE,CAAC;YAEF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,UAAU,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;SACJ;IACH,CAAC,CAAC;IAEF,kCAAkC;IAClC,MAAM,IAAA,oBAAS,EAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEtC,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uCAAuC,YAAY,oBACjD,SAAS,GAAG,IACd,YAAY,CACb,CACF,CAAC;IACF,IAAI,YAAY,EAAE;QAChB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,cAAc,YAAY,sDAAsD;YAC9E,yCAAyC,CAC5C,CACF,CAAC;KACH;IACD,IAAI,UAAU,EAAE;QACd,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,cAAc,UAAU,sFAAsF,CAC/G,CACF,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IACD,OAAO,aAAa,CAAC,MAAM,CAAC;AAC9B,CAAC;AAjID,gEAiIC"}
|