@transcend-io/cli 6.26.6 → 6.26.8

package/build/cli-cron-pull-identifiers.js

@@ -36,7 +36,7 @@ const requests_1 = require("./requests");
  */
 async function main() {
     // Parse command line arguments
-    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, actions, pageLimit = '100', skipRequestCount = false, chunkSize = '100000', } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, actions, pageLimit = '100', skipRequestCount = false, chunkSize = '10000', } = (0, yargs_parser_1.default)(process.argv.slice(2));
     // Ensure auth is passed
     if (!auth) {
         logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
@@ -63,34 +63,37 @@ async function main() {
             `Expected one of: \n${Object.values(privacy_types_1.RequestAction).join('\n')}`));
         process.exit(1);
     }
-    // Validate chunk size
+    const parsedPageLimit = parseInt(pageLimit, 10);
     const parsedChunkSize = parseInt(chunkSize, 10);
-    if (Number.isNaN(parsedChunkSize) || parsedChunkSize <= 0) {
-        logger_1.logger.error(colors_1.default.red(`Invalid chunk size: "${chunkSize}". Must be a positive integer.`));
+    if (Number.isNaN(parsedChunkSize) ||
+        parsedChunkSize <= 0 ||
+        parsedChunkSize % parsedPageLimit !== 0) {
+        logger_1.logger.error(colors_1.default.red(`Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${parsedPageLimit}.`));
         process.exit(1);
     }
+    const { baseName, extension } = (0, cron_1.parseFilePath)(file);
+    let fileCount = 0;
+    const onSave = (chunk) => {
+        const numberedFileName = `${baseName}-${fileCount}${extension}`;
+        logger_1.logger.info(colors_1.default.blue(`Saving ${chunk.length} identifiers to file "${numberedFileName}"`));
+        const headers = (0, uniq_1.default)(chunk.map((d) => Object.keys(d)).flat());
+        (0, cron_1.writeCsv)(numberedFileName, chunk, headers);
+        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${chunk.length} identifiers to file "${numberedFileName}"`));
+        fileCount += 1;
+        return Promise.resolve();
+    };
     // Pull down outstanding identifiers
-    const { identifiersFormattedForCsv } = await (0, cron_1.pullCustomSiloOutstandingIdentifiers)({
+    await (0, cron_1.pullChunkedCustomSiloOutstandingIdentifiers)({
         transcendUrl,
-        pageLimit: parseInt(pageLimit, 10),
+        apiPageSize: parsedPageLimit,
+        savePageSize: parsedChunkSize,
+        onSave,
         actions: parsedActions,
         auth,
         sombraAuth,
         dataSiloId,
         skipRequestCount: skipRequestCount === 'true',
     });
-    // Write CSV (split into multiple files if too large)
-    const headers = (0, uniq_1.default)(identifiersFormattedForCsv.map((d) => Object.keys(d)).flat());
-    const writtenFiles = await (0, cron_1.writeLargeCsv)(file, identifiersFormattedForCsv, headers, parsedChunkSize);
-    if (writtenFiles.length === 1) {
-        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to file "${file}"`));
-    }
-    else {
-        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to ${writtenFiles.length} files:`));
-        writtenFiles.forEach((fileName) => {
-            logger_1.logger.info(colors_1.default.green(`  - ${fileName}`));
-        });
-    }
 }
 main();
 //# sourceMappingURL=cli-cron-pull-identifiers.js.map

package/build/cli-cron-pull-identifiers.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAA6E;AAC7E,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAE5C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,OAAO,EACP,SAAS,GAAG,KAAK,EACjB,gBAAgB,GAAG,KAAK,EACxB,SAAS,GAAG,QAAQ,GACrB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,gBAAgB,KAAK,MAAM,EAAE;QAC/B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CACX,uEAAuE,CACxE,CACF,CAAC;KACH;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,sBAAsB;IACtB,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE;QACzD,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,wBAAwB,SAAS,gCAAgC,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oCAAoC;IACpC,MAAM,EAAE,0BAA0B,EAAE,GAClC,MAAM,IAAA,2CAAoC,EAAC;QACzC,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU;QACV,gBAAgB,EAAE,gBAAgB,KAAK,MAAM;KAC9C,CAAC,CAAC;IAEL,qDAAqD;IACrD,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,IAAA,oBAAa,EACtC,IAAI,EACJ,0BAA0B,EAC1B,OAAO,EACP,eAAe,CAChB,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC7B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACxF,CACF,CAAC;KACH;SAAM;QACL,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,mBAAmB,YAAY,CAAC,MAAM,SAAS,CACvG,CACF,CAAC;QACF,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChC,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,KAAK,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAKgB;AAChB,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAE5C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,OAAO,EACP,SAAS,GAAG,KAAK,EACjB,gBAAgB,GAAG,KAAK,EACxB,SAAS,GAAG,OAAO,GACpB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,gBAAgB,KAAK,MAAM,EAAE;QAC/B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CACX,uEAAuE,CACxE,CACF,CAAC;KACH;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IACE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC;QAC7B,eAAe,IAAI,CAAC;QACpB,eAAe,GAAG,eAAe,KAAK,CAAC,EACvC;QACA,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,wBAAwB,SAAS,uDAAuD,eAAe,GAAG,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IACD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,oBAAa,EAAC,IAAI,CAAC,CAAC;IACpD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,MAAM,GAAG,CAAC,KAA+B,EAAiB,EAAE;QAChE,MAAM,gBAAgB,GAAG,GAAG,QAAQ,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;QAChE,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,IAAI,CACT,UAAU,KAAK,CAAC,MAAM,yBAAyB,gBAAgB,GAAG,CACnE,CACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAA,cAAI,EAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,IAAA,eAAQ,EAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,KAAK,CAAC,MAAM,yBAAyB,gBAAgB,GAAG,CAC/E,CACF,CAAC;QACF,SAAS,IAAI,CAAC,CAAC;QACf,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC,CAAC;IAEF,oCAAoC;IACpC,MAAM,IAAA,kDAA2C,EAAC;QAChD,YAAY;QACZ,WAAW,EAAE,eAAe;QAC5B,YAAY,EAAE,eAAe;QAC7B,MAAM;QACN,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU;QACV,gBAAgB,EAAE,gBAAgB,KAAK,MAAM;KAC9C,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-cron-pull-profiles.js

@@ -8,12 +8,13 @@ const yargs_parser_1 = __importDefault(require("yargs-parser"));
 const colors_1 = __importDefault(require("colors"));
 const logger_1 = require("./logger");
 const uniq_1 = __importDefault(require("lodash/uniq"));
-const cron_1 = require("./cron");
+const pullChunkedCustomSiloOutstandingIdentifiers_1 = require("./cron/pullChunkedCustomSiloOutstandingIdentifiers");
 const privacy_types_1 = require("@transcend-io/privacy-types");
 const constants_1 = require("./constants");
 const requests_1 = require("./requests");
 const bluebird_1 = require("bluebird");
 const graphql_1 = require("./graphql");
+const writeCsv_1 = require("./cron/writeCsv");
 /**
  * This is a temporary script that can be removed after the launch of workflows v2
  * TODO: https://transcend.height.app/T-39035 - remove this
@@ -39,7 +40,7 @@ const graphql_1 = require("./graphql");
  */
 async function main() {
     // Parse command line arguments
-    const { file = './cron-identifiers.csv', fileTarget = './cron-identifiers-target.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, cronDataSiloId, targetDataSiloId, actions, pageLimit = '100', } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    const { file = './cron-identifiers.csv', fileTarget = './cron-identifiers-target.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, cronDataSiloId, targetDataSiloId, actions, skipRequestCount = false, pageLimit = '100', chunkSize = '10000', } = (0, yargs_parser_1.default)(process.argv.slice(2));
     // Ensure auth is passed
     if (!auth) {
         logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
@@ -60,6 +61,9 @@ async function main() {
         logger_1.logger.error(colors_1.default.red('At least one action must be provided. You can specify using --actions=ERASURE'));
         process.exit(1);
     }
+    if (skipRequestCount === 'true') {
+        logger_1.logger.info(colors_1.default.yellow('Skipping request count as requested. This may help speed up the call.'));
+    }
     // Validate actions
     const parsedActions = (0, requests_1.splitCsvToList)(actions);
     const invalidActions = parsedActions.filter(
@@ -70,50 +74,79 @@ async function main() {
             `Expected one of: \n${Object.values(privacy_types_1.RequestAction).join('\n')}`));
         process.exit(1);
     }
-    // Pull down outstanding identifiers
-    const { identifiersFormattedForCsv } = await (0, cron_1.pullCustomSiloOutstandingIdentifiers)({
-        transcendUrl,
-        pageLimit: parseInt(pageLimit, 10),
-        actions: parsedActions,
-        auth,
-        sombraAuth,
-        dataSiloId: cronDataSiloId,
-    });
-    // Grab the requestIds from the list of silos to process
-    const requestIds = identifiersFormattedForCsv.map((d) => d.requestId);
+    const parsedPageLimit = parseInt(pageLimit, 10);
+    const parsedChunkSize = parseInt(chunkSize, 10);
+    if (Number.isNaN(parsedChunkSize) ||
+        parsedChunkSize <= 0 ||
+        parsedChunkSize % parsedPageLimit !== 0) {
+        logger_1.logger.error(colors_1.default.red(`Invalid chunk size: "${chunkSize}". Must be a positive integer that is a multiple of ${parsedPageLimit}.`));
+        process.exit(1);
+    }
     // Create GraphQL client to connect to Transcend backend
     const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
-    // Pull down target identifiers
-    const results = await (0, bluebird_1.map)((0, uniq_1.default)(requestIds), async (requestId) => {
-        const results = await (0, graphql_1.fetchRequestFilesForRequest)(client, {
-            requestId,
-            dataSiloId: targetDataSiloId,
-        });
-        return results.map(({ fileName, remoteId }) => {
-            var _a;
-            if (!remoteId) {
-                throw new Error(`Failed to find remoteId for ${fileName} request: ${requestId}`);
-            }
-            return {
-                RecordId: remoteId,
-                Object: (_a = fileName
-                    .replace('.json', '')
-                    .split('/')
-                    .pop()) === null || _a === void 0 ? void 0 : _a.replace(' Information', ''),
-                Comment: 'Customer data deletion request submitted via transcend.io',
-            };
+    const { baseName, extension } = (0, writeCsv_1.parseFilePath)(file);
+    const { baseName: baseNameTarget, extension: extensionTarget } = (0, writeCsv_1.parseFilePath)(fileTarget);
+    let allIdentifiersCount = 0;
+    let allTargetIdentifiersCount = 0;
+    let fileCount = 0;
+    // Create onSave callback to handle chunked processing
+    const onSave = async (chunk) => {
+        // Add to all identifiers
+        allIdentifiersCount += chunk.length;
+        // Get unique request IDs from this chunk
+        const requestIds = chunk.map((d) => d.requestId);
+        const uniqueRequestIds = (0, uniq_1.default)(requestIds);
+        // Pull down target identifiers for this chunk
+        const results = await (0, bluebird_1.map)(uniqueRequestIds, async (requestId) => {
+            const results = await (0, graphql_1.fetchRequestFilesForRequest)(client, {
+                requestId,
+                dataSiloId: targetDataSiloId,
+            });
+            return results.map(({ fileName, remoteId }) => {
+                var _a;
+                if (!remoteId) {
+                    throw new Error(`Failed to find remoteId for ${fileName} request: ${requestId}`);
+                }
+                return {
+                    RecordId: remoteId,
+                    Object: (_a = fileName
+                        .replace('.json', '')
+                        .split('/')
+                        .pop()) === null || _a === void 0 ? void 0 : _a.replace(' Information', ''),
+                    Comment: 'Customer data deletion request submitted via transcend.io',
+                };
+            });
+        }, {
+            concurrency: 10,
         });
-    }, {
-        concurrency: 10,
+        allTargetIdentifiersCount += results.flat().length;
+        // Write the identifiers and target identifiers to CSV
+        const headers = (0, uniq_1.default)(chunk.map((d) => Object.keys(d)).flat());
+        const numberedFileName = `${baseName}-${fileCount}${extension}`;
+        const numberedFileNameTarget = `${baseNameTarget}-${fileCount}${extensionTarget}`;
+        (0, writeCsv_1.writeCsv)(numberedFileName, chunk, headers);
+        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${chunk.length} identifiers to file "${file}"`));
+        const targetIdentifiers = results.flat();
+        const headers2 = (0, uniq_1.default)(targetIdentifiers.map((d) => Object.keys(d)).flat());
+        (0, writeCsv_1.writeCsv)(numberedFileNameTarget, targetIdentifiers, headers2);
+        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"`));
+        logger_1.logger.info(colors_1.default.blue(`Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`));
+        fileCount += 1;
+    };
+    // Pull down outstanding identifiers using the new chunked function
+    await (0, pullChunkedCustomSiloOutstandingIdentifiers_1.pullChunkedCustomSiloOutstandingIdentifiers)({
+        dataSiloId: cronDataSiloId,
+        auth,
+        sombraAuth,
+        actions: parsedActions,
+        apiPageSize: parsedPageLimit,
+        savePageSize: parsedChunkSize,
+        onSave,
+        transcendUrl,
+        skipRequestCount: skipRequestCount === 'true',
     });
-    // Write CSV
-    const headers = (0, uniq_1.default)(identifiersFormattedForCsv.map((d) => Object.keys(d)).flat());
-    (0, cron_1.writeCsv)(file, identifiersFormattedForCsv, headers);
-    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to file "${file}"`));
-    const targetIdentifiers = results.flat();
-    const headers2 = (0, uniq_1.default)(targetIdentifiers.map((d) => Object.keys(d)).flat());
-    (0, cron_1.writeCsv)(fileTarget, targetIdentifiers, headers2);
-    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${targetIdentifiers.length} identifiers to file "${fileTarget}"`));
+    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${allIdentifiersCount} identifiers to file "${file}"`));
+    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${allTargetIdentifiersCount} identifiers to file "${fileTarget}"`));
 }
 main();
 //# sourceMappingURL=cli-cron-pull-profiles.js.map

package/build/cli-cron-pull-profiles.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-cron-pull-profiles.js","sourceRoot":"","sources":["../src/cli-cron-pull-profiles.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAAwE;AACxE,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAC5C,uCAA+B;AAC/B,uCAGmB;AAEnB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,UAAU,GAAG,+BAA+B,EAC5C,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,OAAO,EACP,SAAS,GAAG,KAAK,GAClB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,IAAI,CAAC,cAAc,EAAE;QACnB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,gHAAgH,CACjH,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IAAI,CAAC,gBAAgB,EAAE;QACrB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,oHAAoH,CACrH,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,iBAAiB;IACjB,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oCAAoC;IACpC,MAAM,EAAE,0BAA0B,EAAE,GAClC,MAAM,IAAA,2CAAoC,EAAC;QACzC,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU,EAAE,cAAc;KAC3B,CAAC,CAAC;IAEL,wDAAwD;IACxD,MAAM,UAAU,GAAG,0BAA0B,CAAC,GAAG,CAC/C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAC7B,CAAC;IAEF,wDAAwD;IACxD,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,+BAA+B;IAC/B,MAAM,OAAO,GAAG,MAAM,IAAA,cAAG,EACvB,IAAA,cAAI,EAAC,UAAU,CAAC,EAChB,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,OAAO,GAAG,MAAM,IAAA,qCAA2B,EAAC,MAAM,EAAE;YACxD,SAAS;YACT,UAAU,EAAE,gBAAgB;SAC7B,CAAC,CAAC;QACH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;;YAC5C,IAAI,CAAC,QAAQ,EAAE;gBACb,MAAM,IAAI,KAAK,CACb,+BAA+B,QAAQ,aAAa,SAAS,EAAE,CAChE,CAAC;aACH;YACD,OAAO;gBACL,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,MAAA,QAAQ;qBACb,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;qBACpB,KAAK,CAAC,GAAG,CAAC;qBACV,GAAG,EAAE,0CACJ,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/B,OAAO,EAAE,2DAA2D;aACrE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,EACD;QACE,WAAW,EAAE,EAAE;KAChB,CACF,CAAC;IAEF,YAAY;IACZ,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;IACF,IAAA,eAAQ,EAAC,IAAI,EAAE,0BAA0B,EAAE,OAAO,CAAC,CAAC;IACpD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACxF,CACF,CAAC;IAEF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAA,cAAI,EAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,IAAA,eAAQ,EAAC,UAAU,EAAE,iBAAiB,EAAE,QAAQ,CAAC,CAAC;IAClD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,iBAAiB,CAAC,MAAM,yBAAyB,UAAU,GAAG,CACrF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-cron-pull-profiles.js","sourceRoot":"","sources":["../src/cli-cron-pull-profiles.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,oHAG4D;AAC5D,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAC5C,uCAA+B;AAC/B,uCAGmB;AACnB,8CAA0D;AAE1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,UAAU,GAAG,+BAA+B,EAC5C,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,OAAO,EACP,gBAAgB,GAAG,KAAK,EACxB,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,OAAO,GACpB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,IAAI,CAAC,cAAc,EAAE;QACnB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,gHAAgH,CACjH,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IAAI,CAAC,gBAAgB,EAAE;QACrB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,oHAAoH,CACrH,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,iBAAiB;IACjB,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,gBAAgB,KAAK,MAAM,EAAE;QAC/B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CACX,uEAAuE,CACxE,CACF,CAAC;KACH;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IACE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC;QAC7B,eAAe,IAAI,CAAC;QACpB,eAAe,GAAG,eAAe,KAAK,CAAC,EACvC;QACA,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,wBAAwB,SAAS,uDAAuD,eAAe,GAAG,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wDAAwD;IACxD,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAA,wBAAa,EAAC,IAAI,CAAC,CAAC;IACpD,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,GAC5D,IAAA,wBAAa,EAAC,UAAU,CAAC,CAAC;IAE5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,yBAAyB,GAAG,CAAC,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,sDAAsD;IACtD,MAAM,MAAM,GAAG,KAAK,EAAE,KAA+B,EAAiB,EAAE;QACtE,yBAAyB;QACzB,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC;QAEpC,yCAAyC;QACzC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAC;QAC3D,MAAM,gBAAgB,GAAG,IAAA,cAAI,EAAC,UAAU,CAAC,CAAC;QAE1C,8CAA8C;QAC9C,MAAM,OAAO,GAAG,MAAM,IAAA,cAAG,EACvB,gBAAgB,EAChB,KAAK,EAAE,SAAS,EAAE,EAAE;YAClB,MAAM,OAAO,GAAG,MAAM,IAAA,qCAA2B,EAAC,MAAM,EAAE;gBACxD,SAAS;gBACT,UAAU,EAAE,gBAAgB;aAC7B,CAAC,CAAC;YACH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;;gBAC5C,IAAI,CAAC,QAAQ,EAAE;oBACb,MAAM,IAAI,KAAK,CACb,+BAA+B,QAAQ,aAAa,SAAS,EAAE,CAChE,CAAC;iBACH;gBACD,OAAO;oBACL,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,MAAA,QAAQ;yBACb,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;yBACpB,KAAK,CAAC,GAAG,CAAC;yBACV,GAAG,EAAE,0CACJ,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;oBAC/B,OAAO,EACL,2DAA2D;iBAC9D,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,EACD;YACE,WAAW,EAAE,EAAE;SAChB,CACF,CAAC;QAEF,yBAAyB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;QAEnD,sDAAsD;QACtD,MAAM,OAAO,GAAG,IAAA,cAAI,EAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,MAAM,gBAAgB,GAAG,GAAG,QAAQ,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;QAChE,MAAM,sBAAsB,GAAG,GAAG,cAAc,IAAI,SAAS,GAAG,eAAe,EAAE,CAAC;QAClF,IAAA,mBAAQ,EAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3C,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,KAAK,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACnE,CACF,CAAC;QAEF,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAA,cAAI,EAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3E,IAAA,mBAAQ,EAAC,sBAAsB,EAAE,iBAAiB,EAAE,QAAQ,CAAC,CAAC;QAC9D,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,iBAAiB,CAAC,MAAM,yBAAyB,UAAU,GAAG,CACrF,CACF,CAAC;QAEF,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,IAAI,CACT,sBAAsB,KAAK,CAAC,MAAM,uBAAuB,iBAAiB,CAAC,MAAM,qBAAqB,CACvG,CACF,CAAC;QACF,SAAS,IAAI,CAAC,CAAC;IACjB,CAAC,CAAC;IAEF,mEAAmE;IACnE,MAAM,IAAA,yFAA2C,EAAC;QAChD,UAAU,EAAE,cAAc;QAC1B,IAAI;QACJ,UAAU;QACV,OAAO,EAAE,aAAa;QACtB,WAAW,EAAE,eAAe;QAC5B,YAAY,EAAE,eAAe;QAC7B,MAAM;QACN,YAAY;QACZ,gBAAgB,EAAE,gBAAgB,KAAK,MAAM;KAC9C,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,mBAAmB,yBAAyB,IAAI,GAAG,CAC1E,CACF,CAAC;IACF,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,yBAAyB,yBAAyB,UAAU,GAAG,CACtF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cron/index.d.ts

@@ -1,7 +1,7 @@
-export * from './pullCustomSiloOutstandingIdentifiers';
 export * from './pullCronPageOfIdentifiers';
 export * from './writeCsv';
 export * from './pushCronIdentifiersFromCsv';
 export * from './markCronIdentifierCompleted';
 export * from './markRequestDataSiloIdsCompleted';
+export * from './pullChunkedCustomSiloOutstandingIdentifiers';
 //# sourceMappingURL=index.d.ts.map

package/build/cron/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":"AAAA,cAAc,wCAAwC,CAAC;AACvD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,YAAY,CAAC;AAC3B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,mCAAmC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,YAAY,CAAC;AAC3B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,mCAAmC,CAAC;AAClD,cAAc,+CAA+C,CAAC"}

package/build/cron/index.js

@@ -14,10 +14,10 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./pullCustomSiloOutstandingIdentifiers"), exports);
 __exportStar(require("./pullCronPageOfIdentifiers"), exports);
 __exportStar(require("./writeCsv"), exports);
 __exportStar(require("./pushCronIdentifiersFromCsv"), exports);
 __exportStar(require("./markCronIdentifierCompleted"), exports);
 __exportStar(require("./markRequestDataSiloIdsCompleted"), exports);
+__exportStar(require("./pullChunkedCustomSiloOutstandingIdentifiers"), exports);
 //# sourceMappingURL=index.js.map

package/build/cron/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yEAAuD;AACvD,8DAA4C;AAC5C,6CAA2B;AAC3B,+DAA6C;AAC7C,gEAA8C;AAC9C,oEAAkD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,8DAA4C;AAC5C,6CAA2B;AAC3B,+DAA6C;AAC7C,gEAA8C;AAC9C,oEAAkD;AAClD,gFAA8D"}

package/build/cron/pullChunkedCustomSiloOutstandingIdentifiers.d.ts

@@ -0,0 +1,44 @@
+import { CronIdentifier } from './pullCronPageOfIdentifiers';
+import { RequestAction } from '@transcend-io/privacy-types';
+/**
+ * A CSV formatted identifier
+ */
+export type CsvFormattedIdentifier = {
+    [k in string]: string | null | boolean | number;
+};
+export interface CronIdentifierWithAction extends CronIdentifier {
+    /** The request action that the identifier relates to */
+    action: RequestAction;
+}
+/**
+ * Pull the set of identifiers outstanding for a cron or AVC integration
+ *
+ * This function is designed to be used in a loop, and will call the onSave callback
+ * with a chunk of identifiers when the savePageSize is reached.
+ *
+ * @param options - Options
+ */
+export declare function pullChunkedCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAuth, actions, apiPageSize, savePageSize, onSave, transcendUrl, skipRequestCount, }: {
+    /** Transcend API key authentication */
+    auth: string;
+    /** Data Silo ID to pull down jobs for */
+    dataSiloId: string;
+    /** The request actions to fetch */
+    actions: RequestAction[];
+    /** How many identifiers to pull in a single call to the backend */
+    apiPageSize: number;
+    /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */
+    savePageSize: number;
+    /** Callback function called when a chunk of identifiers is ready to be saved */
+    onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;
+    /** API URL for Transcend backend */
+    transcendUrl?: string;
+    /** Sombra API key authentication */
+    sombraAuth?: string;
+    /** Skip request count */
+    skipRequestCount?: boolean;
+}): Promise<{
+    /** Raw Identifiers */
+    identifiers: CronIdentifierWithAction[];
+}>;
+//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers.d.ts.map

package/build/cron/pullChunkedCustomSiloOutstandingIdentifiers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers.d.ts","sourceRoot":"","sources":["../../src/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"names":[],"mappings":"AAOA,OAAO,EAEL,cAAc,EACf,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAI5D;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;KAClC,CAAC,IAAI,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,GAAG,MAAM;CAChD,CAAC;AAEF,MAAM,WAAW,wBAAyB,SAAQ,cAAc;IAC9D,wDAAwD;IACxD,MAAM,EAAE,aAAa,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,wBAAsB,2CAA2C,CAAC,EAChE,UAAU,EACV,IAAI,EACJ,UAAU,EACV,OAAO,EACP,WAAiB,EACjB,YAAmB,EACnB,MAAM,EACN,YAAoC,EACpC,gBAAwB,GACzB,EAAE;IACD,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,0GAA0G;IAC1G,YAAY,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,MAAM,EAAE,CAAC,KAAK,EAAE,sBAAsB,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3D,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GAAG,OAAO,CAAC;IACV,sBAAsB;IACtB,WAAW,EAAE,wBAAwB,EAAE,CAAC;CACzC,CAAC,CAkID"}

package/build/cron/{pullCustomSiloOutstandingIdentifiers.js → pullChunkedCustomSiloOutstandingIdentifiers.js}

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pullCustomSiloOutstandingIdentifiers = void 0;
+exports.pullChunkedCustomSiloOutstandingIdentifiers = void 0;
 const graphql_1 = require("../graphql");
 const colors_1 = __importDefault(require("colors"));
 const cli_progress_1 = __importDefault(require("cli-progress"));
@@ -14,9 +14,16 @@ const bluebird_1 = require("bluebird");
 /**
  * Pull the set of identifiers outstanding for a cron or AVC integration
  *
+ * This function is designed to be used in a loop, and will call the onSave callback
+ * with a chunk of identifiers when the savePageSize is reached.
+ *
  * @param options - Options
  */
-async function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAuth, actions, pageLimit = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, skipRequestCount = false, }) {
+async function pullChunkedCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAuth, actions, apiPageSize = 100, savePageSize = 1000, onSave, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, skipRequestCount = false, }) {
+    // Validate savePageSize
+    if (savePageSize % apiPageSize !== 0) {
+        throw new Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`);
+    }
     // Create sombra instance to communicate with
     const sombra = await (0, graphql_1.createSombraGotInstance)(transcendUrl, auth, sombraAuth);
     // Create GraphQL client to connect to Transcend backend
@@ -36,6 +43,8 @@ async function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAu
     const foundRequestIds = new Set();
     // identifiers found in total
     const identifiers = [];
+    // current chunk of identifiers to be saved
+    let currentChunk = [];
     // map over each action
     if (!skipRequestCount) {
         progressBar.start(totalRequestCount, 0);
@@ -48,19 +57,33 @@ async function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAu
             // eslint-disable-next-line no-await-in-loop
             const pageIdentifiers = await (0, pullCronPageOfIdentifiers_1.pullCronPageOfIdentifiers)(sombra, {
                 dataSiloId,
-                limit: pageLimit,
+                limit: apiPageSize,
                 offset,
                 requestType: action,
             });
-            identifiers.push(...pageIdentifiers.map((identifier) => {
+            const identifiersWithAction = pageIdentifiers.map((identifier) => {
                 foundRequestIds.add(identifier.requestId);
                 return {
                     ...identifier,
                     action,
                 };
+            });
+            const csvFormattedIdentifiers = identifiersWithAction.map(({ attributes, ...identifier }) => ({
+                ...identifier,
+                ...attributes.reduce((acc, val) => Object.assign(acc, {
+                    [val.key]: val.values.join(','),
+                }), {}),
             }));
-            shouldContinue = pageIdentifiers.length === pageLimit;
-            offset += pageLimit;
+            identifiers.push(...identifiersWithAction);
+            currentChunk.push(...csvFormattedIdentifiers);
+            // Check if we've reached the savePageSize and call the onSave callback
+            if (currentChunk.length >= savePageSize) {
+                // eslint-disable-next-line no-await-in-loop
+                await onSave(currentChunk);
+                currentChunk = [];
+            }
+            shouldContinue = pageIdentifiers.length === apiPageSize;
+            offset += apiPageSize;
             if (!skipRequestCount) {
                 progressBar.update(foundRequestIds.size);
             }
@@ -69,20 +92,17 @@ async function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAu
             }
         }
     });
+    // Save any remaining identifiers in the current chunk
+    if (currentChunk.length > 0) {
+        await onSave(currentChunk);
+    }
     if (!skipRequestCount) {
         progressBar.stop();
     }
     const t1 = new Date().getTime();
     const totalTime = t1 - t0;
     logger_1.logger.info(colors_1.default.green(`Successfully pulled ${identifiers.length} outstanding identifiers from ${foundRequestIds.size} requests in "${totalTime / 1000}" seconds!`));
-    // Write out to CSV
-    const data = identifiers.map(({ attributes, ...identifier }) => ({
-        ...identifier,
-        ...attributes.reduce((acc, val) => Object.assign(acc, {
-            [val.key]: val.values.join(','),
-        }), {}),
-    }));
-    return { identifiers, identifiersFormattedForCsv: data };
+    return { identifiers };
 }
-exports.pullCustomSiloOutstandingIdentifiers = pullCustomSiloOutstandingIdentifiers;
-//# sourceMappingURL=pullCustomSiloOutstandingIdentifiers.js.map
+exports.pullChunkedCustomSiloOutstandingIdentifiers = pullChunkedCustomSiloOutstandingIdentifiers;
+//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers.js.map

package/build/cron/pullChunkedCustomSiloOutstandingIdentifiers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers.js","sourceRoot":"","sources":["../../src/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"names":[],"mappings":";;;;;;AAAA,wCAIoB;AACpB,oDAA4B;AAC5B,gEAAuC;AACvC,2EAGqC;AAErC,sCAAmC;AACnC,4CAAqD;AACrD,uCAAqC;AAarC;;;;;;;GAOG;AACI,KAAK,UAAU,2CAA2C,CAAC,EAChE,UAAU,EACV,IAAI,EACJ,UAAU,EACV,OAAO,EACP,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,IAAI,EACnB,MAAM,EACN,YAAY,GAAG,iCAAqB,EACpC,gBAAgB,GAAG,KAAK,GAoBzB;IAIC,wBAAwB;IACxB,IAAI,YAAY,GAAG,WAAW,KAAK,CAAC,EAAE;QACpC,MAAM,IAAI,KAAK,CACb,iEAAiE,YAAY,kBAAkB,WAAW,EAAE,CAC7G,CAAC;KACH;IAED,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,wDAAwD;IACxD,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,gBAAgB,EAAE;QACrB,iBAAiB,GAAG,MAAM,IAAA,yCAA+B,EAAC,MAAM,EAAE;YAChE,UAAU;SACX,CAAC,CAAC;KACJ;IAED,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,WACE,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,iBAC7B,mCAAmC;QACjC,mBAAmB,UAAU,4BAA4B,OAAO,CAAC,IAAI,CACnE,MAAM,CACP,GAAG,CACP,CACF,CAAC;IAEF,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,6BAA6B;IAC7B,MAAM,WAAW,GAA+B,EAAE,CAAC;IACnD,2CAA2C;IAC3C,IAAI,YAAY,GAA6B,EAAE,CAAC;IAEhD,uBAAuB;IACvB,IAAI,CAAC,gBAAgB,EAAE;QACrB,WAAW,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;KACzC;IACD,MAAM,IAAA,oBAAS,EAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,cAAc,GAAG,IAAI,CAAC;QAE1B,8BAA8B;QAC9B,OAAO,cAAc,EAAE;YACrB,4CAA4C;YAC5C,MAAM,eAAe,GAAG,MAAM,IAAA,qDAAyB,EAAC,MAAM,EAAE;gBAC9D,UAAU;gBACV,KAAK,EAAE,WAAW;gBAClB,MAAM;gBACN,WAAW,EAAE,MAAM;aACpB,CAAC,CAAC;YAEH,MAAM,qBAAqB,GACzB,eAAe,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;gBACjC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBAC1C,OAAO;oBACL,GAAG,UAAU;oBACb,MAAM;iBACP,CAAC;YACJ,CAAC,CAAC,CAAC;YAEL,MAAM,uBAAuB,GAAG,qBAAqB,CAAC,GAAG,CACvD,CAAC,EAAE,UAAU,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;gBAClC,GAAG,UAAU;gBACb,GAAG,UAAU,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CACX,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;oBACjB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;iBAChC,CAAC,EACJ,EAAE,CACH;aACF,CAAC,CACH,CAAC;YAEF,WAAW,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;YAC3C,YAAY,CAAC,IAAI,CAAC,GAAG,uBAAuB,CAAC,CAAC;YAE9C,uEAAuE;YACvE,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,EAAE;gBACvC,4CAA4C;gBAC5C,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC3B,YAAY,GAAG,EAAE,CAAC;aACnB;YAED,cAAc,GAAG,eAAe,CAAC,MAAM,KAAK,WAAW,CAAC;YACxD,MAAM,IAAI,WAAW,CAAC;YACtB,IAAI,CAAC,gBAAgB,EAAE;gBACrB,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;aAC1C;iBAAM;gBACL,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,UAAU,eAAe,CAAC,MAAM,gCAAgC,eAAe,CAAC,IAAI,WAAW,CAChG,CACF,CAAC;aACH;SACF;IACH,CAAC,CAAC,CAAC;IAEH,sDAAsD;IACtD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3B,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;KAC5B;IAED,IAAI,CAAC,gBAAgB,EAAE;QACrB,WAAW,CAAC,IAAI,EAAE,CAAC;KACpB;IACD,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uBAAuB,WAAW,CAAC,MAAM,iCACvC,eAAe,CAAC,IAClB,iBAAiB,SAAS,GAAG,IAAI,YAAY,CAC9C,CACF,CAAC;IAEF,OAAO,EAAE,WAAW,EAAE,CAAC;AACzB,CAAC;AAlKD,kGAkKC"}

package/build/cron/pushCronIdentifiersFromCsv.d.ts

@@ -4,7 +4,7 @@
  * @param options - Options
  * @returns Number of items marked as completed
  */
-export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency, transcendUrl, }: {
+export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency, transcendUrl, sleepSeconds, }: {
     /** CSV file path */
     file: string;
     /** Transcend API key authentication */
@@ -17,5 +17,7 @@ export declare function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, som
     transcendUrl?: string;
     /** Sombra API key authentication */
     sombraAuth?: string;
+    /** Sleep time in seconds between chunks of concurrent calls */
+    sleepSeconds?: number;
 }): Promise<number>;
 //# sourceMappingURL=pushCronIdentifiersFromCsv.d.ts.map

package/build/cron/pushCronIdentifiersFromCsv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pushCronIdentifiersFromCsv.d.ts","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":"AAYA;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAiB,EACjB,YAAoC,GACrC,EAAE;IACD,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CA8ElB"}
+{"version":3,"file":"pushCronIdentifiersFromCsv.d.ts","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":"AAaA;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAiB,EACjB,YAAoC,EACpC,YAAiB,GAClB,EAAE;IACD,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0GlB"}

package/build/cron/pushCronIdentifiersFromCsv.js

@@ -12,13 +12,14 @@ const cli_progress_1 = __importDefault(require("cli-progress"));
 const logger_1 = require("../logger");
 const requests_1 = require("../requests");
 const constants_1 = require("../constants");
+const chunk_1 = __importDefault(require("lodash/chunk"));
 /**
  * Given a CSV of cron job outputs, mark all requests as completed in Transcend
  *
  * @param options - Options
  * @returns Number of items marked as completed
  */
-async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
+async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, sleepSeconds = 10, }) {
     // Create sombra instance to communicate with
     const sombra = await (0, graphql_1.createSombraGotInstance)(transcendUrl, auth, sombraAuth);
     // Read from CSV
@@ -34,22 +35,38 @@ async function pushCronIdentifiersFromCsv({ file, dataSiloId, auth, sombraAuth,
     let failureCount = 0;
     let errorCount = 0;
     progressBar.start(activeResults.length, 0);
-    await (0, bluebird_1.map)(activeResults, async (identifier) => {
-        try {
-            const success = await (0, markCronIdentifierCompleted_1.markCronIdentifierCompleted)(sombra, identifier);
-            if (success) {
-                successCount += 1;
+    // Process in chunks with sleep intervals
+    const chunks = (0, chunk_1.default)(activeResults, concurrency);
+    const totalChunks = chunks.length;
+    const processChunk = async (items, chunkIndex) => {
+        logger_1.logger.info(colors_1.default.blue(`Processing chunk ${chunkIndex + 1}/${totalChunks} (${chunk_1.default.length} items)`));
+        // Process the items of the chunk concurrently
+        await (0, bluebird_1.map)(items, async (identifier) => {
+            try {
+                const success = await (0, markCronIdentifierCompleted_1.markCronIdentifierCompleted)(sombra, identifier);
+                if (success) {
+                    successCount += 1;
+                }
+                else {
+                    failureCount += 1;
+                }
             }
-            else {
-                failureCount += 1;
+            catch (e) {
+                logger_1.logger.error(colors_1.default.red(`Error notifying Transcend for identifier "${identifier.identifier}" - ${e === null || e === void 0 ? void 0 : e.message}`));
+                errorCount += 1;
             }
+            progressBar.update(successCount + failureCount);
+        });
+        // Sleep between chunks (except for the last chunk)
+        if (sleepSeconds > 0 && chunkIndex < totalChunks - 1) {
+            logger_1.logger.info(colors_1.default.yellow(`Sleeping for ${sleepSeconds}s before next chunk...`));
+            await new Promise((resolve) => {
+                setTimeout(resolve, sleepSeconds * 1000);
+            });
         }
-        catch (e) {
-            logger_1.logger.error(colors_1.default.red(`Error notifying Transcend for identifier "${identifier.identifier}" - ${e.message}`));
-            errorCount += 1;
-        }
-        progressBar.update(successCount + failureCount);
-    }, { concurrency });
+    };
+    // Process all chunks sequentially
+    await (0, bluebird_1.mapSeries)(chunks, processChunk);
     progressBar.stop();
     const t1 = new Date().getTime();
     const totalTime = t1 - t0;

package/build/cron/pushCronIdentifiersFromCsv.js.map

@@ -1 +1 @@
-{"version":3,"file":"pushCronIdentifiersFromCsv.js","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA+B;AAC/B,wCAAqD;AACrD,oDAA4B;AAC5B,+EAGuC;AACvC,gEAAuC;AACvC,sCAAmC;AACnC,0CAAsC;AACtC,4CAAqD;AAErD;;;;;GAKG;AACI,KAAK,UAAU,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,GAcrC;IACC,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,gDAAkB,CAAC,CAAC;IAExD,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,sCAAsC,UAAU,cAAc,aAAa,CAAC,MAAM,6BAA6B,CAChH,CACF,CAAC;IAEF,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3C,MAAM,IAAA,cAAG,EACP,aAAa,EACb,KAAK,EAAE,UAAU,EAAE,EAAE;QACnB,IAAI;YACF,MAAM,OAAO,GAAG,MAAM,IAAA,yDAA2B,EAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YACtE,IAAI,OAAO,EAAE;gBACX,YAAY,IAAI,CAAC,CAAC;aACnB;iBAAM;gBACL,YAAY,IAAI,CAAC,CAAC;aACnB;SACF;QAAC,OAAO,CAAC,EAAE;YACV,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6CAA6C,UAAU,CAAC,UAAU,OAAO,CAAC,CAAC,OAAO,EAAE,CACrF,CACF,CAAC;YACF,UAAU,IAAI,CAAC,CAAC;SACjB;QACD,WAAW,CAAC,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC;IAClD,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uCAAuC,YAAY,oBACjD,SAAS,GAAG,IACd,YAAY,CACb,CACF,CAAC;IACF,IAAI,YAAY,EAAE;QAChB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,cAAc,YAAY,sDAAsD;YAC9E,yCAAyC,CAC5C,CACF,CAAC;KACH;IACD,IAAI,UAAU,EAAE;QACd,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,cAAc,UAAU,sFAAsF,CAC/G,CACF,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IACD,OAAO,aAAa,CAAC,MAAM,CAAC;AAC9B,CAAC;AAlGD,gEAkGC"}
+{"version":3,"file":"pushCronIdentifiersFromCsv.js","sourceRoot":"","sources":["../../src/cron/pushCronIdentifiersFromCsv.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA0C;AAC1C,wCAAqD;AACrD,oDAA4B;AAC5B,+EAGuC;AACvC,gEAAuC;AACvC,sCAAmC;AACnC,0CAAsC;AACtC,4CAAqD;AACrD,yDAAiC;AAEjC;;;;;GAKG;AACI,KAAK,UAAU,0BAA0B,CAAC,EAC/C,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,UAAU,EACV,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,EACpC,YAAY,GAAG,EAAE,GAgBlB;IACC,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,gDAAkB,CAAC,CAAC;IAExD,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,sCAAsC,UAAU,cAAc,aAAa,CAAC,MAAM,6BAA6B,CAChH,CACF,CAAC;IAEF,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAE3C,yCAAyC;IACzC,MAAM,MAAM,GAAG,IAAA,eAAK,EAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,MAAM,YAAY,GAAG,KAAK,EACxB,KAA2B,EAC3B,UAAkB,EACH,EAAE;QACjB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,IAAI,CACT,oBAAoB,UAAU,GAAG,CAAC,IAAI,WAAW,KAC/C,eAAK,CAAC,MACR,SAAS,CACV,CACF,CAAC;QAEF,8CAA8C;QAC9C,MAAM,IAAA,cAAG,EAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;YACpC,IAAI;gBACF,MAAM,OAAO,GAAG,MAAM,IAAA,yDAA2B,EAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACtE,IAAI,OAAO,EAAE;oBACX,YAAY,IAAI,CAAC,CAAC;iBACnB;qBAAM;oBACL,YAAY,IAAI,CAAC,CAAC;iBACnB;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6CAA6C,UAAU,CAAC,UAAU,OAAO,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,OAAO,EAAE,CACtF,CACF,CAAC;gBACF,UAAU,IAAI,CAAC,CAAC;aACjB;YACD,WAAW,CAAC,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,mDAAmD;QACnD,IAAI,YAAY,GAAG,CAAC,IAAI,UAAU,GAAG,WAAW,GAAG,CAAC,EAAE;YACpD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CAAC,gBAAgB,YAAY,wBAAwB,CAAC,CACpE,CAAC;YAEF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,UAAU,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;SACJ;IACH,CAAC,CAAC;IAEF,kCAAkC;IAClC,MAAM,IAAA,oBAAS,EAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEtC,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uCAAuC,YAAY,oBACjD,SAAS,GAAG,IACd,YAAY,CACb,CACF,CAAC;IACF,IAAI,YAAY,EAAE;QAChB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,cAAc,YAAY,sDAAsD;YAC9E,yCAAyC,CAC5C,CACF,CAAC;KACH;IACD,IAAI,UAAU,EAAE;QACd,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,cAAc,UAAU,sFAAsF,CAC/G,CACF,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;KACrD;IACD,OAAO,aAAa,CAAC,MAAM,CAAC;AAC9B,CAAC;AAjID,gEAiIC"}

package/build/cron/writeCsv.d.ts

@@ -23,6 +23,18 @@ export declare function appendCsvSync(filePath: string, data: ObjByString[]): vo
  * @param headers - Headers
  */
 export declare function writeCsv(filePath: string, data: ObjByString[], headers?: boolean | string[]): Promise<void>;
+/**
+ * Parse a file path into a base name and extension
+ *
+ * @param filePath - File path to parse
+ * @returns Base name and extension
+ */
+export declare function parseFilePath(filePath: string): {
+    /** Base name of the file */
+    baseName: string;
+    /** Extension of the file */
+    extension: string;
+};
 /**
  * Write a large CSV dataset to multiple files to avoid file size limits
  *

package/build/cron/writeCsv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writeCsv.d.ts","sourceRoot":"","sources":["../../src/cron/writeCsv.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAcvD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,EAAE,MAAM,EAAE,GAChB,IAAI,CAaN;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,IAAI,CAWzE;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,GAAE,OAAO,GAAG,MAAM,EAAS,GACjC,OAAO,CAAC,IAAI,CAAC,CAaf;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,GAAE,OAAO,GAAG,MAAM,EAAS,EAClC,SAAS,SAAS,GACjB,OAAO,CAAC,MAAM,EAAE,CAAC,CAiCnB"}
+{"version":3,"file":"writeCsv.d.ts","sourceRoot":"","sources":["../../src/cron/writeCsv.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAcvD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,EAAE,MAAM,EAAE,GAChB,IAAI,CAaN;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,IAAI,CAWzE;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,GAAE,OAAO,GAAG,MAAM,EAAS,GACjC,OAAO,CAAC,IAAI,CAAC,CAaf;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAC/C,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB,CAOA;AAED;;;;;;;GAOG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,WAAW,EAAE,EACnB,OAAO,GAAE,OAAO,GAAG,MAAM,EAAS,EAClC,SAAS,SAAS,GACjB,OAAO,CAAC,MAAM,EAAE,CAAC,CA2BnB"}