@transcend-io/cli 6.26.2 → 6.26.4

package/README.md

@@ -1752,6 +1752,9 @@ yarn tr-request-export --auth=$TRANSCEND_API_KEY --file=./path/to/file.csv
 If you are using the cron job integration, you can run this command to pull the outstanding identifiers
 for the data silo to a CSV.
 
+For large datasets, the output will be automatically split into multiple CSV files to avoid file system size limits.
+Use the `--chunkSize` parameter to control the maximum number of rows per file.
+
 Read more at https://docs.transcend.io/docs/integrations/cron-job-integration.
 
 https://user-images.githubusercontent.com/10264973/205483055-f4050645-bdf5-4ea2-8464-3183abd63074.mov
@@ -1774,6 +1777,7 @@ The API key must be associated to the ID of the integration/data silo that is be
 | sombraAuth       | The sombra internal key, use for additional authentication when self-hosting sombra.                                                                                                                                                                                                                                     | string             | N/A                      | false    |
 | pageLimit        | The page limit to use when pulling in pages of identifiers.                                                                                                                                                                                                                                                              | number             | 100                      | false    |
 | skipRequestCount | Whether to skip the count of all outstanding requests. This is required to render the progress bar, but can take a long time to run if you have a large number of outstanding requests to process. In that case, we recommend setting skipRequestCount=true so that you can still proceed with fetching the identifiers. | boolean            | false                    | false    |
+| chunkSize        | Maximum number of rows per CSV file. For large datasets, the output will be automatically split into multiple files to avoid file system size limits. Each file will contain at most this many rows.                                                                                                                     | number             | 100000                   | false    |
 
 #### Usage
 
@@ -1809,6 +1813,19 @@ yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf
  --pageLimit=300
 ```
 
+Specifying the chunk size for large datasets to avoid file size limits (defaults to 100,000 rows per file).
+
+```sh
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE \
+ --chunkSize=50000
+```
+
+**Note:** For large datasets exceeding the chunk size, multiple CSV files will be created automatically:
+
+- `filename_part01_of_05.csv` (50,000 rows)
+- `filename_part02_of_05.csv` (50,000 rows)
+- ... and so on
+
 ### tr-cron-mark-identifiers-completed
 
 This command takes the output of `tr-cron-pull-identifiers` and notifies Transcend that all of the requests in the CSV have been processed.

package/build/cli-cron-pull-identifiers.js

@@ -15,23 +15,28 @@ const requests_1 = require("./requests");
 /**
  * Pull the set of active identifiers for a cron job silo.
  *
+ * For large datasets, the output will be automatically split into multiple CSV files
+ * to avoid file system size limits. Use --chunkSize to control the number of rows per file.
+ *
  * Requires an API key with scope for the cron integration being checked on.
  *
  * Dev Usage:
  * yarn ts-node ./src/cli-cron-pull-identifiers.ts --auth=$TRANSCEND_API_KEY \
  *   --dataSiloId=92636cda-b7c6-48c6-b1b1-2df574596cbc \
  *   --actions=ERASURE \
- *   --file=/Users/michaelfarrell/Desktop/test.csv
+ *   --file=/Users/michaelfarrell/Desktop/test.csv \
+ *   --chunkSize=100000
  *
  * Standard usage:
  * yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY  \
  *   --dataSiloId=92636cda-b7c6-48c6-b1b1-2df574596cbc \
  *   --actions=ERASURE \
- *   --file=/Users/michaelfarrell/Desktop/test.csv
+ *   --file=/Users/michaelfarrell/Desktop/test.csv \
+ *   --chunkSize=100000
  */
 async function main() {
     // Parse command line arguments
-    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, actions, pageLimit = '100', skipRequestCount = false, } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, actions, pageLimit = '100', skipRequestCount = false, chunkSize = '100000', } = (0, yargs_parser_1.default)(process.argv.slice(2));
     // Ensure auth is passed
     if (!auth) {
         logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
@@ -58,6 +63,12 @@ async function main() {
             `Expected one of: \n${Object.values(privacy_types_1.RequestAction).join('\n')}`));
         process.exit(1);
     }
+    // Validate chunk size
+    const parsedChunkSize = parseInt(chunkSize, 10);
+    if (Number.isNaN(parsedChunkSize) || parsedChunkSize <= 0) {
+        logger_1.logger.error(colors_1.default.red(`Invalid chunk size: "${chunkSize}". Must be a positive integer.`));
+        process.exit(1);
+    }
     // Pull down outstanding identifiers
     const { identifiersFormattedForCsv } = await (0, cron_1.pullCustomSiloOutstandingIdentifiers)({
         transcendUrl,
@@ -68,10 +79,18 @@ async function main() {
         dataSiloId,
         skipRequestCount: skipRequestCount === 'true',
     });
-    // Write CSV
+    // Write CSV (split into multiple files if too large)
     const headers = (0, uniq_1.default)(identifiersFormattedForCsv.map((d) => Object.keys(d)).flat());
-    (0, cron_1.writeCsv)(file, identifiersFormattedForCsv, headers);
-    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to file "${file}"`));
+    const writtenFiles = await (0, cron_1.writeLargeCsv)(file, identifiersFormattedForCsv, headers, parsedChunkSize);
+    if (writtenFiles.length === 1) {
+        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to file "${file}"`));
+    }
+    else {
+        logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to ${writtenFiles.length} files:`));
+        writtenFiles.forEach((fileName) => {
+            logger_1.logger.info(colors_1.default.green(`  - ${fileName}`));
+        });
+    }
 }
 main();
 //# sourceMappingURL=cli-cron-pull-identifiers.js.map

package/build/cli-cron-pull-identifiers.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAAwE;AACxE,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAE5C;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,OAAO,EACP,SAAS,GAAG,KAAK,EACjB,gBAAgB,GAAG,KAAK,GACzB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,gBAAgB,KAAK,MAAM,EAAE;QAC/B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CACX,uEAAuE,CACxE,CACF,CAAC;KACH;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oCAAoC;IACpC,MAAM,EAAE,0BAA0B,EAAE,GAClC,MAAM,IAAA,2CAAoC,EAAC;QACzC,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU;QACV,gBAAgB,EAAE,gBAAgB,KAAK,MAAM;KAC9C,CAAC,CAAC;IAEL,YAAY;IACZ,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;IACF,IAAA,eAAQ,EAAC,IAAI,EAAE,0BAA0B,EAAE,OAAO,CAAC,CAAC;IACpD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACxF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAA6E;AAC7E,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAE5C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,OAAO,EACP,SAAS,GAAG,KAAK,EACjB,gBAAgB,GAAG,KAAK,EACxB,SAAS,GAAG,QAAQ,GACrB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,CAAC,OAAO,EAAE;QACZ,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,+EAA+E,CAChF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,IAAI,gBAAgB,KAAK,MAAM,EAAE;QAC/B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,MAAM,CACX,uEAAuE,CACxE,CACF,CAAC;KACH;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,sBAAsB;IACtB,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE;QACzD,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,wBAAwB,SAAS,gCAAgC,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oCAAoC;IACpC,MAAM,EAAE,0BAA0B,EAAE,GAClC,MAAM,IAAA,2CAAoC,EAAC;QACzC,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU;QACV,gBAAgB,EAAE,gBAAgB,KAAK,MAAM;KAC9C,CAAC,CAAC;IAEL,qDAAqD;IACrD,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,IAAA,oBAAa,EACtC,IAAI,EACJ,0BAA0B,EAC1B,OAAO,EACP,eAAe,CAChB,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE;QAC7B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACxF,CACF,CAAC;KACH;SAAM;QACL,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,mBAAmB,YAAY,CAAC,MAAM,SAAS,CACvG,CACF,CAAC;QACF,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChC,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,KAAK,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-sync-ot.js

@@ -11,7 +11,7 @@ const enums_1 = require("./enums");
 const graphql_1 = require("./graphql");
 const helpers_1 = require("./oneTrust/helpers");
 /**
- * Pull configuration from OneTrust down locally to disk
+ * Pull resources from OneTrust to Transcend or locally to disk
  *
  * Dev Usage:
  * yarn ts-node ./src/cli-sync-ot.ts --hostname=customer.my.onetrust.com --oneTrustAuth=$ONE_TRUST_OAUTH_TOKEN --transcendAuth=$TRANSCEND_API_KEY

package/build/codecs.d.ts

@@ -190,6 +190,7 @@ export declare const EnricherInput: t.IntersectionC<[t.TypeC<{
         LOOKER: unknown;
         WAIT_PERIOD: unknown;
         GOVERNMENT_ID: unknown;
+        CUSTOM_FUNCTION: unknown;
         email: unknown;
         phone: unknown;
         SOMBRA: unknown;
@@ -48729,6 +48730,7 @@ export declare const TranscendInput: t.PartialC<{
             LOOKER: unknown;
             WAIT_PERIOD: unknown;
             GOVERNMENT_ID: unknown;
+            CUSTOM_FUNCTION: unknown;
             email: unknown;
             phone: unknown;
             SOMBRA: unknown;