@transcend-io/cli 4.85.0 → 4.86.0

package/README.md

@@ -1157,47 +1157,47 @@ The API key must be associated to the ID of the integration/data silo that is be
 
 #### Arguments
 
-| Argument     | Description                                                                                                                             | Type                   | Default                  | Required |
-| ------------ | --------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- | ------------------------ | -------- |
-| auth         | The Transcend API key with the scopes necessary for the command.                                                                        | string                 | N/A                      | true     |
-| dataSiloId   | The ID of the data silo to pull in.                                                                                                     | string - UUID          | N/A                      | true     |
-| requestType  | The [request action](https://docs.transcend.io/docs/privacy-requests/configuring-requests/data-subject-requests#data-actions) to fetch. | string - RequestAction | N/A                      | true     |
-| file         | Path to the CSV file where identifiers will be written to.                                                                              | string - file-path     | ./cron-identifiers.csv   | false    |
-| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting.                                                           | string - URL           | https://api.transcend.io | false    |
-| sombraAuth   | The sombra internal key, use for additional authentication when self-hosting sombra.                                                    | string                 | N/A                      | false    |
-| pageLimit    | The page limit to use when pulling in pages of identifiers.                                                                             | number                 | 100                      | false    |
+| Argument     | Description                                                                                                                               | Type               | Default                  | Required |
+| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | ------------------------ | -------- |
+| auth         | The Transcend API key with the scopes necessary for the command.                                                                          | string             | N/A                      | true     |
+| dataSiloId   | The ID of the data silo to pull in.                                                                                                       | string - UUID      | N/A                      | true     |
+| actions      | The [request action](https://docs.transcend.io/docs/privacy-requests/configuring-requests/data-subject-requests#data-actions) to restart. | RequestAction[]    | N/A                      | true     |
+| file         | Path to the CSV file where identifiers will be written to.                                                                                | string - file-path | ./cron-identifiers.csv   | false    |
+| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting.                                                             | string - URL       | https://api.transcend.io | false    |
+| sombraAuth   | The sombra internal key, use for additional authentication when self-hosting sombra.                                                      | string             | N/A                      | false    |
+| pageLimit    | The page limit to use when pulling in pages of identifiers.                                                                               | number             | 100                      | false    |
 
 #### Usage
 
 ```sh
-yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE
 ```
 
 Pull to a specific file location
 
 ```sh
-yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE \
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE \
    --file=/Users/transcend/Desktop/test.csv
 ```
 
 For self-hosted sombras that use an internal key:
 
 ```sh
-yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE  \
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE  \
    --sombraAuth=$SOMBRA_INTERNAL_KEY
 ```
 
 Specifying the backend URL, needed for US hosted backend infrastructure.
 
 ```sh
-yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE \
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE \
  --transcendUrl=https://api.us.transcend.io
 ```
 
 Specifying the page limit, defaults to 100.
 
 ```sh
-yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE \
+yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE \
  --pageLimit=300
 ```
 
@@ -1207,7 +1207,7 @@ This command takes the output of `tr-cron-pull-identifiers` and notifies Transce
 This is used in the workflow like:
 
 1. Pull identifiers to CSV:
-   `yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --requestType=ERASURE --file=./outstanding-requests.csv`
+   `yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --actions=ERASURE --file=./outstanding-requests.csv`
 2. Run your process to operate on that CSV of requests.
 3. Notify Transcend of completion
    `yarn tr-cron-mark-identifiers-completed --auth=$TRANSCEND_API_KEY --dataSiloId=70810f2e-cf90-43f6-9776-901a5950599f --file=./outstanding-requests.csv`
@@ -1280,13 +1280,14 @@ The API key must have the following scopes:
 
 #### Arguments
 
-| Argument     | Description                                                                   | Type               | Default                             | Required |
-| ------------ | ----------------------------------------------------------------------------- | ------------------ | ----------------------------------- | -------- |
-| auth         | The Transcend API key with the scopes necessary for the command.              | string             | N/A                                 | true     |
-| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting. | string - URL       | https://api.transcend.io            | false    |
-| file         | Path to the CSV file where requests will be written to.                       | string - file-path | ./manual-enrichment-identifiers.csv | false    |
-| actions      | The set of request actions to pull requests for.                              | RequestAction[]    | []                                  | false    |
-| concurrency  | The concurrency to use when uploading requests in parallel.                   | number             | 100                                 | false    |
+| Argument     | Description                                                                                                                                | Type               | Default                             | Required |
+| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | ------------------ | ----------------------------------- | -------- |
+| auth         | The Transcend API key with the scopes necessary for the command.                                                                           | string             | N/A                                 | true     |
+| transcendUrl | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting.                                                              | string - URL       | https://api.transcend.io            | false    |
+| file         | Path to the CSV file where requests will be written to.                                                                                    | string - file-path | ./manual-enrichment-identifiers.csv | false    |
+| actions      | The [request action](https://docs.transcend.io/docs/privacy-requests/configuring-requests/data-subject-requests#data-actions) to pull for. | RequestAction[]    | N/A                                 | false    |
+
+| concurrency | The concurrency to use when uploading requests in parallel. | number | 100 | false |
 
 #### Usage
 
@@ -1390,7 +1391,7 @@ yarn tr-manual-enrichment-push-identifiers --auth=$TRANSCEND_API_KEY --enricherI
 ### tr-mark-request-data-silos-completed
 
 This command takes in a CSV of Request IDs as well as a Data Silo ID and marks all associated privacy request jobs as completed.
-This command is useful with the "Bulk Response" UI.
+This command is useful with the "Bulk Response" UI. The CSV is expected to have 1 column named "Request Id".
 
 #### Authentication
 

package/build/cli-cron-pull-identifiers.js

@@ -7,9 +7,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const yargs_parser_1 = __importDefault(require("yargs-parser"));
 const colors_1 = __importDefault(require("colors"));
 const logger_1 = require("./logger");
+const uniq_1 = __importDefault(require("lodash/uniq"));
 const cron_1 = require("./cron");
 const privacy_types_1 = require("@transcend-io/privacy-types");
 const constants_1 = require("./constants");
+const requests_1 = require("./requests");
 /**
  * Pull the set of active identifiers for a cron job silo.
  *
@@ -18,40 +20,46 @@ const constants_1 = require("./constants");
  * Dev Usage:
  * yarn ts-node ./src/cli-cron-pull-identifiers.ts --auth=$TRANSCEND_API_KEY \
  *   --dataSiloId=92636cda-b7c6-48c6-b1b1-2df574596cbc \
- *   --requestType=ERASURE \
+ *   --actions=ERASURE \
  *   --file=/Users/michaelfarrell/Desktop/test.csv
  *
  * Standard usage:
  * yarn tr-cron-pull-identifiers --auth=$TRANSCEND_API_KEY  \
  *   --dataSiloId=92636cda-b7c6-48c6-b1b1-2df574596cbc \
- *   --requestType=ERASURE \
+ *   --actions=ERASURE \
  *   --file=/Users/michaelfarrell/Desktop/test.csv
  */
 async function main() {
     // Parse command line arguments
-    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, requestType, pageLimit = '100', } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    const { file = './cron-identifiers.csv', transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, sombraAuth, dataSiloId, actions, pageLimit = '100', } = (0, yargs_parser_1.default)(process.argv.slice(2));
     // Ensure auth is passed
     if (!auth) {
         logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
         process.exit(1);
     }
-    // Validate request type
+    // Validate actions
+    const parsedActions = (0, requests_1.splitCsvToList)(actions);
+    const invalidActions = parsedActions.filter(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    if (!Object.values(privacy_types_1.RequestAction).includes(requestType)) {
-        logger_1.logger.error(colors_1.default.red(`Failed to parse requestType:"${requestType}".\n` +
+    (type) => !Object.values(privacy_types_1.RequestAction).includes(type));
+    if (invalidActions.length > 0) {
+        logger_1.logger.error(colors_1.default.red(`Failed to parse actions:"${invalidActions.join(',')}".\n` +
             `Expected one of: \n${Object.values(privacy_types_1.RequestAction).join('\n')}`));
         process.exit(1);
     }
-    // Upload privacy requests
-    await (0, cron_1.pullCronIdentifiersToCsv)({
-        file,
+    // Pull down outstanding identifiers
+    const { identifiersFormattedForCsv } = await (0, cron_1.pullCustomSiloOutstandingIdentifiers)({
         transcendUrl,
         pageLimit: parseInt(pageLimit, 10),
-        requestType: requestType,
+        actions: parsedActions,
         auth,
         sombraAuth,
         dataSiloId,
     });
+    // Write CSV
+    const headers = (0, uniq_1.default)(identifiersFormattedForCsv.map((d) => Object.keys(d)).flat());
+    (0, cron_1.writeCsv)(file, identifiersFormattedForCsv, headers);
+    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${identifiersFormattedForCsv.length} identifiers to file "${file}"`));
 }
 main();
 //# sourceMappingURL=cli-cron-pull-identifiers.js.map

package/build/cli-cron-pull-identifiers.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,iCAAkD;AAClD,+DAA4D;AAC5D,2CAAoD;AAEpD;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,WAAW,EACX,SAAS,GAAG,KAAK,GAClB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,8DAA8D;IAC9D,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,WAAkB,CAAC,EAAE;QAC9D,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,gCAAgC,WAAW,MAAM;YAC/C,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,MAAM,IAAA,+BAAwB,EAAC;QAC7B,IAAI;QACJ,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,WAAW,EAAE,WAA4B;QACzC,IAAI;QACJ,UAAU;QACV,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-cron-pull-identifiers.js","sourceRoot":"","sources":["../src/cli-cron-pull-identifiers.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,iCAAwE;AACxE,+DAA4D;AAC5D,2CAAoD;AACpD,yCAA4C;AAE5C;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,wBAAwB,EAC/B,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,EACV,UAAU,EACV,OAAO,EACP,SAAS,GAAG,KAAK,GAClB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oCAAoC;IACpC,MAAM,EAAE,0BAA0B,EAAE,GAClC,MAAM,IAAA,2CAAoC,EAAC;QACzC,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,IAAI;QACJ,UAAU;QACV,UAAU;KACX,CAAC,CAAC;IAEL,YAAY;IACZ,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC7D,CAAC;IACF,IAAA,eAAQ,EAAC,IAAI,EAAE,0BAA0B,EAAE,OAAO,CAAC,CAAC;IACpD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,0BAA0B,CAAC,MAAM,yBAAyB,IAAI,GAAG,CACxF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-mark-request-data-silos-completed.js

@@ -1,14 +1,42 @@
 #!/usr/bin/env node
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const yargs_parser_1 = __importDefault(require("yargs-parser"));
 const colors_1 = __importDefault(require("colors"));
+const t = __importStar(require("io-ts"));
 const logger_1 = require("./logger");
 const cron_1 = require("./cron");
 const constants_1 = require("./constants");
+const requests_1 = require("./requests");
+const RequestIdRow = t.type({
+    'Request Id': t.string,
+});
 /**
  * Given a set of Request IDs and a Data Silo ID, mark the RequestDataSilos as completed
  *
@@ -38,9 +66,12 @@ async function main() {
         logger_1.logger.error(colors_1.default.red('A Data Silo ID must be provided. You can specify using --dataSiloId=2560bb81-b837-4c6f-a57e-dcef87069d43'));
         process.exit(1);
     }
+    // Read from CSV
+    logger_1.logger.info(colors_1.default.magenta(`Reading "${file}" from disk`));
+    const activeResults = (0, requests_1.readCsv)(file, RequestIdRow);
     // Upload privacy requests
     await (0, cron_1.markRequestDataSiloIdsCompleted)({
-        file,
+        requestIds: activeResults.map((request) => request['Request Id']),
         transcendUrl,
         auth,
         dataSiloId,

package/build/cli-mark-request-data-silos-completed.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-mark-request-data-silos-completed.js","sourceRoot":"","sources":["../src/cli-mark-request-data-silos-completed.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,iCAAyD;AACzD,2CAAoD;AAEpD;;;;;;;;;;;;;;;GAeG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,2BAA2B,EAClC,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,GACX,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gCAAgC;IAChC,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,MAAM,IAAA,sCAA+B,EAAC;QACpC,IAAI;QACJ,YAAY;QACZ,IAAI;QACJ,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-mark-request-data-silos-completed.js","sourceRoot":"","sources":["../src/cli-mark-request-data-silos-completed.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAC5B,yCAA2B;AAE3B,qCAAkC;AAClC,iCAAyD;AACzD,2CAAoD;AACpD,yCAAqC;AAErC,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC;IAC1B,YAAY,EAAE,CAAC,CAAC,MAAM;CACvB,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;GAeG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,2BAA2B,EAClC,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,UAAU,GACX,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gCAAgC;IAChC,IAAI,CAAC,UAAU,EAAE;QACf,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,0GAA0G,CAC3G,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAElD,0BAA0B;IAC1B,MAAM,IAAA,sCAA+B,EAAC;QACpC,UAAU,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACjE,YAAY;QACZ,IAAI;QACJ,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-request-export.js

@@ -7,9 +7,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const yargs_parser_1 = __importDefault(require("yargs-parser"));
 const colors_1 = __importDefault(require("colors"));
 const logger_1 = require("./logger");
+const uniq_1 = __importDefault(require("lodash/uniq"));
 const privacy_types_1 = require("@transcend-io/privacy-types");
 const requests_1 = require("./requests");
 const constants_1 = require("./constants");
+const cron_1 = require("./cron");
 /**
  * Pull down the metadata for a set of Privacy requests.
  *
@@ -56,8 +58,7 @@ async function main() {
         process.exit(1);
     }
     // Pull privacy requests
-    await (0, requests_1.pullRequestsToCsv)({
-        file,
+    const { requestsFormattedForCsv } = await (0, requests_1.pullPrivacyRequests)({
         transcendUrl,
         pageLimit: parseInt(pageLimit, 10),
         actions: parsedActions,
@@ -67,6 +68,10 @@ async function main() {
         createdAtAfter: createdAtAfter ? new Date(createdAtAfter) : undefined,
         showTests: showTests === 'true' ? true : showTests === 'false' ? false : undefined,
     });
+    // Write to CSV
+    const headers = (0, uniq_1.default)(requestsFormattedForCsv.map((d) => Object.keys(d)).flat());
+    (0, cron_1.writeCsv)(file, requestsFormattedForCsv, headers);
+    logger_1.logger.info(colors_1.default.green(`Successfully wrote ${requestsFormattedForCsv.length} requests to file "${file}"`));
 }
 main();
 //# sourceMappingURL=cli-request-export.js.map

package/build/cli-request-export.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-request-export.js","sourceRoot":"","sources":["../src/cli-request-export.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,+DAA2E;AAC3E,yCAA+D;AAC/D,2CAAoD;AAEpD;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,gCAAgC,EACvC,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,EACf,cAAc,EACd,SAAS,GAAG,KAAK,GAClB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oBAAoB;IACpB,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,QAAQ,CAAoB,CAAC;IACnE,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM;IAC3C,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;QAC9B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6BAA6B,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YAC1D,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,MAAM,IAAA,4BAAiB,EAAC;QACtB,IAAI;QACJ,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,cAAc;QACxB,IAAI;QACJ,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;QACxE,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;QACrE,SAAS,EACP,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC1E,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-request-export.js","sourceRoot":"","sources":["../src/cli-request-export.ts"],"names":[],"mappings":";;;;;;AAEA,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,uDAA+B;AAC/B,+DAA2E;AAC3E,yCAAiE;AACjE,2CAAoD;AACpD,iCAAkC;AAElC;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,IAAI,GAAG,gCAAgC,EACvC,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,EACf,cAAc,EACd,SAAS,GAAG,KAAK,GAClB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,mBAAmB;IACnB,MAAM,aAAa,GAAG,IAAA,yBAAc,EAAC,OAAO,CAAoB,CAAC;IACjE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM;IACzC,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YACxD,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,oBAAoB;IACpB,MAAM,cAAc,GAAG,IAAA,yBAAc,EAAC,QAAQ,CAAoB,CAAC;IACnE,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM;IAC3C,8DAA8D;IAC9D,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,QAAQ,CAAC,IAAW,CAAC,CAC9D,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;QAC9B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6BAA6B,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM;YAC1D,sBAAsB,MAAM,CAAC,MAAM,CAAC,6BAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,MAAM,EAAE,uBAAuB,EAAE,GAAG,MAAM,IAAA,8BAAmB,EAAC;QAC5D,YAAY;QACZ,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;QAClC,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,cAAc;QACxB,IAAI;QACJ,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;QACxE,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;QACrE,SAAS,EACP,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC1E,CAAC,CAAC;IAEH,eAAe;IACf,MAAM,OAAO,GAAG,IAAA,cAAI,EAClB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1D,CAAC;IACF,IAAA,eAAQ,EAAC,IAAI,EAAE,uBAAuB,EAAE,OAAO,CAAC,CAAC;IACjD,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,uBAAuB,CAAC,MAAM,sBAAsB,IAAI,GAAG,CAClF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cron/index.d.ts

@@ -1,4 +1,4 @@
-export * from './pullCronIdentifiersToCsv';
+export * from './pullCustomSiloOutstandingIdentifiers';
 export * from './pullCronPageOfIdentifiers';
 export * from './writeCsv';
 export * from './pushCronIdentifiersFromCsv';

package/build/cron/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAC;AAC3C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,YAAY,CAAC;AAC3B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,mCAAmC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":"AAAA,cAAc,wCAAwC,CAAC;AACvD,cAAc,6BAA6B,CAAC;AAC5C,cAAc,YAAY,CAAC;AAC3B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,mCAAmC,CAAC"}

package/build/cron/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./pullCronIdentifiersToCsv"), exports);
+__exportStar(require("./pullCustomSiloOutstandingIdentifiers"), exports);
 __exportStar(require("./pullCronPageOfIdentifiers"), exports);
 __exportStar(require("./writeCsv"), exports);
 __exportStar(require("./pushCronIdentifiersFromCsv"), exports);

package/build/cron/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA2C;AAC3C,8DAA4C;AAC5C,6CAA2B;AAC3B,+DAA6C;AAC7C,gEAA8C;AAC9C,oEAAkD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cron/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yEAAuD;AACvD,8DAA4C;AAC5C,6CAA2B;AAC3B,+DAA6C;AAC7C,gEAA8C;AAC9C,oEAAkD"}

package/build/cron/markRequestDataSiloIdsCompleted.d.ts

@@ -1,16 +1,19 @@
+import { RequestDataSiloStatus } from '@transcend-io/privacy-types';
 /**
  * Given a CSV of Request IDs, mark associated RequestDataSilos as completed
  *
  * @param options - Options
  * @returns Number of items marked as completed
  */
-export declare function markRequestDataSiloIdsCompleted({ file, dataSiloId, auth, concurrency, transcendUrl, }: {
-    /** CSV file path */
-    file: string;
+export declare function markRequestDataSiloIdsCompleted({ requestIds, dataSiloId, auth, concurrency, status, transcendUrl, }: {
+    /** The list of request ids to mark as completed  */
+    requestIds: string[];
     /** Transcend API key authentication */
     auth: string;
     /** Data Silo ID to pull down jobs for */
     dataSiloId: string;
+    /** Status to update requests to */
+    status?: RequestDataSiloStatus;
     /** Upload concurrency */
     concurrency?: number;
     /** API URL for Transcend backend */

package/build/cron/markRequestDataSiloIdsCompleted.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markRequestDataSiloIdsCompleted.d.ts","sourceRoot":"","sources":["../../src/cron/markRequestDataSiloIdsCompleted.ts"],"names":[],"mappings":"AAkBA;;;;;GAKG;AACH,wBAAsB,+BAA+B,CAAC,EACpD,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,WAAiB,EACjB,YAAoC,GACrC,EAAE;IACD,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CAyDlB"}
+{"version":3,"file":"markRequestDataSiloIdsCompleted.d.ts","sourceRoot":"","sources":["../../src/cron/markRequestDataSiloIdsCompleted.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAEpE;;;;;GAKG;AACH,wBAAsB,+BAA+B,CAAC,EACpD,UAAU,EACV,UAAU,EACV,IAAI,EACJ,WAAiB,EACjB,MAAuC,EACvC,YAAoC,GACrC,EAAE;IACD,oDAAoD;IACpD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,MAAM,CAAC,CAqDlB"}

package/build/cron/markRequestDataSiloIdsCompleted.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -30,42 +7,35 @@ exports.markRequestDataSiloIdsCompleted = void 0;
 const bluebird_1 = require("bluebird");
 const colors_1 = __importDefault(require("colors"));
 const logger_1 = require("../logger");
-const requests_1 = require("../requests");
 const graphql_1 = require("../graphql");
-const t = __importStar(require("io-ts"));
 const cli_progress_1 = __importDefault(require("cli-progress"));
 const constants_1 = require("../constants");
-const RequestIdRow = t.type({
-    'Request Id': t.string,
-});
+const privacy_types_1 = require("@transcend-io/privacy-types");
 /**
  * Given a CSV of Request IDs, mark associated RequestDataSilos as completed
  *
  * @param options - Options
  * @returns Number of items marked as completed
  */
-async function markRequestDataSiloIdsCompleted({ file, dataSiloId, auth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
+async function markRequestDataSiloIdsCompleted({ requestIds, dataSiloId, auth, concurrency = 100, status = privacy_types_1.RequestDataSiloStatus.Resolved, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
     // Find all requests made before createdAt that are in a removing data state
     const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
     // Time duration
     const t0 = new Date().getTime();
     // create a new progress bar instance and use shades_classic theme
     const progressBar = new cli_progress_1.default.SingleBar({}, cli_progress_1.default.Presets.shades_classic);
-    // Read from CSV
-    logger_1.logger.info(colors_1.default.magenta(`Reading "${file}" from disk`));
-    const activeResults = (0, requests_1.readCsv)(file, RequestIdRow);
     // Notify Transcend
-    logger_1.logger.info(colors_1.default.magenta(`Notifying Transcend for data silo "${dataSiloId}" marking "${activeResults.length}" requests as completed.`));
+    logger_1.logger.info(colors_1.default.magenta(`Notifying Transcend for data silo "${dataSiloId}" marking "${requestIds.length}" requests as completed.`));
     let total = 0;
-    progressBar.start(activeResults.length, 0);
-    await (0, bluebird_1.map)(activeResults, async (identifier) => {
+    progressBar.start(requestIds.length, 0);
+    await (0, bluebird_1.map)(requestIds, async (requestId) => {
         const requestDataSilo = await (0, graphql_1.fetchRequestDataSilo)(client, {
-            requestId: identifier['Request Id'],
+            requestId,
             dataSiloId,
         });
         await (0, graphql_1.makeGraphQLRequest)(client, graphql_1.CHANGE_REQUEST_DATA_SILO_STATUS, {
             requestDataSiloId: requestDataSilo.id,
-            status: 'RESOLVED',
+            status,
         });
         total += 1;
         progressBar.update(total);
@@ -74,7 +44,7 @@ async function markRequestDataSiloIdsCompleted({ file, dataSiloId, auth, concurr
     const t1 = new Date().getTime();
     const totalTime = t1 - t0;
     logger_1.logger.info(colors_1.default.green(`Successfully notified Transcend in "${totalTime / 1000}" seconds!`));
-    return activeResults.length;
+    return requestIds.length;
 }
 exports.markRequestDataSiloIdsCompleted = markRequestDataSiloIdsCompleted;
 //# sourceMappingURL=markRequestDataSiloIdsCompleted.js.map

package/build/cron/markRequestDataSiloIdsCompleted.js.map

@@ -1 +1 @@
-{"version":3,"file":"markRequestDataSiloIdsCompleted.js","sourceRoot":"","sources":["../../src/cron/markRequestDataSiloIdsCompleted.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAA+B;AAC/B,oDAA4B;AAC5B,sCAAmC;AACnC,0CAAsC;AACtC,wCAKoB;AACpB,yCAA2B;AAC3B,gEAAuC;AACvC,4CAAqD;AAErD,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC;IAC1B,YAAY,EAAE,CAAC,CAAC,MAAM;CACvB,CAAC,CAAC;AAEH;;;;;GAKG;AACI,KAAK,UAAU,+BAA+B,CAAC,EACpD,IAAI,EACJ,UAAU,EACV,IAAI,EACJ,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,GAYrC;IACC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,gBAAgB;IAChB,eAAM,CAAC,IAAI,CAAC,gBAAM,CAAC,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,IAAA,kBAAO,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAElD,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,sCAAsC,UAAU,cAAc,aAAa,CAAC,MAAM,0BAA0B,CAC7G,CACF,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3C,MAAM,IAAA,cAAG,EACP,aAAa,EACb,KAAK,EAAE,UAAU,EAAE,EAAE;QACnB,MAAM,eAAe,GAAG,MAAM,IAAA,8BAAoB,EAAC,MAAM,EAAE;YACzD,SAAS,EAAE,UAAU,CAAC,YAAY,CAAC;YACnC,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,IAAA,4BAAkB,EAGrB,MAAM,EAAE,yCAA+B,EAAE;YAC1C,iBAAiB,EAAE,eAAe,CAAC,EAAE;YACrC,MAAM,EAAE,UAAU;SACnB,CAAC,CAAC;QAEH,KAAK,IAAI,CAAC,CAAC;QACX,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uCAAuC,SAAS,GAAG,IAAI,YAAY,CACpE,CACF,CAAC;IACF,OAAO,aAAa,CAAC,MAAM,CAAC;AAC9B,CAAC;AA1ED,0EA0EC"}
+{"version":3,"file":"markRequestDataSiloIdsCompleted.js","sourceRoot":"","sources":["../../src/cron/markRequestDataSiloIdsCompleted.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA+B;AAC/B,oDAA4B;AAC5B,sCAAmC;AACnC,wCAKoB;AACpB,gEAAuC;AACvC,4CAAqD;AACrD,+DAAoE;AAEpE;;;;;GAKG;AACI,KAAK,UAAU,+BAA+B,CAAC,EACpD,UAAU,EACV,UAAU,EACV,IAAI,EACJ,WAAW,GAAG,GAAG,EACjB,MAAM,GAAG,qCAAqB,CAAC,QAAQ,EACvC,YAAY,GAAG,iCAAqB,GAcrC;IACC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,sCAAsC,UAAU,cAAc,UAAU,CAAC,MAAM,0BAA0B,CAC1G,CACF,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACxC,MAAM,IAAA,cAAG,EACP,UAAU,EACV,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,eAAe,GAAG,MAAM,IAAA,8BAAoB,EAAC,MAAM,EAAE;YACzD,SAAS;YACT,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,IAAA,4BAAkB,EAGrB,MAAM,EAAE,yCAA+B,EAAE;YAC1C,iBAAiB,EAAE,eAAe,CAAC,EAAE;YACrC,MAAM;SACP,CAAC,CAAC;QAEH,KAAK,IAAI,CAAC,CAAC;QACX,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,uCAAuC,SAAS,GAAG,IAAI,YAAY,CACpE,CACF,CAAC;IACF,OAAO,UAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAzED,0EAyEC"}

package/build/cron/pullCustomSiloOutstandingIdentifiers.d.ts

@@ -0,0 +1,33 @@
+import { CronIdentifier } from './pullCronPageOfIdentifiers';
+import { RequestAction } from '@transcend-io/privacy-types';
+export interface CronIdentifierWithAction extends CronIdentifier {
+    /** The request action that the identifier relates to */
+    action: RequestAction;
+}
+/**
+ * Pull the set of identifiers outstanding for a cron or AVC integration
+ *
+ * @param options - Options
+ */
+export declare function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAuth, actions, pageLimit, transcendUrl, }: {
+    /** Transcend API key authentication */
+    auth: string;
+    /** Data Silo ID to pull down jobs for */
+    dataSiloId: string;
+    /** The request actions to fetch */
+    actions: RequestAction[];
+    /** Page limit when fetching identifiers */
+    pageLimit?: number;
+    /** API URL for Transcend backend */
+    transcendUrl?: string;
+    /** Sombra API key authentication */
+    sombraAuth?: string;
+}): Promise<{
+    /** Raw Identifiers */
+    identifiers: CronIdentifierWithAction[];
+    /** Identifiers formatted for CSV */
+    identifiersFormattedForCsv: {
+        [k in string]: string | null | boolean | number;
+    }[];
+}>;
+//# sourceMappingURL=pullCustomSiloOutstandingIdentifiers.d.ts.map

package/build/cron/pullCustomSiloOutstandingIdentifiers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pullCustomSiloOutstandingIdentifiers.d.ts","sourceRoot":"","sources":["../../src/cron/pullCustomSiloOutstandingIdentifiers.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,cAAc,EACf,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAK5D,MAAM,WAAW,wBAAyB,SAAQ,cAAc;IAC9D,wDAAwD;IACxD,MAAM,EAAE,aAAa,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAsB,oCAAoC,CAAC,EACzD,UAAU,EACV,IAAI,EACJ,UAAU,EACV,OAAO,EACP,SAAe,EACf,YAAoC,GACrC,EAAE;IACD,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oCAAoC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CAAC;IACV,sBAAsB;IACtB,WAAW,EAAE,wBAAwB,EAAE,CAAC;IACxC,oCAAoC;IACpC,0BAA0B,EAAE;SACzB,CAAC,IAAI,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,GAAG,MAAM;KAChD,EAAE,CAAC;CACL,CAAC,CA2DD"}

package/build/cron/pullCustomSiloOutstandingIdentifiers.js

@@ -0,0 +1,56 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pullCustomSiloOutstandingIdentifiers = void 0;
+const graphql_1 = require("../graphql");
+const colors_1 = __importDefault(require("colors"));
+const pullCronPageOfIdentifiers_1 = require("./pullCronPageOfIdentifiers");
+const logger_1 = require("../logger");
+const constants_1 = require("../constants");
+const bluebird_1 = require("bluebird");
+/**
+ * Pull the set of identifiers outstanding for a cron or AVC integration
+ *
+ * @param options - Options
+ */
+async function pullCustomSiloOutstandingIdentifiers({ dataSiloId, auth, sombraAuth, actions, pageLimit = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
+    // Create sombra instance to communicate with
+    const sombra = await (0, graphql_1.createSombraGotInstance)(transcendUrl, auth, sombraAuth);
+    logger_1.logger.info(colors_1.default.magenta(`Pulling outstanding request identifiers for data silo: "${dataSiloId}" for requests of types "${actions.join('", "')}"`));
+    // identifiers found in total
+    const identifiers = [];
+    // map over each action
+    await (0, bluebird_1.mapSeries)(actions, async (action) => {
+        let offset = 0;
+        let shouldContinue = true;
+        // Fetch a page of identifiers
+        while (shouldContinue) {
+            // eslint-disable-next-line no-await-in-loop
+            const pageIdentifiers = await (0, pullCronPageOfIdentifiers_1.pullCronPageOfIdentifiers)(sombra, {
+                dataSiloId,
+                limit: pageLimit,
+                offset,
+                requestType: action,
+            });
+            identifiers.push(...pageIdentifiers.map((identifier) => ({
+                ...identifier,
+                action,
+            })));
+            shouldContinue = pageIdentifiers.length === pageLimit;
+            offset += pageLimit;
+        }
+    });
+    logger_1.logger.info(colors_1.default.magenta(`Found: ${identifiers.length} outstanding identifiers to parse`));
+    // Write out to CSV
+    const data = identifiers.map(({ attributes, ...identifier }) => ({
+        ...identifier,
+        ...attributes.reduce((acc, val) => Object.assign(acc, {
+            [val.key]: val.values.join(','),
+        }), {}),
+    }));
+    return { identifiers, identifiersFormattedForCsv: data };
+}
+exports.pullCustomSiloOutstandingIdentifiers = pullCustomSiloOutstandingIdentifiers;
+//# sourceMappingURL=pullCustomSiloOutstandingIdentifiers.js.map

package/build/cron/pullCustomSiloOutstandingIdentifiers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pullCustomSiloOutstandingIdentifiers.js","sourceRoot":"","sources":["../../src/cron/pullCustomSiloOutstandingIdentifiers.ts"],"names":[],"mappings":";;;;;;AAAA,wCAAqD;AACrD,oDAA4B;AAC5B,2EAGqC;AAErC,sCAAmC;AACnC,4CAAqD;AACrD,uCAAqC;AAOrC;;;;GAIG;AACI,KAAK,UAAU,oCAAoC,CAAC,EACzD,UAAU,EACV,IAAI,EACJ,UAAU,EACV,OAAO,EACP,SAAS,GAAG,GAAG,EACf,YAAY,GAAG,iCAAqB,GAcrC;IAQC,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,iCAAuB,EAAC,YAAY,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAE7E,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,2DAA2D,UAAU,4BAA4B,OAAO,CAAC,IAAI,CAC3G,MAAM,CACP,GAAG,CACL,CACF,CAAC;IAEF,6BAA6B;IAC7B,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,uBAAuB;IACvB,MAAM,IAAA,oBAAS,EAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,cAAc,GAAG,IAAI,CAAC;QAE1B,8BAA8B;QAC9B,OAAO,cAAc,EAAE;YACrB,4CAA4C;YAC5C,MAAM,eAAe,GAAG,MAAM,IAAA,qDAAyB,EAAC,MAAM,EAAE;gBAC9D,UAAU;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM;gBACN,WAAW,EAAE,MAAM;aACpB,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CACd,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBACtC,GAAG,UAAU;gBACb,MAAM;aACP,CAAC,CAAC,CACJ,CAAC;YACF,cAAc,GAAG,eAAe,CAAC,MAAM,KAAK,SAAS,CAAC;YACtD,MAAM,IAAI,SAAS,CAAC;SACrB;IACH,CAAC,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,UAAU,WAAW,CAAC,MAAM,mCAAmC,CAChE,CACF,CAAC;IAEF,mBAAmB;IACnB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,GAAG,UAAU;QACb,GAAG,UAAU,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CACX,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;YACjB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;SAChC,CAAC,EACJ,EAAE,CACH;KACF,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,WAAW,EAAE,0BAA0B,EAAE,IAAI,EAAE,CAAC;AAC3D,CAAC;AAtFD,oFAsFC"}

package/build/requests/index.d.ts

@@ -20,6 +20,6 @@ export * from './uploadPrivacyRequestsFromCsv';
 export * from './bulkRestartRequests';
 export * from './restartPrivacyRequest';
 export * from './retryRequestDataSilos';
-export * from './pullRequestsToCsv';
+export * from './pullPrivacyRequests';
 export * from './skipRequestDataSilos';
 //# sourceMappingURL=index.d.ts.map

package/build/requests/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/requests/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,WAAW,CAAC;AAC1B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/requests/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,WAAW,CAAC;AAC1B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,cAAc,CAAC;AAC7B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC"}

package/build/requests/index.js

@@ -36,6 +36,6 @@ __exportStar(require("./uploadPrivacyRequestsFromCsv"), exports);
 __exportStar(require("./bulkRestartRequests"), exports);
 __exportStar(require("./restartPrivacyRequest"), exports);
 __exportStar(require("./retryRequestDataSilos"), exports);
-__exportStar(require("./pullRequestsToCsv"), exports);
+__exportStar(require("./pullPrivacyRequests"), exports);
 __exportStar(require("./skipRequestDataSilos"), exports);
 //# sourceMappingURL=index.js.map

package/build/requests/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/requests/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,8CAA4B;AAC5B,kDAAgC;AAChC,mDAAiC;AACjC,8DAA4C;AAC5C,4CAA0B;AAC1B,6DAA2C;AAC3C,+CAA6B;AAC7B,8DAA4C;AAC5C,uDAAqC;AACrC,yDAAuC;AACvC,8DAA4C;AAC5C,yDAAuC;AACvC,2DAAyC;AACzC,4DAA0C;AAC1C,0DAAwC;AACxC,2DAAyC;AACzC,uDAAqC;AACrC,iEAA+C;AAC/C,wDAAsC;AACtC,0DAAwC;AACxC,0DAAwC;AACxC,sDAAoC;AACpC,yDAAuC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/requests/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,8CAA4B;AAC5B,kDAAgC;AAChC,mDAAiC;AACjC,8DAA4C;AAC5C,4CAA0B;AAC1B,6DAA2C;AAC3C,+CAA6B;AAC7B,8DAA4C;AAC5C,uDAAqC;AACrC,yDAAuC;AACvC,8DAA4C;AAC5C,yDAAuC;AACvC,2DAAyC;AACzC,4DAA0C;AAC1C,0DAAwC;AACxC,2DAAyC;AACzC,uDAAqC;AACrC,iEAA+C;AAC/C,wDAAsC;AACtC,0DAAwC;AACxC,0DAAwC;AACxC,wDAAsC;AACtC,yDAAuC"}

package/build/requests/{pullRequestsToCsv.d.ts → pullPrivacyRequests.d.ts}

@@ -5,13 +5,11 @@ export interface ExportedPrivacyRequest extends PrivacyRequest {
     requestIdentifiers: RequestIdentifier[];
 }
 /**
- * Pull the set of cron job requests to CSV
+ * Pull down a list of privacy requests
  *
  * @param options - Options
  */
-export declare function pullRequestsToCsv({ file, auth, actions, statuses, pageLimit, transcendUrl, createdAtBefore, createdAtAfter, showTests, }: {
-    /** CSV file path */
-    file: string;
+export declare function pullPrivacyRequests({ auth, actions, statuses, pageLimit, transcendUrl, createdAtBefore, createdAtAfter, showTests, }: {
     /** Transcend API key authentication */
     auth: string;
     /** API URL for Transcend backend */
@@ -28,5 +26,12 @@ export declare function pullRequestsToCsv({ file, auth, actions, statuses, pageL
     createdAtAfter?: Date;
     /** Return test requests */
     showTests?: boolean;
-}): Promise<ExportedPrivacyRequest[]>;
-//# sourceMappingURL=pullRequestsToCsv.d.ts.map
+}): Promise<{
+    /** All request information with attached identifiers */
+    requestsWithRequestIdentifiers: ExportedPrivacyRequest[];
+    /** Requests that are formatted for CSV */
+    requestsFormattedForCsv: {
+        [k in string]: string | null | boolean;
+    }[];
+}>;
+//# sourceMappingURL=pullPrivacyRequests.d.ts.map