npm - @transcend-io/cli - Versions diffs - 8.28.0 → 8.30.0 - Mend

@transcend-io/cli 8.28.0 → 8.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (143) hide show

package/README.md CHANGED Viewed

@@ -2250,7 +2250,7 @@ transcend consent upload-preferences \
 ```txt
 USAGE
-  transcend inventory pull (--auth value) [--resources all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes] [--file value] [--transcendUrl value] [--dataSiloIds value]... [--integrationNames value]... [--trackerStatuses LIVE|NEEDS_REVIEW] [--pageSize value] [--skipDatapoints] [--skipSubDatapoints] [--includeGuessedCategories] [--debug]
+  transcend inventory pull (--auth value) [--resources all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes|siloDiscoveryResults] [--file value] [--transcendUrl value] [--dataSiloIds value]... [--integrationNames value]... [--trackerStatuses LIVE|NEEDS_REVIEW] [--pageSize value] [--skipDatapoints] [--skipSubDatapoints] [--includeGuessedCategories] [--debug]
   transcend inventory pull --help
 Generates a transcend.yml by pulling the configuration from your Transcend instance.
@@ -2264,7 +2264,7 @@ This command can be helpful if you are looking to:
 FLAGS
       --auth                       The Transcend API key. The scopes required will vary depending on the operation performed. If in doubt, the Full Admin scope will always work.
-     [--resources]                 The different resource types to pull in. Defaults to dataSilos,enrichers,templates,apiKeys.                                                    [all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes, separator = ,]
+     [--resources]                 The different resource types to pull in. Defaults to dataSilos,enrichers,templates,apiKeys.                                                    [all|apiKeys|customFields|templates|dataSilos|enrichers|dataFlows|businessEntities|processingActivities|actions|dataSubjects|identifiers|cookies|consentManager|partitions|prompts|promptPartials|promptGroups|agents|agentFunctions|agentFiles|vendors|dataCategories|processingPurposes|actionItems|actionItemCollections|teams|privacyCenters|policies|messages|assessments|assessmentTemplates|purposes|siloDiscoveryResults, separator = ,]
      [--file]                      Path to the YAML file to pull into                                                                                                             [default = ./transcend.yml]
      [--transcendUrl]              URL of the Transcend backend. Use https://api.us.transcend.io for US hosting                                                                   [default = https://api.transcend.io]
      [--dataSiloIds]...            The UUIDs of the data silos that should be pulled into the YAML file                                                                           [separator = ,]
@@ -2316,6 +2316,7 @@ The API key permissions for this command vary based on the `resources` argument:
 | `assessments`           | `assessments`             | Assessment responses.                                                                                                                                            | View Assessments                                     | [Assessments -> Assessments](https://app.transcend.io/assessments/groups)                                                                                                                                                                              |
 | `assessmentTemplates`   | `assessment-templates`    | Assessment template configurations.                                                                                                                              | View Assessments                                     | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates)                                                                                                                                                                         |
 | `purposes`              | `purposes`                | Consent purposes and related preference management topics.                                                                                                       | View Consent Manager, View Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes)                                                                                                                       |
+| `siloDiscoveryResults`  | `siloDiscoveryResults`    | System discovery results                                                                                                                                         | View Data Map                                        | [System Discovery](https://app.transcend.io/data-map/data-inventory/silo-discovery)                                                                                                                                                                    |
 #### Examples
@@ -2527,6 +2528,7 @@ The API key permissions for this command vary based on the resources declared as
 | `assessments`           | `assessments`             | Assessment responses.                                                                                                                                            | Manage Assessments                                       | [Assessments -> Assessments](https://app.transcend.io/assessments/groups)                                                                                                                                                                              |
 | `assessmentTemplates`   | `assessment-templates`    | Assessment template configurations.                                                                                                                              | Manage Assessments                                       | [Assessment -> Templates](https://app.transcend.io/assessments/form-templates)                                                                                                                                                                         |
 | `purposes`              | `purposes`                | Consent purposes and related preference management topics.                                                                                                       | Manage Consent Manager, Manage Preference Store Settings | [Consent Management -> Regional Experiences -> Purposes](https://app.transcend.io/consent-manager/regional-experiences/purposes)                                                                                                                       |
+| `siloDiscoveryResults`  | `siloDiscoveryResults`    | System discovery results                                                                                                                                         | Manage Data Map                                          | [System Discovery](https://app.transcend.io/data-map/data-inventory/silo-discovery)                                                                                                                                                                    |
 #### Examples

package/dist/bin/bash-complete.cjs CHANGED Viewed

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
-"use strict"; function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkT462ONFXcjs = require('../chunk-T462ONFX.cjs');var _chunkY35YHYXPcjs = require('../chunk-Y35YHYXP.cjs');require('../chunk-FER7XKVN.cjs');require('../chunk-EYOQLGDV.cjs');require('../chunk-Q7I37FJV.cjs');var _core = require('@stricli/core');async function e(){let o=process.argv.slice(3);_optionalChain([process, 'access', _ => _.env, 'access', _2 => _2.COMP_LINE, 'optionalAccess', _3 => _3.endsWith, 'call', _4 => _4(" ")])&&o.push(""),await _core.proposeCompletions.call(void 0, _chunkY35YHYXPcjs.a,o,_chunkT462ONFXcjs.a.call(void 0, process));try{for(let{completion:r}of await _core.proposeCompletions.call(void 0, _chunkY35YHYXPcjs.a,o,_chunkT462ONFXcjs.a.call(void 0, process)))process.stdout.write(`${r}
+"use strict"; function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkT462ONFXcjs = require('../chunk-T462ONFX.cjs');var _chunkU7B2ZKV5cjs = require('../chunk-U7B2ZKV5.cjs');require('../chunk-BKPEELIQ.cjs');require('../chunk-A3GRLGNU.cjs');require('../chunk-T3BBQIRY.cjs');var _core = require('@stricli/core');async function e(){let o=process.argv.slice(3);_optionalChain([process, 'access', _ => _.env, 'access', _2 => _2.COMP_LINE, 'optionalAccess', _3 => _3.endsWith, 'call', _4 => _4(" ")])&&o.push(""),await _core.proposeCompletions.call(void 0, _chunkU7B2ZKV5cjs.a,o,_chunkT462ONFXcjs.a.call(void 0, process));try{for(let{completion:r}of await _core.proposeCompletions.call(void 0, _chunkU7B2ZKV5cjs.a,o,_chunkT462ONFXcjs.a.call(void 0, process)))process.stdout.write(`${r}
 `)}catch (e2){}}e();
 //# sourceMappingURL=bash-complete.cjs.map

package/dist/bin/cli.cjs CHANGED Viewed

@@ -1,3 +1,3 @@
 #!/usr/bin/env node
-"use strict";var _chunkT462ONFXcjs = require('../chunk-T462ONFX.cjs');var _chunkY35YHYXPcjs = require('../chunk-Y35YHYXP.cjs');require('../chunk-FER7XKVN.cjs');require('../chunk-EYOQLGDV.cjs');require('../chunk-Q7I37FJV.cjs');var _core = require('@stricli/core');async function n(){await _core.run.call(void 0, _chunkY35YHYXPcjs.a,process.argv.slice(2),_chunkT462ONFXcjs.a.call(void 0, process))}n();
+"use strict";var _chunkT462ONFXcjs = require('../chunk-T462ONFX.cjs');var _chunkU7B2ZKV5cjs = require('../chunk-U7B2ZKV5.cjs');require('../chunk-BKPEELIQ.cjs');require('../chunk-A3GRLGNU.cjs');require('../chunk-T3BBQIRY.cjs');var _core = require('@stricli/core');async function n(){await _core.run.call(void 0, _chunkU7B2ZKV5cjs.a,process.argv.slice(2),_chunkT462ONFXcjs.a.call(void 0, process))}n();
 //# sourceMappingURL=cli.cjs.map

package/dist/bin/deprecated-command.cjs CHANGED Viewed

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
-"use strict"; function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkY35YHYXPcjs = require('../chunk-Y35YHYXP.cjs');require('../chunk-FER7XKVN.cjs');var _chunkZUNVPK23cjs = require('../chunk-ZUNVPK23.cjs');require('../chunk-EYOQLGDV.cjs');require('../chunk-Q7I37FJV.cjs');var _core = require('@stricli/core');var o={"tr-build-xdi-sync-endpoint":["consent","build-xdi-sync-endpoint"],"tr-consent-manager-service-json-to-yml":["inventory","consent-manager-service-json-to-yml"],"tr-consent-managers-to-business-entities":["inventory","consent-managers-to-business-entities"],"tr-cron-mark-identifiers-completed":["request","cron","mark-identifiers-completed"],"tr-cron-pull-identifiers":["request","cron","pull-identifiers"],"tr-derive-data-silos-from-data-flows":["inventory","derive-data-silos-from-data-flows"],"tr-derive-data-silos-from-data-flows-cross-instance":["inventory","derive-data-silos-from-data-flows-cross-instance"],"tr-discover-silos":["inventory","discover-silos"],"tr-generate-api-keys":["admin","generate-api-keys"],"tr-manual-enrichment-pull-identifiers":["request","preflight","pull-identifiers"],"tr-manual-enrichment-push-identifiers":["request","preflight","push-identifiers"],"tr-mark-request-data-silos-completed":["request","system","mark-request-data-silos-completed"],"tr-pull":["inventory","pull"],"tr-pull-consent-metrics":["consent","pull-consent-metrics"],"tr-pull-consent-preferences":["consent","pull-consent-preferences"],"tr-pull-datapoints":["inventory","pull-datapoints"],"tr-pull-pull-unstructured-discovery-files":["inventory","pull-unstructured-discovery-files"],"tr-push":["inventory","push"],"tr-request-approve":["request","approve"],"tr-request-cancel":["request","cancel"],"tr-request-download-files":["request","download-files"],"tr-request-enricher-restart":["request","enricher-restart"],"tr-request-export":["request","export"],"tr-request-mark-silent":["request","mark-silent"],"tr-request-notify-additional-time":["request","notify-additional-time"],"tr-request-reject-unverified-identifiers":["request","reject-unverified-identifiers"],"tr-request-restart":["request","restart"],"tr-request-upload":["request","upload"],"tr-retry-request-data-silos":["request","system","retry-request-data-silos"],"tr-scan-packages":["inventory","scan-packages"],"tr-skip-preflight-jobs":["request","skip-preflight-jobs"],"tr-skip-request-data-silos":["request","system","skip-request-data-silos"],"tr-sync-ot":["migration","sync-ot"],"tr-update-consent-manager":["consent","update-consent-manager"],"tr-upload-consent-preferences":["consent","upload-consent-preferences"],"tr-upload-cookies-from-csv":["consent","upload-cookies-from-csv"],"tr-upload-data-flows-from-csv":["consent","upload-data-flows-from-csv"],"tr-upload-preferences":["consent","upload-preferences"]};function p(t){return _optionalChain([_core.generateHelpTextForAllCommands.call(void 0, _chunkY35YHYXPcjs.a), 'access', _ => _.find, 'call', _2 => _2(s=>s[0]===`${_chunkY35YHYXPcjs.a.config.name} ${t.join(" ")}`), 'optionalAccess', _3 => _3[1]])}function i(t){_chunkZUNVPK23cjs.a.log("[DEPRECATION NOTICE]");let e=o[t];if(!e){let a=Object.entries(o).map(([l,d])=>`\`${l}\` -> \`${_chunkY35YHYXPcjs.a.config.name} ${d.join(" ")}\``).join(`
+"use strict"; function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkU7B2ZKV5cjs = require('../chunk-U7B2ZKV5.cjs');require('../chunk-BKPEELIQ.cjs');var _chunkZUNVPK23cjs = require('../chunk-ZUNVPK23.cjs');require('../chunk-A3GRLGNU.cjs');require('../chunk-T3BBQIRY.cjs');var _core = require('@stricli/core');var o={"tr-build-xdi-sync-endpoint":["consent","build-xdi-sync-endpoint"],"tr-consent-manager-service-json-to-yml":["inventory","consent-manager-service-json-to-yml"],"tr-consent-managers-to-business-entities":["inventory","consent-managers-to-business-entities"],"tr-cron-mark-identifiers-completed":["request","cron","mark-identifiers-completed"],"tr-cron-pull-identifiers":["request","cron","pull-identifiers"],"tr-derive-data-silos-from-data-flows":["inventory","derive-data-silos-from-data-flows"],"tr-derive-data-silos-from-data-flows-cross-instance":["inventory","derive-data-silos-from-data-flows-cross-instance"],"tr-discover-silos":["inventory","discover-silos"],"tr-generate-api-keys":["admin","generate-api-keys"],"tr-manual-enrichment-pull-identifiers":["request","preflight","pull-identifiers"],"tr-manual-enrichment-push-identifiers":["request","preflight","push-identifiers"],"tr-mark-request-data-silos-completed":["request","system","mark-request-data-silos-completed"],"tr-pull":["inventory","pull"],"tr-pull-consent-metrics":["consent","pull-consent-metrics"],"tr-pull-consent-preferences":["consent","pull-consent-preferences"],"tr-pull-datapoints":["inventory","pull-datapoints"],"tr-pull-pull-unstructured-discovery-files":["inventory","pull-unstructured-discovery-files"],"tr-push":["inventory","push"],"tr-request-approve":["request","approve"],"tr-request-cancel":["request","cancel"],"tr-request-download-files":["request","download-files"],"tr-request-enricher-restart":["request","enricher-restart"],"tr-request-export":["request","export"],"tr-request-mark-silent":["request","mark-silent"],"tr-request-notify-additional-time":["request","notify-additional-time"],"tr-request-reject-unverified-identifiers":["request","reject-unverified-identifiers"],"tr-request-restart":["request","restart"],"tr-request-upload":["request","upload"],"tr-retry-request-data-silos":["request","system","retry-request-data-silos"],"tr-scan-packages":["inventory","scan-packages"],"tr-skip-preflight-jobs":["request","skip-preflight-jobs"],"tr-skip-request-data-silos":["request","system","skip-request-data-silos"],"tr-sync-ot":["migration","sync-ot"],"tr-update-consent-manager":["consent","update-consent-manager"],"tr-upload-consent-preferences":["consent","upload-consent-preferences"],"tr-upload-cookies-from-csv":["consent","upload-cookies-from-csv"],"tr-upload-data-flows-from-csv":["consent","upload-data-flows-from-csv"],"tr-upload-preferences":["consent","upload-preferences"]};function p(t){return _optionalChain([_core.generateHelpTextForAllCommands.call(void 0, _chunkU7B2ZKV5cjs.a), 'access', _ => _.find, 'call', _2 => _2(s=>s[0]===`${_chunkU7B2ZKV5cjs.a.config.name} ${t.join(" ")}`), 'optionalAccess', _3 => _3[1]])}function i(t){_chunkZUNVPK23cjs.a.log("[DEPRECATION NOTICE]");let e=o[t];if(!e){let a=Object.entries(o).map(([l,d])=>`\`${l}\` -> \`${_chunkU7B2ZKV5cjs.a.config.name} ${d.join(" ")}\``).join(`
 `);_chunkZUNVPK23cjs.a.log(`This command is deprecated as of v7.0.0. Here is a list of new commands, mapped to their legacy command names:
 ${a}`);return}_chunkZUNVPK23cjs.a.log(`\`${t}\` is deprecated as of v7.0.0.
-Use \`${_chunkY35YHYXPcjs.a.config.name} ${e.join(" ")}\` instead.
+Use \`${_chunkU7B2ZKV5cjs.a.config.name} ${e.join(" ")}\` instead.
 `);let s=p(e);if(!s)throw new Error(`Failed to get help text for command: \`${e.join(" ")}\``);_chunkZUNVPK23cjs.a.log(s)}function m(){let e=_optionalChain([process, 'access', _4 => _4.argv, 'access', _5 => _5.at, 'call', _6 => _6(-1), 'optionalAccess', _7 => _7.split, 'call', _8 => _8("/"), 'access', _9 => _9.pop, 'call', _10 => _10(), 'optionalAccess', _11 => _11.trim, 'call', _12 => _12()]);if(e)i(e);else throw new Error("Deprecated command");process.exit(1)}m();
 //# sourceMappingURL=deprecated-command.cjs.map

package/dist/{chunk-THOOR2KA.cjs → chunk-4WBBQCOV.cjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkKUBXRT3Vcjs = require('./chunk-KUBXRT3V.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkEYOQLGDVcjs = require('./chunk-EYOQLGDV.cjs');var _privacytypes = require('@transcend-io/privacy-types');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _graphqlrequest = require('graphql-request');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function q(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=[],m=new Date().getTime(),d=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),s={...c.length>0?{category:c}:{},...t.length>0?{subCategoryIds:t}:{},...c.length+t.length>0&&!l?{status:_privacytypes.SubDataPointDataSubCategoryGuessStatus.Approved}:{},...e.length>0?{dataSilos:e}:{}},{subDataPoints:{totalCount:o}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_chunkKUBXRT3Vcjs.d,{filterBy:s});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),d.start(o,0);let y=0,D=!1,r,b=0;do try{let{subDataPoints:{nodes:P}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_graphqlrequest.gql`
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunk56COHDFTcjs = require('./chunk-56COHDFT.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkA3GRLGNUcjs = require('./chunk-A3GRLGNU.cjs');var _privacytypes = require('@transcend-io/privacy-types');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _graphqlrequest = require('graphql-request');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function q(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=[],m=new Date().getTime(),d=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),s={...c.length>0?{category:c}:{},...t.length>0?{subCategoryIds:t}:{},...c.length+t.length>0&&!l?{status:_privacytypes.SubDataPointDataSubCategoryGuessStatus.Approved}:{},...e.length>0?{dataSilos:e}:{}},{subDataPoints:{totalCount:o}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_chunk56COHDFTcjs.d,{filterBy:s});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),d.start(o,0);let y=0,D=!1,r,b=0;do try{let{subDataPoints:{nodes:P}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_graphqlrequest.gql`
           query TranscendCliSubDataPointCsvExport(
             $filterBy: SubDataPointFiltersInput
             $first: Int!
@@ -41,7 +41,7 @@
               }
             }
           }
-        `,{first:p,offset:b,filterBy:{...s}});r=_optionalChain([P, 'access', _2 => _2[P.length-1], 'optionalAccess', _3 => _3.id]),n.push(...P),D=P.length===p,y+=P.length,b+=P.length,d.update(y)}catch(P){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${r} and offset ${b}`)),P}while(D);d.stop();let C=new Date().getTime()-m,g=_chunkEYOQLGDVcjs.g.call(void 0, n,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${g.length} subdatapoints in ${C/1e3} seconds!`)),g}async function F(u,{dataPointIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 2/3] Fetching metadata for ${e.length} datapoints`));let p=_chunkEYOQLGDVcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataPoints:{nodes:o}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_chunkKUBXRT3Vcjs.g,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} dataPoints in ${d/1e3} seconds!`)),a}async function Q(u,{dataSiloIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 3/3] Fetching metadata for ${e.length} data silos`));let p=_chunkEYOQLGDVcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataSilos:{nodes:o}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_chunkKUBXRT3Vcjs.j,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching data silos for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} data silos in ${d/1e3} seconds!`)),a}async function Y(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=await q(u,{dataSiloIds:e,includeGuessedCategories:l,includeAttributes:a,parentCategories:c,subCategories:t,pageSize:p}),m=_chunkEYOQLGDVcjs.j.call(void 0, n.map(r=>r.dataPointId)),d=await F(u,{dataPointIds:m}),s=_chunkEYOQLGDVcjs.e.call(void 0, d,"id"),o=_chunkEYOQLGDVcjs.j.call(void 0, n.map(r=>r.dataSiloId)),y=await Q(u,{dataSiloIds:o}),D=_chunkEYOQLGDVcjs.e.call(void 0, y,"id");return n.map(r=>({...r,dataPoint:s[r.dataPointId],dataSilo:D[r.dataSiloId]}))}async function nt(u,{dataSiloIds:e=[],status:l,subCategories:a=[],includeEncryptedSnippets:c,pageSize:t=100}={}){let p=[],n=new Date().getTime(),m=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d={...a.length>0?{subCategoryIds:a}:{},...l?{status:l}:{},...e.length>0?{dataSilos:e}:{}},{unstructuredSubDataPointRecommendations:{totalCount:s}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_chunkKUBXRT3Vcjs.h,{filterBy:d});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),m.start(s,0);let o=0,y=!1,D,r=0;do try{let{unstructuredSubDataPointRecommendations:{nodes:g}}=await _chunkKUBXRT3Vcjs.sg.call(void 0, u,_graphqlrequest.gql`
+        `,{first:p,offset:b,filterBy:{...s}});r=_optionalChain([P, 'access', _2 => _2[P.length-1], 'optionalAccess', _3 => _3.id]),n.push(...P),D=P.length===p,y+=P.length,b+=P.length,d.update(y)}catch(P){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${r} and offset ${b}`)),P}while(D);d.stop();let C=new Date().getTime()-m,g=_chunkA3GRLGNUcjs.g.call(void 0, n,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${g.length} subdatapoints in ${C/1e3} seconds!`)),g}async function F(u,{dataPointIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 2/3] Fetching metadata for ${e.length} datapoints`));let p=_chunkA3GRLGNUcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataPoints:{nodes:o}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_chunk56COHDFTcjs.g,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} dataPoints in ${d/1e3} seconds!`)),a}async function Q(u,{dataSiloIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 3/3] Fetching metadata for ${e.length} data silos`));let p=_chunkA3GRLGNUcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataSilos:{nodes:o}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_chunk56COHDFTcjs.j,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching data silos for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} data silos in ${d/1e3} seconds!`)),a}async function Y(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=await q(u,{dataSiloIds:e,includeGuessedCategories:l,includeAttributes:a,parentCategories:c,subCategories:t,pageSize:p}),m=_chunkA3GRLGNUcjs.j.call(void 0, n.map(r=>r.dataPointId)),d=await F(u,{dataPointIds:m}),s=_chunkA3GRLGNUcjs.e.call(void 0, d,"id"),o=_chunkA3GRLGNUcjs.j.call(void 0, n.map(r=>r.dataSiloId)),y=await Q(u,{dataSiloIds:o}),D=_chunkA3GRLGNUcjs.e.call(void 0, y,"id");return n.map(r=>({...r,dataPoint:s[r.dataPointId],dataSilo:D[r.dataSiloId]}))}async function nt(u,{dataSiloIds:e=[],status:l,subCategories:a=[],includeEncryptedSnippets:c,pageSize:t=100}={}){let p=[],n=new Date().getTime(),m=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d={...a.length>0?{subCategoryIds:a}:{},...l?{status:l}:{},...e.length>0?{dataSilos:e}:{}},{unstructuredSubDataPointRecommendations:{totalCount:s}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_chunk56COHDFTcjs.h,{filterBy:d});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),m.start(s,0);let o=0,y=!1,D,r=0;do try{let{unstructuredSubDataPointRecommendations:{nodes:g}}=await _chunk56COHDFTcjs.ug.call(void 0, u,_graphqlrequest.gql`
           query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(
             $filterBy: UnstructuredSubDataPointRecommendationsFilterInput
             $first: Int!
@@ -71,5 +71,5 @@
               }
             }
           }
-        `,{first:t,offset:r,filterBy:{...d}});D=_optionalChain([g, 'access', _4 => _4[g.length-1], 'optionalAccess', _5 => _5.id]),p.push(...g),y=g.length===t,o+=g.length,r+=g.length,m.update(o)}catch(g){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${D} and offset ${r}`)),g}while(y);m.stop();let T=new Date().getTime()-n,C=_chunkEYOQLGDVcjs.g.call(void 0, p,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${C.length} subdatapoints in ${T/1e3} seconds!`)),C}exports.a = Y; exports.b = nt;
-//# sourceMappingURL=chunk-THOOR2KA.cjs.map
+        `,{first:t,offset:r,filterBy:{...d}});D=_optionalChain([g, 'access', _4 => _4[g.length-1], 'optionalAccess', _5 => _5.id]),p.push(...g),y=g.length===t,o+=g.length,r+=g.length,m.update(o)}catch(g){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${D} and offset ${r}`)),g}while(y);m.stop();let T=new Date().getTime()-n,C=_chunkA3GRLGNUcjs.g.call(void 0, p,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${C.length} subdatapoints in ${T/1e3} seconds!`)),C}exports.a = Y; exports.b = nt;
+//# sourceMappingURL=chunk-4WBBQCOV.cjs.map

package/dist/{chunk-THOOR2KA.cjs.map → chunk-4WBBQCOV.cjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-THOOR2KA.cjs","../src/lib/data-inventory/pullAllDatapoints.ts","../src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts"],"names":["pullSubDatapoints","client","dataSiloIds","includeGuessedCategories","includeAttributes","parentCategories","subCategories","pageSize","subDataPoints","t0","progressBar","cliProgress","filterBy","SubDataPointDataSubCategoryGuessStatus","totalCount","makeGraphQLRequest","SUB_DATA_POINTS_COUNT","logger","colors","total","shouldContinue","cursor","offset","nodes","gql","err"],"mappings":"AAAA,quBAAqE,wDAAyC,wDAA8D,2DCKrK,qGACiB,iDACJ,gFACD,oCAWO,MAuE1B,SAAeA,CAAAA,CACbC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CAAc,CAAC,CAAA,CACf,wBAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CAAmB,CAAC,CAAA,CACpB,aAAA,CAAAC,CAAAA,CAAgB,CAAC,CAAA,CACjB,QAAA,CAAAC,CAAAA,CAAW,GACb,CAAA,CAGI,CAAC,CAAA,CAC8B,CACnC,IAAMC,CAAAA,CAA0C,CAAC,CAAA,CAG3CC,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CAGxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CAGMC,CAAAA,CAAW,CACf,GAAIP,CAAAA,CAAiB,MAAA,CAAS,CAAA,CAAI,CAAE,QAAA,CAAUA,CAAiB,CAAA,CAAI,CAAC,CAAA,CACpE,GAAIC,CAAAA,CAAc,MAAA,CAAS,CAAA,CAAI,CAAE,cAAA,CAAgBA,CAAc,CAAA,CAAI,CAAC,CAAA,CAEpE,GAAID,CAAAA,CAAiB,MAAA,CAASC,CAAAA,CAAc,MAAA,CAAS,CAAA,EACrD,CAACH,CAAAA,CAEG,CAAE,MAAA,CAAQU,oDAAAA,CAAuC,QAAS,CAAA,CAC1D,CAAC,CAAA,CACL,GAAIX,CAAAA,CAAY,MAAA,CAAS,CAAA,CAAI,CAAE,SAAA,CAAWA,CAAY,CAAA,CAAI,CAAC,CAC7D,CAAA,CAGM,CACJ,aAAA,CAAe,CAAE,UAAA,CAAAY,CAAW,CAC9B,CAAA,CAAI,MAAMC,kCAAAA,CAMPd,CAAQe,mBAAAA,CAAuB,CAChC,QAAA,CAAAJ,CACF,CAAC,CAAA,CAEDK,mBAAAA,CAAO,IAAA,CAAKC,gBAAAA,CAAO,OAAA,CAAQ,yCAAyC,CAAC,CAAA,CAErER,CAAAA,CAAY,KAAA,CAAMI,CAAAA,CAAY,CAAC,CAAA,CAC/B,IAAIK,CAAAA,CAAQ,CAAA,CACRC,CAAAA,CAAiB,CAAA,CAAA,CACjBC,CAAAA,CACAC,CAAAA,CAAS,CAAA,CACb,GACE,GAAI,CACF,GAAM,CACJ,aAAA,CAAe,CAAE,KAAA,CAAAC,CAAM,CACzB,CAAA,CAAI,MAAMR,kCAAAA,CAORd,CACAuB,mBAAAA,CAAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAAA,EA2BUrB,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAQA,EACN,CAAA;AAAA,gBAAA,EAEEC,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAMA,EACN,CAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA,CAKR,CACE,KAAA,CAAOG,CAAAA,CACP,MAAA,CAAAe,CAAAA,CACA,QAAA,CAAU,CACR,GAAGV,CAGL,CACF,CACF,CAAA,CAEAS,CAAAA,iBAASE,CAAAA,qBAAMA,CAAAA,CAAM,MAAA,CAAS,CAAC,CAAA,6BAAG,IAAA,CAClCf,CAAAA,CAAc,IAAA,CAAK,GAAGe,CAAK,CAAA,CAC3BH,CAAAA,CAAiBG,CAAAA,CAAM,MAAA,GAAWhB,CAAAA,CAClCY,CAAAA,EAASI,CAAAA,CAAM,MAAA,CACfD,CAAAA,EAAUC,CAAAA,CAAM,MAAA,CAChBb,CAAAA,CAAY,MAAA,CAAOS,CAAK,CAC1B,CAAA,KAAA,CAASM,CAAAA,CAAK,CACZ,MAAAR,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,2CAAA,EAA8CG,CAAM,CAAA,YAAA,EAAeC,CAAM,CAAA,CAAA;AC7G3E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBgD,gBAAA;AACU,gBAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+BiB,QAAA","file":"/home/runner/work/cli/cli/dist/chunk-THOOR2KA.cjs","sourcesContent":[null,"/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\nimport {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport { gql } from 'graphql-request';\nimport colors from 'colors';\nimport type { GraphQLClient } from 'graphql-request';\nimport {\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n type DataSiloAttributeValue,\n SUB_DATA_POINTS_COUNT,\n makeGraphQLRequest,\n} from '../graphql';\nimport { logger } from '../../logger';\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs';\nimport { mapSeries } from 'bluebird';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 &&\n !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`,\n ),\n );\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(\n ', ',\n )}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`,\n ),\n );\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines /\n","import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\nimport { sortBy } from 'lodash-es';\nimport type { DataCategoryInput } from '../../codecs';\nimport { ENTRY_COUNT, makeGraphQLRequest } from '../graphql';\nimport { logger } from '../../logger';\n\ninterface UnstructuredSubDataPointRecommendationCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Entry or Named Entity recognized by the classifier /\n name: string;\n /* Context snippet including entry /\n contextSnippet: string;\n /* Scanned object ID /\n scannedObjectId: string;\n /* Scanned object path ID /\n scannedObjectPathId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* Personal data category /\n dataSubCategory: DataCategoryInput;\n /* Classification Status /\n status: UnstructuredSubDataPointRecommendationStatus;\n /* Confidence /\n confidence: number;\n /* Classification method /\n classificationMethod: string;\n /* Classifier version /\n classifierVersion: string;\n}\n\ninterface EntryFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Parent categories to filter down for /\n status?: UnstructuredSubDataPointRecommendationStatus[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n /* Include entry and snippet /\n includeEncryptedSnippets?: boolean;\n /* Include encryptedSamplesS3Key /\n includeEncryptedSamplesS3Key?: boolean;\n}\n/\n Pull unstructured subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @param options.dataSiloIds - IDs of data silos to filter down\n * @param options.status - Parent categories to filter down for\n * @param options.subCategories - Sub categories to filter down for\n * @param options.includeEncryptedSnippets - Include entry and snippet\n * @param options.includeEncryptedSamplesS3Key - Include encryptedSamplesS3Key\n * @param options.pageSize - Page size to pull in\n * @returns A promise that resolves to an array of unstructured subdatapoint recommendations\n /\nexport async function pullUnstructuredSubDataPointRecommendations(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n status,\n subCategories = [],\n includeEncryptedSnippets,\n pageSize = 100,\n }: EntryFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<UnstructuredSubDataPointRecommendationCsvPreview[]> {\n const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] =\n [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n ...(status ? { status } : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n unstructuredSubDataPointRecommendations: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* Count /\n totalCount: number;\n };\n }>(client, ENTRY_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n unstructuredSubDataPointRecommendations: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* List of matches */\n nodes: UnstructuredSubDataPointRecommendationCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(\n $filterBy: UnstructuredSubDataPointRecommendationsFilterInput\n $first: Int!\n $offset: Int!\n ) {\n unstructuredSubDataPointRecommendations(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n dataSiloId\n scannedObjectPathId\n scannedObjectId\n ${includeEncryptedSnippets ? 'name' : ''}\n ${includeEncryptedSnippets ? 'contextSnippet' : ''}\n dataSubCategory {\n name\n category\n }\n status\n confidence\n classificationMethod\n classifierVersion\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n unstructuredSubDataPointRecommendations.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n"]}
1	+ {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-4WBBQCOV.cjs","../src/lib/data-inventory/pullAllDatapoints.ts","../src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts"],"names":["pullSubDatapoints","client","dataSiloIds","includeGuessedCategories","includeAttributes","parentCategories","subCategories","pageSize","subDataPoints","t0","progressBar","cliProgress","filterBy","SubDataPointDataSubCategoryGuessStatus","totalCount","makeGraphQLRequest","SUB_DATA_POINTS_COUNT","logger","colors","total","shouldContinue","cursor","offset","nodes","gql","err"],"mappings":"AAAA,quBAAqE,wDAAyC,wDAA8D,2DCKrK,qGACiB,iDACJ,gFACD,oCAWO,MAuE1B,SAAeA,CAAAA,CACbC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CAAc,CAAC,CAAA,CACf,wBAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CAAmB,CAAC,CAAA,CACpB,aAAA,CAAAC,CAAAA,CAAgB,CAAC,CAAA,CACjB,QAAA,CAAAC,CAAAA,CAAW,GACb,CAAA,CAGI,CAAC,CAAA,CAC8B,CACnC,IAAMC,CAAAA,CAA0C,CAAC,CAAA,CAG3CC,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CAGxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CAGMC,CAAAA,CAAW,CACf,GAAIP,CAAAA,CAAiB,MAAA,CAAS,CAAA,CAAI,CAAE,QAAA,CAAUA,CAAiB,CAAA,CAAI,CAAC,CAAA,CACpE,GAAIC,CAAAA,CAAc,MAAA,CAAS,CAAA,CAAI,CAAE,cAAA,CAAgBA,CAAc,CAAA,CAAI,CAAC,CAAA,CAEpE,GAAID,CAAAA,CAAiB,MAAA,CAASC,CAAAA,CAAc,MAAA,CAAS,CAAA,EACrD,CAACH,CAAAA,CAEG,CAAE,MAAA,CAAQU,oDAAAA,CAAuC,QAAS,CAAA,CAC1D,CAAC,CAAA,CACL,GAAIX,CAAAA,CAAY,MAAA,CAAS,CAAA,CAAI,CAAE,SAAA,CAAWA,CAAY,CAAA,CAAI,CAAC,CAC7D,CAAA,CAGM,CACJ,aAAA,CAAe,CAAE,UAAA,CAAAY,CAAW,CAC9B,CAAA,CAAI,MAAMC,kCAAAA,CAMPd,CAAQe,mBAAAA,CAAuB,CAChC,QAAA,CAAAJ,CACF,CAAC,CAAA,CAEDK,mBAAAA,CAAO,IAAA,CAAKC,gBAAAA,CAAO,OAAA,CAAQ,yCAAyC,CAAC,CAAA,CAErER,CAAAA,CAAY,KAAA,CAAMI,CAAAA,CAAY,CAAC,CAAA,CAC/B,IAAIK,CAAAA,CAAQ,CAAA,CACRC,CAAAA,CAAiB,CAAA,CAAA,CACjBC,CAAAA,CACAC,CAAAA,CAAS,CAAA,CACb,GACE,GAAI,CACF,GAAM,CACJ,aAAA,CAAe,CAAE,KAAA,CAAAC,CAAM,CACzB,CAAA,CAAI,MAAMR,kCAAAA,CAORd,CACAuB,mBAAAA,CAAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAAA,EA2BUrB,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAQA,EACN,CAAA;AAAA,gBAAA,EAEEC,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAMA,EACN,CAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA,CAKR,CACE,KAAA,CAAOG,CAAAA,CACP,MAAA,CAAAe,CAAAA,CACA,QAAA,CAAU,CACR,GAAGV,CAGL,CACF,CACF,CAAA,CAEAS,CAAAA,iBAASE,CAAAA,qBAAMA,CAAAA,CAAM,MAAA,CAAS,CAAC,CAAA,6BAAG,IAAA,CAClCf,CAAAA,CAAc,IAAA,CAAK,GAAGe,CAAK,CAAA,CAC3BH,CAAAA,CAAiBG,CAAAA,CAAM,MAAA,GAAWhB,CAAAA,CAClCY,CAAAA,EAASI,CAAAA,CAAM,MAAA,CACfD,CAAAA,EAAUC,CAAAA,CAAM,MAAA,CAChBb,CAAAA,CAAY,MAAA,CAAOS,CAAK,CAC1B,CAAA,KAAA,CAASM,CAAAA,CAAK,CACZ,MAAAR,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,2CAAA,EAA8CG,CAAM,CAAA,YAAA,EAAeC,CAAM,CAAA,CAAA;AC7G3E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBgD,gBAAA;AACU,gBAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+BiB,QAAA","file":"/home/runner/work/cli/cli/dist/chunk-4WBBQCOV.cjs","sourcesContent":[null,"/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\nimport {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport { gql } from 'graphql-request';\nimport colors from 'colors';\nimport type { GraphQLClient } from 'graphql-request';\nimport {\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n type DataSiloAttributeValue,\n SUB_DATA_POINTS_COUNT,\n makeGraphQLRequest,\n} from '../graphql';\nimport { logger } from '../../logger';\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs';\nimport { mapSeries } from 'bluebird';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 &&\n !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`,\n ),\n );\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(\n ', ',\n )}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`,\n ),\n );\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines /\n","import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\nimport { sortBy } from 'lodash-es';\nimport type { DataCategoryInput } from '../../codecs';\nimport { ENTRY_COUNT, makeGraphQLRequest } from '../graphql';\nimport { logger } from '../../logger';\n\ninterface UnstructuredSubDataPointRecommendationCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Entry or Named Entity recognized by the classifier /\n name: string;\n /* Context snippet including entry /\n contextSnippet: string;\n /* Scanned object ID /\n scannedObjectId: string;\n /* Scanned object path ID /\n scannedObjectPathId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* Personal data category /\n dataSubCategory: DataCategoryInput;\n /* Classification Status /\n status: UnstructuredSubDataPointRecommendationStatus;\n /* Confidence /\n confidence: number;\n /* Classification method /\n classificationMethod: string;\n /* Classifier version /\n classifierVersion: string;\n}\n\ninterface EntryFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Parent categories to filter down for /\n status?: UnstructuredSubDataPointRecommendationStatus[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n /* Include entry and snippet /\n includeEncryptedSnippets?: boolean;\n /* Include encryptedSamplesS3Key /\n includeEncryptedSamplesS3Key?: boolean;\n}\n/\n Pull unstructured subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @param options.dataSiloIds - IDs of data silos to filter down\n * @param options.status - Parent categories to filter down for\n * @param options.subCategories - Sub categories to filter down for\n * @param options.includeEncryptedSnippets - Include entry and snippet\n * @param options.includeEncryptedSamplesS3Key - Include encryptedSamplesS3Key\n * @param options.pageSize - Page size to pull in\n * @returns A promise that resolves to an array of unstructured subdatapoint recommendations\n /\nexport async function pullUnstructuredSubDataPointRecommendations(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n status,\n subCategories = [],\n includeEncryptedSnippets,\n pageSize = 100,\n }: EntryFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<UnstructuredSubDataPointRecommendationCsvPreview[]> {\n const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] =\n [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n ...(status ? { status } : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n unstructuredSubDataPointRecommendations: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* Count /\n totalCount: number;\n };\n }>(client, ENTRY_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n unstructuredSubDataPointRecommendations: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* List of matches */\n nodes: UnstructuredSubDataPointRecommendationCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(\n $filterBy: UnstructuredSubDataPointRecommendationsFilterInput\n $first: Int!\n $offset: Int!\n ) {\n unstructuredSubDataPointRecommendations(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n dataSiloId\n scannedObjectPathId\n scannedObjectId\n ${includeEncryptedSnippets ? 'name' : ''}\n ${includeEncryptedSnippets ? 'contextSnippet' : ''}\n dataSubCategory {\n name\n category\n }\n status\n confidence\n classificationMethod\n classifierVersion\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n unstructuredSubDataPointRecommendations.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n"]}

package/dist/{chunk-I7PFP5VB.cjs → chunk-4Y7PZGF3.cjs} RENAMED Viewed

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkKUBXRT3Vcjs = require('./chunk-KUBXRT3V.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkEYOQLGDVcjs = require('./chunk-EYOQLGDV.cjs');var _iots = require('io-ts'); var e = _interopRequireWildcard(_iots); var P = _interopRequireWildcard(_iots);var _typeutils = require('@transcend-io/type-utils');var X=e.type({identifier:e.string,type:e.string,coreIdentifier:e.string,dataSiloId:e.string,requestId:e.string,nonce:e.string,requestCreatedAt:e.string,daysUntilOverdue:e.number,attributes:e.array(e.type({key:e.string,values:e.array(e.string)}))});async function U(r,{dataSiloId:s,limit:c=100,offset:n=0,requestType:a}){try{let i=await r.get(`v1/data-silo/${s}/pending-requests/${a}`,{searchParams:{offset:n,limit:c}}).json(),{items:f}=_typeutils.decodeCodec.call(void 0, e.type({items:e.array(X)}),i);return f}catch(i){throw new Error(`Received an error from server: ${_optionalChain([i, 'optionalAccess', _2 => _2.response, 'optionalAccess', _3 => _3.body])||_optionalChain([i, 'optionalAccess', _4 => _4.message])}`)}}var L=P.type({nonce:P.string,identifier:P.string});async function B(r,{nonce:s,identifier:c}){try{return await r.put("v1/data-silo",{headers:{"x-transcend-nonce":s},json:{profiles:[{profileId:c}]}}),!0}catch(n){if(_optionalChain([n, 'access', _5 => _5.response, 'optionalAccess', _6 => _6.statusCode])===409)return!1;throw new Error(`Received an error from server: ${_optionalChain([n, 'optionalAccess', _7 => _7.response, 'optionalAccess', _8 => _8.body])||_optionalChain([n, 'optionalAccess', _9 => _9.message])}`)}}var _bluebird = require('bluebird');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);async function ge({file:r,dataSiloId:s,auth:c,sombraAuth:n,concurrency:a=100,transcendUrl:i=_chunkEYOQLGDVcjs.r,sleepSeconds:f=10}){let y=await _chunkKUBXRT3Vcjs.xc.call(void 0, i,c,n);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Reading "${r}" from disk`));let o=_chunkKUBXRT3Vcjs.rc.call(void 0, r,L);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Notifying Transcend for data silo "${s}" marking "${o.length}" identifiers as completed.`));let C=new Date().getTime(),w=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d=0,u=0,l=0;w.start(o.length,0);let m=_chunkEYOQLGDVcjs.b.call(void 0, o,a),I=m.length;await _bluebird.mapSeries.call(void 0, m,async($,S)=>{_chunkZUNVPK23cjs.a.info(_colors2.default.blue(`Processing chunk ${S+1}/${I} (${_chunkEYOQLGDVcjs.b.length} items)`)),await _bluebird.map.call(void 0, $,async h=>{try{await B(y,h)?d+=1:u+=1}catch(b){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Error notifying Transcend for identifier "${h.identifier}" - ${_optionalChain([b, 'optionalAccess', _10 => _10.message])}`)),l+=1}w.update(d+u)}),f>0&&S<I-1&&(_chunkZUNVPK23cjs.a.info(_colors2.default.yellow(`Sleeping for ${f}s before next chunk...`)),await new Promise(h=>{setTimeout(h,f*1e3)}))}),w.stop();let D=new Date().getTime()-C;if(_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully notified Transcend for ${d} identifiers in "${D/1e3}" seconds!`)),u&&_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`There were ${u} identifiers that were not in a state to be updated.They likely have already been resolved.`)),l)throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`There were ${l} identifiers that failed to be updated. Please review the logs for more information.`)),new Error("Failed to update all identifiers");return o.length}var _privacytypes = require('@transcend-io/privacy-types');async function Pe({requestIds:r,dataSiloId:s,auth:c,concurrency:n=100,status:a=_privacytypes.RequestDataSiloStatus.Resolved,transcendUrl:i=_chunkEYOQLGDVcjs.r}){let f=_chunkKUBXRT3Vcjs.wc.call(void 0, i,c),y=new Date().getTime(),o=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Notifying Transcend for data silo "${s}" marking "${r.length}" requests as completed.`));let C=0;o.start(r.length,0),await _bluebird.map.call(void 0, r,async u=>{let l=await _chunkKUBXRT3Vcjs.Gd.call(void 0, f,{requestId:u,dataSiloId:s});try{await _chunkKUBXRT3Vcjs.sg.call(void 0, f,_chunkKUBXRT3Vcjs.va,{requestDataSiloId:l.id,status:a})}catch(m){if(!m.message.includes("Client error: Request must be active:")&&!m.message.includes("Failed to find RequestDataSilo"))throw m}C+=1,o.update(C)},{concurrency:n}),o.stop();let d=new Date().getTime()-y;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully notified Transcend in "${d/1e3}" seconds!`)),r.length}async function _e({dataSiloId:r,auth:s,sombraAuth:c,actions:n,apiPageSize:a=100,savePageSize:i=1e3,onSave:f,transcendUrl:y=_chunkEYOQLGDVcjs.r,skipRequestCount:o=!1}){if(i%a!==0)throw new Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${i}, apiPageSize: ${a}`);let C=await _chunkKUBXRT3Vcjs.xc.call(void 0, y,s,c),w=_chunkKUBXRT3Vcjs.wc.call(void 0, y,s),d=0;o||(d=await _chunkKUBXRT3Vcjs.Hd.call(void 0, w,{dataSiloId:r})),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Pulling ${o?"all":d} outstanding request identifiers for data silo: "${r}" for requests of types "${n.join('", "')}"`));let u=new Date().getTime(),l=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),m=new Set,I=[],g=[];o||l.start(d,0),await _bluebird.mapSeries.call(void 0, n,async $=>{let S=0,h=!0;for(;h;){let b=await U(C,{dataSiloId:r,limit:a,offset:S,requestType:$}),k=b.map(A=>(m.add(A.requestId),{...A,action:$})),J=k.map(({attributes:A,...K})=>({...K,...A.reduce((M,E)=>Object.assign(M,{[E.key]:E.values.join(",")}),{})}));I.push(...k),g.push(...J),g.length>=i&&(await f(g),g=[]),h=b.length===a,S+=a,o?_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Pulled ${b.length} outstanding identifiers for ${m.size} requests`)):l.update(m.size)}}),g.length>0&&await f(g),o||l.stop();let D=new Date().getTime()-u;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled ${I.length} outstanding identifiers from ${m.size} requests in "${D/1e3}" seconds!`)),{identifiers:I}}exports.a = X; exports.b = U; exports.c = L; exports.d = B; exports.e = ge; exports.f = Pe; exports.g = _e;
-//# sourceMappingURL=chunk-I7PFP5VB.cjs.map
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunk56COHDFTcjs = require('./chunk-56COHDFT.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkA3GRLGNUcjs = require('./chunk-A3GRLGNU.cjs');var _iots = require('io-ts'); var e = _interopRequireWildcard(_iots); var P = _interopRequireWildcard(_iots);var _typeutils = require('@transcend-io/type-utils');var X=e.type({identifier:e.string,type:e.string,coreIdentifier:e.string,dataSiloId:e.string,requestId:e.string,nonce:e.string,requestCreatedAt:e.string,daysUntilOverdue:e.number,attributes:e.array(e.type({key:e.string,values:e.array(e.string)}))});async function U(r,{dataSiloId:s,limit:c=100,offset:n=0,requestType:a}){try{let i=await r.get(`v1/data-silo/${s}/pending-requests/${a}`,{searchParams:{offset:n,limit:c}}).json(),{items:f}=_typeutils.decodeCodec.call(void 0, e.type({items:e.array(X)}),i);return f}catch(i){throw new Error(`Received an error from server: ${_optionalChain([i, 'optionalAccess', _2 => _2.response, 'optionalAccess', _3 => _3.body])||_optionalChain([i, 'optionalAccess', _4 => _4.message])}`)}}var L=P.type({nonce:P.string,identifier:P.string});async function B(r,{nonce:s,identifier:c}){try{return await r.put("v1/data-silo",{headers:{"x-transcend-nonce":s},json:{profiles:[{profileId:c}]}}),!0}catch(n){if(_optionalChain([n, 'access', _5 => _5.response, 'optionalAccess', _6 => _6.statusCode])===409)return!1;throw new Error(`Received an error from server: ${_optionalChain([n, 'optionalAccess', _7 => _7.response, 'optionalAccess', _8 => _8.body])||_optionalChain([n, 'optionalAccess', _9 => _9.message])}`)}}var _bluebird = require('bluebird');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);async function ge({file:r,dataSiloId:s,auth:c,sombraAuth:n,concurrency:a=100,transcendUrl:i=_chunkA3GRLGNUcjs.r,sleepSeconds:f=10}){let y=await _chunk56COHDFTcjs.yc.call(void 0, i,c,n);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Reading "${r}" from disk`));let o=_chunk56COHDFTcjs.sc.call(void 0, r,L);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Notifying Transcend for data silo "${s}" marking "${o.length}" identifiers as completed.`));let C=new Date().getTime(),w=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d=0,u=0,l=0;w.start(o.length,0);let m=_chunkA3GRLGNUcjs.b.call(void 0, o,a),I=m.length;await _bluebird.mapSeries.call(void 0, m,async($,S)=>{_chunkZUNVPK23cjs.a.info(_colors2.default.blue(`Processing chunk ${S+1}/${I} (${_chunkA3GRLGNUcjs.b.length} items)`)),await _bluebird.map.call(void 0, $,async h=>{try{await B(y,h)?d+=1:u+=1}catch(b){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Error notifying Transcend for identifier "${h.identifier}" - ${_optionalChain([b, 'optionalAccess', _10 => _10.message])}`)),l+=1}w.update(d+u)}),f>0&&S<I-1&&(_chunkZUNVPK23cjs.a.info(_colors2.default.yellow(`Sleeping for ${f}s before next chunk...`)),await new Promise(h=>{setTimeout(h,f*1e3)}))}),w.stop();let D=new Date().getTime()-C;if(_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully notified Transcend for ${d} identifiers in "${D/1e3}" seconds!`)),u&&_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`There were ${u} identifiers that were not in a state to be updated.They likely have already been resolved.`)),l)throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`There were ${l} identifiers that failed to be updated. Please review the logs for more information.`)),new Error("Failed to update all identifiers");return o.length}var _privacytypes = require('@transcend-io/privacy-types');async function Pe({requestIds:r,dataSiloId:s,auth:c,concurrency:n=100,status:a=_privacytypes.RequestDataSiloStatus.Resolved,transcendUrl:i=_chunkA3GRLGNUcjs.r}){let f=_chunk56COHDFTcjs.xc.call(void 0, i,c),y=new Date().getTime(),o=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Notifying Transcend for data silo "${s}" marking "${r.length}" requests as completed.`));let C=0;o.start(r.length,0),await _bluebird.map.call(void 0, r,async u=>{let l=await _chunk56COHDFTcjs.Hd.call(void 0, f,{requestId:u,dataSiloId:s});try{await _chunk56COHDFTcjs.ug.call(void 0, f,_chunk56COHDFTcjs.va,{requestDataSiloId:l.id,status:a})}catch(m){if(!m.message.includes("Client error: Request must be active:")&&!m.message.includes("Failed to find RequestDataSilo"))throw m}C+=1,o.update(C)},{concurrency:n}),o.stop();let d=new Date().getTime()-y;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully notified Transcend in "${d/1e3}" seconds!`)),r.length}async function _e({dataSiloId:r,auth:s,sombraAuth:c,actions:n,apiPageSize:a=100,savePageSize:i=1e3,onSave:f,transcendUrl:y=_chunkA3GRLGNUcjs.r,skipRequestCount:o=!1}){if(i%a!==0)throw new Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${i}, apiPageSize: ${a}`);let C=await _chunk56COHDFTcjs.yc.call(void 0, y,s,c),w=_chunk56COHDFTcjs.xc.call(void 0, y,s),d=0;o||(d=await _chunk56COHDFTcjs.Id.call(void 0, w,{dataSiloId:r})),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Pulling ${o?"all":d} outstanding request identifiers for data silo: "${r}" for requests of types "${n.join('", "')}"`));let u=new Date().getTime(),l=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),m=new Set,I=[],g=[];o||l.start(d,0),await _bluebird.mapSeries.call(void 0, n,async $=>{let S=0,h=!0;for(;h;){let b=await U(C,{dataSiloId:r,limit:a,offset:S,requestType:$}),k=b.map(A=>(m.add(A.requestId),{...A,action:$})),J=k.map(({attributes:A,...K})=>({...K,...A.reduce((M,E)=>Object.assign(M,{[E.key]:E.values.join(",")}),{})}));I.push(...k),g.push(...J),g.length>=i&&(await f(g),g=[]),h=b.length===a,S+=a,o?_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Pulled ${b.length} outstanding identifiers for ${m.size} requests`)):l.update(m.size)}}),g.length>0&&await f(g),o||l.stop();let D=new Date().getTime()-u;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled ${I.length} outstanding identifiers from ${m.size} requests in "${D/1e3}" seconds!`)),{identifiers:I}}exports.a = X; exports.b = U; exports.c = L; exports.d = B; exports.e = ge; exports.f = Pe; exports.g = _e;
+//# sourceMappingURL=chunk-4Y7PZGF3.cjs.map

package/dist/{chunk-I7PFP5VB.cjs.map → chunk-4Y7PZGF3.cjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-~~I7PFP5VB~~.cjs","../src/lib/cron/pullCronPageOfIdentifiers.ts"],"names":["CronIdentifier","pullCronPageOfIdentifiers","sombra","dataSiloId","limit","offset","requestType","response"],"mappings":"AAAA,u/BAAyF,wDAAyC,wDAAgD,6GCA/J,qDACS,IAIfA,CAAAA,CAAmB,CAAA,CAAA,IAAA,CAAK,CAEnC,UAAA,CAAc,CAAA,CAAA,MAAA,CAEd,IAAA,CAAQ,CAAA,CAAA,MAAA,CAER,cAAA,CAAkB,CAAA,CAAA,MAAA,CAElB,UAAA,CAAc,CAAA,CAAA,MAAA,CAEd,SAAA,CAAa,CAAA,CAAA,MAAA,CAEb,KAAA,CAAS,CAAA,CAAA,MAAA,CAET,gBAAA,CAAoB,CAAA,CAAA,MAAA,CAEpB,gBAAA,CAAoB,CAAA,CAAA,MAAA,CAEpB,UAAA,CAAc,CAAA,CAAA,KAAA,CACV,CAAA,CAAA,IAAA,CAAK,CACL,GAAA,CAAO,CAAA,CAAA,MAAA,CACP,MAAA,CAAU,CAAA,CAAA,KAAA,CAAQ,CAAA,CAAA,MAAM,CAC1B,CAAC,CACH,CACF,CAAC,CAAA,CAaD,MAAA,SAAsBC,CAAAA,CACpBC,CAAAA,CACA,CACE,UAAA,CAAAC,CAAAA,CACA,KAAA,CAAAC,CAAAA,CAAQ,GAAA,CACR,MAAA,CAAAC,CAAAA,CAAS,CAAA,CACT,WAAA,CAAAC,CACF,CAAA,CAU2B,CAC3B,GAAI,CAEF,IAAMC,CAAAA,CAAW,MAAML,CAAAA,CACpB,GAAA,CAAI,CAAA,aAAA,EAAgBC,CAAU,CAAA,kBAAA,EAAqBG,CAAW,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/chunk-~~I7PFP5VB~~.cjs","sourcesContent":[null,"import * as t from 'io-ts';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport { RequestAction } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\n\nexport const CronIdentifier = t.type({\n /** The identifier value /\n identifier: t.string,\n /* The type of identifier /\n type: t.string,\n /* The core identifier of the request /\n coreIdentifier: t.string,\n /* The ID of the underlying data silo /\n dataSiloId: t.string,\n /* The ID of the underlying request /\n requestId: t.string,\n /* The request nonce /\n nonce: t.string,\n /* The time the request was created /\n requestCreatedAt: t.string,\n /* The number of days until the request is overdue /\n daysUntilOverdue: t.number,\n /* Request attributes /\n attributes: t.array(\n t.type({\n key: t.string,\n values: t.array(t.string),\n }),\n ),\n});\n\n/* Type override /\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/\n Pull a offset of identifiers for a cron job\n \n @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n /\nexport async function pullCronPageOfIdentifiers(\n sombra: Got,\n {\n dataSiloId,\n limit = 100,\n offset = 0,\n requestType,\n }: {\n /* Data Silo ID /\n dataSiloId: string;\n /* Type of request /\n requestType: RequestAction;\n /* Number of identifiers to pull in /\n limit?: number;\n /* Page to pull in */\n offset?: number;\n },\n): Promise<CronIdentifier[]> {\n try {\n // Make the GraphQL request\n const response = await sombra\n .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n searchParams: {\n offset,\n limit,\n },\n })\n .json();\n\n const { items } = decodeCodec(\n t.type({\n items: t.array(CronIdentifier),\n }),\n response,\n );\n return items;\n } catch (err) {\n throw new Error(\n `Received an error from server: ${err?.response?.body \|\| err?.message}`,\n );\n }\n}\n"]}
1	+ {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-4Y7PZGF3.cjs","../src/lib/cron/pullCronPageOfIdentifiers.ts"],"names":["CronIdentifier","pullCronPageOfIdentifiers","sombra","dataSiloId","limit","offset","requestType","response"],"mappings":"AAAA,u/BAAyF,wDAAyC,wDAAgD,6GCA/J,qDACS,IAIfA,CAAAA,CAAmB,CAAA,CAAA,IAAA,CAAK,CAEnC,UAAA,CAAc,CAAA,CAAA,MAAA,CAEd,IAAA,CAAQ,CAAA,CAAA,MAAA,CAER,cAAA,CAAkB,CAAA,CAAA,MAAA,CAElB,UAAA,CAAc,CAAA,CAAA,MAAA,CAEd,SAAA,CAAa,CAAA,CAAA,MAAA,CAEb,KAAA,CAAS,CAAA,CAAA,MAAA,CAET,gBAAA,CAAoB,CAAA,CAAA,MAAA,CAEpB,gBAAA,CAAoB,CAAA,CAAA,MAAA,CAEpB,UAAA,CAAc,CAAA,CAAA,KAAA,CACV,CAAA,CAAA,IAAA,CAAK,CACL,GAAA,CAAO,CAAA,CAAA,MAAA,CACP,MAAA,CAAU,CAAA,CAAA,KAAA,CAAQ,CAAA,CAAA,MAAM,CAC1B,CAAC,CACH,CACF,CAAC,CAAA,CAaD,MAAA,SAAsBC,CAAAA,CACpBC,CAAAA,CACA,CACE,UAAA,CAAAC,CAAAA,CACA,KAAA,CAAAC,CAAAA,CAAQ,GAAA,CACR,MAAA,CAAAC,CAAAA,CAAS,CAAA,CACT,WAAA,CAAAC,CACF,CAAA,CAU2B,CAC3B,GAAI,CAEF,IAAMC,CAAAA,CAAW,MAAML,CAAAA,CACpB,GAAA,CAAI,CAAA,aAAA,EAAgBC,CAAU,CAAA,kBAAA,EAAqBG,CAAW,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/chunk-4Y7PZGF3.cjs","sourcesContent":[null,"import * as t from 'io-ts';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport { RequestAction } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\n\nexport const CronIdentifier = t.type({\n /** The identifier value /\n identifier: t.string,\n /* The type of identifier /\n type: t.string,\n /* The core identifier of the request /\n coreIdentifier: t.string,\n /* The ID of the underlying data silo /\n dataSiloId: t.string,\n /* The ID of the underlying request /\n requestId: t.string,\n /* The request nonce /\n nonce: t.string,\n /* The time the request was created /\n requestCreatedAt: t.string,\n /* The number of days until the request is overdue /\n daysUntilOverdue: t.number,\n /* Request attributes /\n attributes: t.array(\n t.type({\n key: t.string,\n values: t.array(t.string),\n }),\n ),\n});\n\n/* Type override /\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/\n Pull a offset of identifiers for a cron job\n \n @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n /\nexport async function pullCronPageOfIdentifiers(\n sombra: Got,\n {\n dataSiloId,\n limit = 100,\n offset = 0,\n requestType,\n }: {\n /* Data Silo ID /\n dataSiloId: string;\n /* Type of request /\n requestType: RequestAction;\n /* Number of identifiers to pull in /\n limit?: number;\n /* Page to pull in */\n offset?: number;\n },\n): Promise<CronIdentifier[]> {\n try {\n // Make the GraphQL request\n const response = await sombra\n .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n searchParams: {\n offset,\n limit,\n },\n })\n .json();\n\n const { items } = decodeCodec(\n t.type({\n items: t.array(CronIdentifier),\n }),\n response,\n );\n return items;\n } catch (err) {\n throw new Error(\n `Received an error from server: ${err?.response?.body \|\| err?.message}`,\n );\n }\n}\n"]}