npm - @transcend-io/cli - Versions diffs - 8.25.1 → 8.25.3 - Mend

@transcend-io/cli 8.25.1 → 8.25.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (118) hide show

package/dist/{chunk-M2ARSCLF.cjs.map → chunk-G6QNUBBK.cjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-~~M2ARSCLF~~.cjs","../src/lib/consent-manager/createConsentToken.ts","../src/lib/consent-manager/uploadConsents.ts"],"names":["createConsentToken","userId","base64EncryptionKey","base64SigningKey","signingKey","encryptionKey","encryptionAlgorithm","iv","cipher","jwtPayload","USP_STRING_REGEX","PurposeMap","uploadConsents","preferences","partition","concurrency","transcendUrl","DEFAULT_TRANSCEND_CONSENT_API","transcendConsentApi","createTranscendConsentGotInstance","invalidUspStrings","pref"],"mappings":"AAAA,u/BAAyC,wDAAoC,wDAAyC,0ECA9F,4FACH,SAWLA,CAAAA,CACdC,CAAAA,CACAC,CAAAA,CACAC,CAAAA,CACQ,CAER,IAAMC,CAAAA,CAAa,MAAA,CAAO,IAAA,CAAKD,CAAAA,CAAkB,QAAQ,CAAA,CACnDE,CAAAA,CAAgB,MAAA,CAAO,IAAA,CAAKH,CAAAA,CAAqB,QAAQ,CAAA,CAGzDI,CAAAA,CAAsB,oBAAA,CAEtBC,CAAAA,CAAK,MAAA,CAAO,IAAA,CAAK,UAAA,CAAY,KAAK,CAAA,CAElCC,CAAAA,CAAgB,CAAA,CAAA,cAAA,CAAeF,CAAAA,CAAqBD,CAAAA,CAAeE,CAAE,CAAA,CAYrEE,CAAAA,CAAa,CACjB,mBAAA,CAV0B,MAAA,CAAO,MAAA,CAAO,CACxCD,CAAAA,CAAO,MAAA,CAAOP,CAAM,CAAA,CACpBO,CAAAA,CAAO,KAAA,CAAM,CACf,CAAC,CAAA,CAAE,QAAA,CAAS,QAAQ,CAQpB,CAAA,CAOA,OAJyB,CAAA,CAAA,IAAA,CAAKC,CAAAA,CAAYL,CAAAA,CAAY,CACpD,SAAA,CAAW,OACb,CAAC,CAGH,CC/CA,gFAAmB,qEACA,oCAEC,qGAGI,qDACI,IAIfM,CAAAA,CAAmB,0BAAA,CAEnBC,CAAAA,aAAe,CAAA,CAAA,MAAA,CACxB,CAAA,CAAA,MAAA,CACA,CAAA,CAAA,KAAA,CAAM,CAAG,CAAA,CAAA,OAAA,CAAW,CAAA,CAAA,OAAA,CAAQ,MAAM,CAAC,CAAC,CACxC,CAAA,CAOA,MAAA,SAAsBC,CAAAA,CAAe,CACnC,mBAAA,CAAAV,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,WAAA,CAAAU,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,GAAA,CACd,YAAA,CAAAC,CAAAA,CAAeC,mBACjB,CAAA,CAakB,CAEhB,IAAMC,CAAAA,CAAsBC,kCAAAA,CAA8C,CAAA,CAGpEC,CAAAA,CAAoBP,CAAAA,CAAY,MAAA,CACnCQ,CAAAA,EAASA,CAAAA,CAAK,GAAA,EAAO,CAACX,CAAAA,CAAiB,IAAA,CAAKW,CAAAA,CAAK,GAAG,CACvD,CAAA,CACA,EAAA,CAAID,CAAAA,CAAkB,MAAA,CAAS,CAAA,CAC7B,MAAM,IAAI,KAAA,CACR,CAAA,8BAAA,EAAiC,IAAA,CAAK,SAAA,CACpCA,CAAAA,CACA,IAAA,CACA,CACF,CAAC,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/chunk-~~M2ARSCLF~~.cjs","sourcesContent":[null,"import * as crypto from 'crypto';\nimport * as jwt from 'jsonwebtoken';\n\n/*\n Function to create a consent manager token\n \n @see https://docs.transcend.io/docs/consent/reference/managed-consent-database\n * @param userId - User ID\n * @param base64EncryptionKey - Encryption key\n * @param base64SigningKey - Signing key\n * @returns Token\n /\nexport function createConsentToken(\n userId: string,\n base64EncryptionKey: string,\n base64SigningKey: string,\n): string {\n // Read on for where to find these keys\n const signingKey = Buffer.from(base64SigningKey, 'base64');\n const encryptionKey = Buffer.from(base64EncryptionKey, 'base64');\n\n // NIST's AES-KWP implementation { aes 48 } - see https://tools.ietf.org/html/rfc5649\n const encryptionAlgorithm = 'id-aes256-wrap-pad';\n // Initial Value for AES-KWP integrity check - see https://tools.ietf.org/html/rfc5649#section-3\n const iv = Buffer.from('A65959A6', 'hex');\n // Set up encryption algorithm\n const cipher = crypto.createCipheriv(encryptionAlgorithm, encryptionKey, iv);\n\n // Encrypt the userId and base64-encode the result\n const encryptedIdentifier = Buffer.concat([\n cipher.update(userId),\n cipher.final(),\n ]).toString('base64');\n\n // Create the JWT content - jwt.sign will add a 'iat' (issued at) field to the payload\n // If you wanted to add something manually, consider\n // const issued: Date = new Date();\n // const isoDate = issued.toISOString();\n const jwtPayload = {\n encryptedIdentifier,\n };\n\n // Create a JSON web token and HMAC it with SHA-384\n const consentToken = jwt.sign(jwtPayload, signingKey, {\n algorithm: 'HS384',\n });\n\n return consentToken;\n}\n","import { createTranscendConsentGotInstance } from '../graphql';\nimport colors from 'colors';\nimport as t from 'io-ts';\nimport { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants';\nimport { map } from 'bluebird';\nimport { createConsentToken } from './createConsentToken';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { ConsentPreferenceUpload } from './types';\nimport { ConsentPreferencesBody } from '@transcend-io/airgap.js-types';\n\nexport const USP_STRING_REGEX = /^[0-9][Y\|N]([Y\|N])[Y\|N]$/;\n\nexport const PurposeMap = t.record(\n t.string,\n t.union([t.boolean, t.literal('Auto')]),\n);\n\n/*\n Upload a set of consent preferences\n \n @param options - Options\n /\nexport async function uploadConsents({\n base64EncryptionKey,\n base64SigningKey,\n preferences,\n partition,\n concurrency = 100,\n transcendUrl = DEFAULT_TRANSCEND_CONSENT_API,\n}: {\n /* base64 encryption key /\n base64EncryptionKey: string;\n /* base64 signing key /\n base64SigningKey: string;\n /* Partition key /\n partition: string;\n /* Sombra API key authentication /\n preferences: ConsentPreferenceUpload[];\n /* API URL for Transcend backend /\n transcendUrl?: string;\n /* Concurrency limit for approving */\n concurrency?: number;\n}): Promise<void> {\n // Create connection to API\n const transcendConsentApi = createTranscendConsentGotInstance(transcendUrl);\n\n // Ensure usp strings are valid\n const invalidUspStrings = preferences.filter(\n (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp),\n );\n if (invalidUspStrings.length > 0) {\n throw new Error(\n `Received invalid usp strings: ${JSON.stringify(\n invalidUspStrings,\n null,\n 2,\n )}`,\n );\n }\n\n // Ensure purpose maps are valid\n const invalidPurposeMaps = preferences\n .map((pref, ind) => [pref, ind] as [ConsentPreferenceUpload, number])\n .filter(([pref]) => {\n if (!pref.purposes) {\n return false;\n }\n try {\n decodeCodec(PurposeMap, pref.purposes);\n return false;\n } catch {\n return true;\n }\n });\n if (invalidPurposeMaps.length > 0) {\n throw new Error(\n `Received invalid purpose maps: ${JSON.stringify(\n invalidPurposeMaps,\n null,\n 2,\n )}`,\n );\n }\n\n // Ensure usp or preferences are provided\n const invalidInputs = preferences.filter(\n (pref) => !pref.usp && !pref.purposes,\n );\n if (invalidInputs.length > 0) {\n throw new Error(\n `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify(\n invalidInputs,\n null,\n 2,\n )}`,\n );\n }\n\n logger.info(\n colors.magenta(\n `Uploading ${preferences.length} user preferences to partition ${partition}`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Build a GraphQL client\n let total = 0;\n progressBar.start(preferences.length, 0);\n await map(\n preferences,\n async ({\n userId,\n confirmed = 'true',\n updated,\n prompted,\n purposes,\n ...consent\n }) => {\n const token = createConsentToken(\n userId,\n base64EncryptionKey,\n base64SigningKey,\n );\n\n // parse usp string\n const [, saleStatus] = consent.usp\n ? USP_STRING_REGEX.exec(consent.usp) \|\| []\n : [];\n\n const input = {\n token,\n partition,\n consent: {\n confirmed: confirmed === 'true',\n purposes: purposes\n ? decodeCodec(PurposeMap, purposes)\n : consent.usp\n ? { SaleOfInfo: saleStatus === 'Y' }\n : {},\n ...(updated ? { updated: updated === 'true' } : {}),\n ...(prompted ? { prompted: prompted === 'true' } : {}),\n ...consent,\n },\n } as ConsentPreferencesBody;\n\n // Make the request\n try {\n await transcendConsentApi\n .post('sync', {\n json: input,\n })\n .json();\n } catch (err) {\n try {\n const parsed = JSON.parse(err?.response?.body \|\| '{}');\n if (parsed.error) {\n logger.error(colors.red(`Error: ${parsed.error}`));\n }\n } catch (e) {\n // continue\n }\n throw new Error(\n `Received an error from server: ${\n err?.response?.body \|\| err?.message\n }`,\n );\n }\n\n total += 1;\n progressBar.update(total);\n },\n { concurrency },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully uploaded ${\n preferences.length\n } user preferences to partition ${partition} in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n}\n"]}
1	+ {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-G6QNUBBK.cjs","../src/lib/consent-manager/createConsentToken.ts","../src/lib/consent-manager/uploadConsents.ts"],"names":["createConsentToken","userId","base64EncryptionKey","base64SigningKey","signingKey","encryptionKey","encryptionAlgorithm","iv","cipher","jwtPayload","USP_STRING_REGEX","PurposeMap","uploadConsents","preferences","partition","concurrency","transcendUrl","DEFAULT_TRANSCEND_CONSENT_API","transcendConsentApi","createTranscendConsentGotInstance","invalidUspStrings","pref"],"mappings":"AAAA,u/BAAyC,wDAAoC,wDAAyC,0ECA9F,4FACH,SAWLA,CAAAA,CACdC,CAAAA,CACAC,CAAAA,CACAC,CAAAA,CACQ,CAER,IAAMC,CAAAA,CAAa,MAAA,CAAO,IAAA,CAAKD,CAAAA,CAAkB,QAAQ,CAAA,CACnDE,CAAAA,CAAgB,MAAA,CAAO,IAAA,CAAKH,CAAAA,CAAqB,QAAQ,CAAA,CAGzDI,CAAAA,CAAsB,oBAAA,CAEtBC,CAAAA,CAAK,MAAA,CAAO,IAAA,CAAK,UAAA,CAAY,KAAK,CAAA,CAElCC,CAAAA,CAAgB,CAAA,CAAA,cAAA,CAAeF,CAAAA,CAAqBD,CAAAA,CAAeE,CAAE,CAAA,CAYrEE,CAAAA,CAAa,CACjB,mBAAA,CAV0B,MAAA,CAAO,MAAA,CAAO,CACxCD,CAAAA,CAAO,MAAA,CAAOP,CAAM,CAAA,CACpBO,CAAAA,CAAO,KAAA,CAAM,CACf,CAAC,CAAA,CAAE,QAAA,CAAS,QAAQ,CAQpB,CAAA,CAOA,OAJyB,CAAA,CAAA,IAAA,CAAKC,CAAAA,CAAYL,CAAAA,CAAY,CACpD,SAAA,CAAW,OACb,CAAC,CAGH,CC/CA,gFAAmB,qEACA,oCAEC,qGAGI,qDACI,IAIfM,CAAAA,CAAmB,0BAAA,CAEnBC,CAAAA,aAAe,CAAA,CAAA,MAAA,CACxB,CAAA,CAAA,MAAA,CACA,CAAA,CAAA,KAAA,CAAM,CAAG,CAAA,CAAA,OAAA,CAAW,CAAA,CAAA,OAAA,CAAQ,MAAM,CAAC,CAAC,CACxC,CAAA,CAOA,MAAA,SAAsBC,CAAAA,CAAe,CACnC,mBAAA,CAAAV,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,WAAA,CAAAU,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,GAAA,CACd,YAAA,CAAAC,CAAAA,CAAeC,mBACjB,CAAA,CAakB,CAEhB,IAAMC,CAAAA,CAAsBC,kCAAAA,CAA8C,CAAA,CAGpEC,CAAAA,CAAoBP,CAAAA,CAAY,MAAA,CACnCQ,CAAAA,EAASA,CAAAA,CAAK,GAAA,EAAO,CAACX,CAAAA,CAAiB,IAAA,CAAKW,CAAAA,CAAK,GAAG,CACvD,CAAA,CACA,EAAA,CAAID,CAAAA,CAAkB,MAAA,CAAS,CAAA,CAC7B,MAAM,IAAI,KAAA,CACR,CAAA,8BAAA,EAAiC,IAAA,CAAK,SAAA,CACpCA,CAAAA,CACA,IAAA,CACA,CACF,CAAC,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/chunk-G6QNUBBK.cjs","sourcesContent":[null,"import * as crypto from 'crypto';\nimport * as jwt from 'jsonwebtoken';\n\n/*\n Function to create a consent manager token\n \n @see https://docs.transcend.io/docs/consent/reference/managed-consent-database\n * @param userId - User ID\n * @param base64EncryptionKey - Encryption key\n * @param base64SigningKey - Signing key\n * @returns Token\n /\nexport function createConsentToken(\n userId: string,\n base64EncryptionKey: string,\n base64SigningKey: string,\n): string {\n // Read on for where to find these keys\n const signingKey = Buffer.from(base64SigningKey, 'base64');\n const encryptionKey = Buffer.from(base64EncryptionKey, 'base64');\n\n // NIST's AES-KWP implementation { aes 48 } - see https://tools.ietf.org/html/rfc5649\n const encryptionAlgorithm = 'id-aes256-wrap-pad';\n // Initial Value for AES-KWP integrity check - see https://tools.ietf.org/html/rfc5649#section-3\n const iv = Buffer.from('A65959A6', 'hex');\n // Set up encryption algorithm\n const cipher = crypto.createCipheriv(encryptionAlgorithm, encryptionKey, iv);\n\n // Encrypt the userId and base64-encode the result\n const encryptedIdentifier = Buffer.concat([\n cipher.update(userId),\n cipher.final(),\n ]).toString('base64');\n\n // Create the JWT content - jwt.sign will add a 'iat' (issued at) field to the payload\n // If you wanted to add something manually, consider\n // const issued: Date = new Date();\n // const isoDate = issued.toISOString();\n const jwtPayload = {\n encryptedIdentifier,\n };\n\n // Create a JSON web token and HMAC it with SHA-384\n const consentToken = jwt.sign(jwtPayload, signingKey, {\n algorithm: 'HS384',\n });\n\n return consentToken;\n}\n","import { createTranscendConsentGotInstance } from '../graphql';\nimport colors from 'colors';\nimport as t from 'io-ts';\nimport { DEFAULT_TRANSCEND_CONSENT_API } from '../../constants';\nimport { map } from 'bluebird';\nimport { createConsentToken } from './createConsentToken';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { ConsentPreferenceUpload } from './types';\nimport { ConsentPreferencesBody } from '@transcend-io/airgap.js-types';\n\nexport const USP_STRING_REGEX = /^[0-9][Y\|N]([Y\|N])[Y\|N]$/;\n\nexport const PurposeMap = t.record(\n t.string,\n t.union([t.boolean, t.literal('Auto')]),\n);\n\n/*\n Upload a set of consent preferences\n \n @param options - Options\n /\nexport async function uploadConsents({\n base64EncryptionKey,\n base64SigningKey,\n preferences,\n partition,\n concurrency = 100,\n transcendUrl = DEFAULT_TRANSCEND_CONSENT_API,\n}: {\n /* base64 encryption key /\n base64EncryptionKey: string;\n /* base64 signing key /\n base64SigningKey: string;\n /* Partition key /\n partition: string;\n /* Sombra API key authentication /\n preferences: ConsentPreferenceUpload[];\n /* API URL for Transcend backend /\n transcendUrl?: string;\n /* Concurrency limit for approving */\n concurrency?: number;\n}): Promise<void> {\n // Create connection to API\n const transcendConsentApi = createTranscendConsentGotInstance(transcendUrl);\n\n // Ensure usp strings are valid\n const invalidUspStrings = preferences.filter(\n (pref) => pref.usp && !USP_STRING_REGEX.test(pref.usp),\n );\n if (invalidUspStrings.length > 0) {\n throw new Error(\n `Received invalid usp strings: ${JSON.stringify(\n invalidUspStrings,\n null,\n 2,\n )}`,\n );\n }\n\n // Ensure purpose maps are valid\n const invalidPurposeMaps = preferences\n .map((pref, ind) => [pref, ind] as [ConsentPreferenceUpload, number])\n .filter(([pref]) => {\n if (!pref.purposes) {\n return false;\n }\n try {\n decodeCodec(PurposeMap, pref.purposes);\n return false;\n } catch {\n return true;\n }\n });\n if (invalidPurposeMaps.length > 0) {\n throw new Error(\n `Received invalid purpose maps: ${JSON.stringify(\n invalidPurposeMaps,\n null,\n 2,\n )}`,\n );\n }\n\n // Ensure usp or preferences are provided\n const invalidInputs = preferences.filter(\n (pref) => !pref.usp && !pref.purposes,\n );\n if (invalidInputs.length > 0) {\n throw new Error(\n `Received invalid inputs, expected either purposes or usp to be defined: ${JSON.stringify(\n invalidInputs,\n null,\n 2,\n )}`,\n );\n }\n\n logger.info(\n colors.magenta(\n `Uploading ${preferences.length} user preferences to partition ${partition}`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Build a GraphQL client\n let total = 0;\n progressBar.start(preferences.length, 0);\n await map(\n preferences,\n async ({\n userId,\n confirmed = 'true',\n updated,\n prompted,\n purposes,\n ...consent\n }) => {\n const token = createConsentToken(\n userId,\n base64EncryptionKey,\n base64SigningKey,\n );\n\n // parse usp string\n const [, saleStatus] = consent.usp\n ? USP_STRING_REGEX.exec(consent.usp) \|\| []\n : [];\n\n const input = {\n token,\n partition,\n consent: {\n confirmed: confirmed === 'true',\n purposes: purposes\n ? decodeCodec(PurposeMap, purposes)\n : consent.usp\n ? { SaleOfInfo: saleStatus === 'Y' }\n : {},\n ...(updated ? { updated: updated === 'true' } : {}),\n ...(prompted ? { prompted: prompted === 'true' } : {}),\n ...consent,\n },\n } as ConsentPreferencesBody;\n\n // Make the request\n try {\n await transcendConsentApi\n .post('sync', {\n json: input,\n })\n .json();\n } catch (err) {\n try {\n const parsed = JSON.parse(err?.response?.body \|\| '{}');\n if (parsed.error) {\n logger.error(colors.red(`Error: ${parsed.error}`));\n }\n } catch (e) {\n // continue\n }\n throw new Error(\n `Received an error from server: ${\n err?.response?.body \|\| err?.message\n }`,\n );\n }\n\n total += 1;\n progressBar.update(total);\n },\n { concurrency },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully uploaded ${\n preferences.length\n } user preferences to partition ${partition} in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n}\n"]}

package/dist/{chunk-OM2P5WDQ.cjs → chunk-GDPRENOQ.cjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkGC3HFDP4cjs = require('./chunk-GC3HFDP4.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunk6WIX75ZFcjs = require('./chunk-6WIX75ZF.cjs');var _privacytypes = require('@transcend-io/privacy-types');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _graphqlrequest = require('graphql-request');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function q(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=[],m=new Date().getTime(),d=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),s={...c.length>0?{category:c}:{},...t.length>0?{subCategoryIds:t}:{},...c.length+t.length>0&&!l?{status:_privacytypes.SubDataPointDataSubCategoryGuessStatus.Approved}:{},...e.length>0?{dataSilos:e}:{}},{subDataPoints:{totalCount:o}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_chunkGC3HFDP4cjs.d,{filterBy:s});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),d.start(o,0);let y=0,D=!1,r,b=0;do try{let{subDataPoints:{nodes:P}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_graphqlrequest.gql`
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkGLXV5XPPcjs = require('./chunk-GLXV5XPP.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkHJTYLFGJcjs = require('./chunk-HJTYLFGJ.cjs');var _privacytypes = require('@transcend-io/privacy-types');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _graphqlrequest = require('graphql-request');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function q(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=[],m=new Date().getTime(),d=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),s={...c.length>0?{category:c}:{},...t.length>0?{subCategoryIds:t}:{},...c.length+t.length>0&&!l?{status:_privacytypes.SubDataPointDataSubCategoryGuessStatus.Approved}:{},...e.length>0?{dataSilos:e}:{}},{subDataPoints:{totalCount:o}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_chunkGLXV5XPPcjs.d,{filterBy:s});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),d.start(o,0);let y=0,D=!1,r,b=0;do try{let{subDataPoints:{nodes:P}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_graphqlrequest.gql`
           query TranscendCliSubDataPointCsvExport(
             $filterBy: SubDataPointFiltersInput
             $first: Int!
@@ -41,7 +41,7 @@
               }
             }
           }
-        `,{first:p,offset:b,filterBy:{...s}});r=_optionalChain([P, 'access', _2 => _2[P.length-1], 'optionalAccess', _3 => _3.id]),n.push(...P),D=P.length===p,y+=P.length,b+=P.length,d.update(y)}catch(P){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${r} and offset ${b}`)),P}while(D);d.stop();let C=new Date().getTime()-m,g=_chunk6WIX75ZFcjs.g.call(void 0, n,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${g.length} subdatapoints in ${C/1e3} seconds!`)),g}async function F(u,{dataPointIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 2/3] Fetching metadata for ${e.length} datapoints`));let p=_chunk6WIX75ZFcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataPoints:{nodes:o}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_chunkGC3HFDP4cjs.g,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} dataPoints in ${d/1e3} seconds!`)),a}async function Q(u,{dataSiloIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 3/3] Fetching metadata for ${e.length} data silos`));let p=_chunk6WIX75ZFcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataSilos:{nodes:o}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_chunkGC3HFDP4cjs.j,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching data silos for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} data silos in ${d/1e3} seconds!`)),a}async function Y(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=await q(u,{dataSiloIds:e,includeGuessedCategories:l,includeAttributes:a,parentCategories:c,subCategories:t,pageSize:p}),m=_chunk6WIX75ZFcjs.j.call(void 0, n.map(r=>r.dataPointId)),d=await F(u,{dataPointIds:m}),s=_chunk6WIX75ZFcjs.e.call(void 0, d,"id"),o=_chunk6WIX75ZFcjs.j.call(void 0, n.map(r=>r.dataSiloId)),y=await Q(u,{dataSiloIds:o}),D=_chunk6WIX75ZFcjs.e.call(void 0, y,"id");return n.map(r=>({...r,dataPoint:s[r.dataPointId],dataSilo:D[r.dataSiloId]}))}async function nt(u,{dataSiloIds:e=[],status:l,subCategories:a=[],includeEncryptedSnippets:c,pageSize:t=100}={}){let p=[],n=new Date().getTime(),m=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d={...a.length>0?{subCategoryIds:a}:{},...l?{status:l}:{},...e.length>0?{dataSilos:e}:{}},{unstructuredSubDataPointRecommendations:{totalCount:s}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_chunkGC3HFDP4cjs.h,{filterBy:d});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),m.start(s,0);let o=0,y=!1,D,r=0;do try{let{unstructuredSubDataPointRecommendations:{nodes:g}}=await _chunkGC3HFDP4cjs.pg.call(void 0, u,_graphqlrequest.gql`
+        `,{first:p,offset:b,filterBy:{...s}});r=_optionalChain([P, 'access', _2 => _2[P.length-1], 'optionalAccess', _3 => _3.id]),n.push(...P),D=P.length===p,y+=P.length,b+=P.length,d.update(y)}catch(P){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${r} and offset ${b}`)),P}while(D);d.stop();let C=new Date().getTime()-m,g=_chunkHJTYLFGJcjs.g.call(void 0, n,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${g.length} subdatapoints in ${C/1e3} seconds!`)),g}async function F(u,{dataPointIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 2/3] Fetching metadata for ${e.length} datapoints`));let p=_chunkHJTYLFGJcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataPoints:{nodes:o}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_chunkGLXV5XPPcjs.g,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} dataPoints in ${d/1e3} seconds!`)),a}async function Q(u,{dataSiloIds:e=[],pageSize:l=100}){let a=[],c=new Date().getTime(),t=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`[Step 3/3] Fetching metadata for ${e.length} data silos`));let p=_chunkHJTYLFGJcjs.b.call(void 0, e,l);t.start(e.length,0);let n=0;await _bluebird.mapSeries.call(void 0, p,async s=>{try{let{dataSilos:{nodes:o}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_chunkGLXV5XPPcjs.j,{first:l,filterBy:{ids:s}});a.push(...o),n+=s.length,t.update(n)}catch(o){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching data silos for IDs ${s.join(", ")}`)),o}}),t.stop();let d=new Date().getTime()-c;return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${a.length} data silos in ${d/1e3} seconds!`)),a}async function Y(u,{dataSiloIds:e=[],includeGuessedCategories:l,includeAttributes:a,parentCategories:c=[],subCategories:t=[],pageSize:p=1e3}={}){let n=await q(u,{dataSiloIds:e,includeGuessedCategories:l,includeAttributes:a,parentCategories:c,subCategories:t,pageSize:p}),m=_chunkHJTYLFGJcjs.j.call(void 0, n.map(r=>r.dataPointId)),d=await F(u,{dataPointIds:m}),s=_chunkHJTYLFGJcjs.e.call(void 0, d,"id"),o=_chunkHJTYLFGJcjs.j.call(void 0, n.map(r=>r.dataSiloId)),y=await Q(u,{dataSiloIds:o}),D=_chunkHJTYLFGJcjs.e.call(void 0, y,"id");return n.map(r=>({...r,dataPoint:s[r.dataPointId],dataSilo:D[r.dataSiloId]}))}async function nt(u,{dataSiloIds:e=[],status:l,subCategories:a=[],includeEncryptedSnippets:c,pageSize:t=100}={}){let p=[],n=new Date().getTime(),m=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),d={...a.length>0?{subCategoryIds:a}:{},...l?{status:l}:{},...e.length>0?{dataSilos:e}:{}},{unstructuredSubDataPointRecommendations:{totalCount:s}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_chunkGLXV5XPPcjs.h,{filterBy:d});_chunkZUNVPK23cjs.a.info(_colors2.default.magenta("[Step 1/3] Pulling in all subdatapoints")),m.start(s,0);let o=0,y=!1,D,r=0;do try{let{unstructuredSubDataPointRecommendations:{nodes:g}}=await _chunkGLXV5XPPcjs.pg.call(void 0, u,_graphqlrequest.gql`
           query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(
             $filterBy: UnstructuredSubDataPointRecommendationsFilterInput
             $first: Int!
@@ -71,5 +71,5 @@
               }
             }
           }
-        `,{first:t,offset:r,filterBy:{...d}});D=_optionalChain([g, 'access', _4 => _4[g.length-1], 'optionalAccess', _5 => _5.id]),p.push(...g),y=g.length===t,o+=g.length,r+=g.length,m.update(o)}catch(g){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${D} and offset ${r}`)),g}while(y);m.stop();let T=new Date().getTime()-n,C=_chunk6WIX75ZFcjs.g.call(void 0, p,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${C.length} subdatapoints in ${T/1e3} seconds!`)),C}exports.a = Y; exports.b = nt;
-//# sourceMappingURL=chunk-OM2P5WDQ.cjs.map
+        `,{first:t,offset:r,filterBy:{...d}});D=_optionalChain([g, 'access', _4 => _4[g.length-1], 'optionalAccess', _5 => _5.id]),p.push(...g),y=g.length===t,o+=g.length,r+=g.length,m.update(o)}catch(g){throw _chunkZUNVPK23cjs.a.error(_colors2.default.red(`An error fetching subdatapoints for cursor ${D} and offset ${r}`)),g}while(y);m.stop();let T=new Date().getTime()-n,C=_chunkHJTYLFGJcjs.g.call(void 0, p,"name");return _chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pulled in ${C.length} subdatapoints in ${T/1e3} seconds!`)),C}exports.a = Y; exports.b = nt;
+//# sourceMappingURL=chunk-GDPRENOQ.cjs.map

package/dist/{chunk-OM2P5WDQ.cjs.map → chunk-GDPRENOQ.cjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-OM2P5WDQ.cjs","../src/lib/data-inventory/pullAllDatapoints.ts","../src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts"],"names":["pullSubDatapoints","client","dataSiloIds","includeGuessedCategories","includeAttributes","parentCategories","subCategories","pageSize","subDataPoints","t0","progressBar","cliProgress","filterBy","SubDataPointDataSubCategoryGuessStatus","totalCount","makeGraphQLRequest","SUB_DATA_POINTS_COUNT","logger","colors","total","shouldContinue","cursor","offset","nodes","gql","err"],"mappings":"AAAA,quBAAqE,wDAAyC,wDAA8D,2DCKrK,qGACiB,iDACJ,gFACD,oCAWO,MAuE1B,SAAeA,CAAAA,CACbC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CAAc,CAAC,CAAA,CACf,wBAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CAAmB,CAAC,CAAA,CACpB,aAAA,CAAAC,CAAAA,CAAgB,CAAC,CAAA,CACjB,QAAA,CAAAC,CAAAA,CAAW,GACb,CAAA,CAGI,CAAC,CAAA,CAC8B,CACnC,IAAMC,CAAAA,CAA0C,CAAC,CAAA,CAG3CC,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CAGxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CAGMC,CAAAA,CAAW,CACf,GAAIP,CAAAA,CAAiB,MAAA,CAAS,CAAA,CAAI,CAAE,QAAA,CAAUA,CAAiB,CAAA,CAAI,CAAC,CAAA,CACpE,GAAIC,CAAAA,CAAc,MAAA,CAAS,CAAA,CAAI,CAAE,cAAA,CAAgBA,CAAc,CAAA,CAAI,CAAC,CAAA,CAEpE,GAAID,CAAAA,CAAiB,MAAA,CAASC,CAAAA,CAAc,MAAA,CAAS,CAAA,EACrD,CAACH,CAAAA,CAEG,CAAE,MAAA,CAAQU,oDAAAA,CAAuC,QAAS,CAAA,CAC1D,CAAC,CAAA,CACL,GAAIX,CAAAA,CAAY,MAAA,CAAS,CAAA,CAAI,CAAE,SAAA,CAAWA,CAAY,CAAA,CAAI,CAAC,CAC7D,CAAA,CAGM,CACJ,aAAA,CAAe,CAAE,UAAA,CAAAY,CAAW,CAC9B,CAAA,CAAI,MAAMC,kCAAAA,CAMPd,CAAQe,mBAAAA,CAAuB,CAChC,QAAA,CAAAJ,CACF,CAAC,CAAA,CAEDK,mBAAAA,CAAO,IAAA,CAAKC,gBAAAA,CAAO,OAAA,CAAQ,yCAAyC,CAAC,CAAA,CAErER,CAAAA,CAAY,KAAA,CAAMI,CAAAA,CAAY,CAAC,CAAA,CAC/B,IAAIK,CAAAA,CAAQ,CAAA,CACRC,CAAAA,CAAiB,CAAA,CAAA,CACjBC,CAAAA,CACAC,CAAAA,CAAS,CAAA,CACb,GACE,GAAI,CACF,GAAM,CACJ,aAAA,CAAe,CAAE,KAAA,CAAAC,CAAM,CACzB,CAAA,CAAI,MAAMR,kCAAAA,CAORd,CACAuB,mBAAAA,CAAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAAA,EA2BUrB,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAQA,EACN,CAAA;AAAA,gBAAA,EAEEC,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAMA,EACN,CAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA,CAKR,CACE,KAAA,CAAOG,CAAAA,CACP,MAAA,CAAAe,CAAAA,CACA,QAAA,CAAU,CACR,GAAGV,CAGL,CACF,CACF,CAAA,CAEAS,CAAAA,iBAASE,CAAAA,qBAAMA,CAAAA,CAAM,MAAA,CAAS,CAAC,CAAA,6BAAG,IAAA,CAClCf,CAAAA,CAAc,IAAA,CAAK,GAAGe,CAAK,CAAA,CAC3BH,CAAAA,CAAiBG,CAAAA,CAAM,MAAA,GAAWhB,CAAAA,CAClCY,CAAAA,EAASI,CAAAA,CAAM,MAAA,CACfD,CAAAA,EAAUC,CAAAA,CAAM,MAAA,CAChBb,CAAAA,CAAY,MAAA,CAAOS,CAAK,CAC1B,CAAA,KAAA,CAASM,CAAAA,CAAK,CACZ,MAAAR,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,2CAAA,EAA8CG,CAAM,CAAA,YAAA,EAAeC,CAAM,CAAA,CAAA;AC7G3E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBgD,gBAAA;AACU,gBAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+BiB,QAAA","file":"/home/runner/work/cli/cli/dist/chunk-OM2P5WDQ.cjs","sourcesContent":[null,"/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\nimport {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport { gql } from 'graphql-request';\nimport colors from 'colors';\nimport type { GraphQLClient } from 'graphql-request';\nimport {\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n type DataSiloAttributeValue,\n SUB_DATA_POINTS_COUNT,\n makeGraphQLRequest,\n} from '../graphql';\nimport { logger } from '../../logger';\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs';\nimport { mapSeries } from 'bluebird';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 &&\n !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`,\n ),\n );\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(\n ', ',\n )}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`,\n ),\n );\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines /\n","import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\nimport { sortBy } from 'lodash-es';\nimport type { DataCategoryInput } from '../../codecs';\nimport { ENTRY_COUNT, makeGraphQLRequest } from '../graphql';\nimport { logger } from '../../logger';\n\ninterface UnstructuredSubDataPointRecommendationCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Entry or Named Entity recognized by the classifier /\n name: string;\n /* Context snippet including entry /\n contextSnippet: string;\n /* Scanned object ID /\n scannedObjectId: string;\n /* Scanned object path ID /\n scannedObjectPathId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* Personal data category /\n dataSubCategory: DataCategoryInput;\n /* Classification Status /\n status: UnstructuredSubDataPointRecommendationStatus;\n /* Confidence /\n confidence: number;\n /* Classification method /\n classificationMethod: string;\n /* Classifier version /\n classifierVersion: string;\n}\n\ninterface EntryFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Parent categories to filter down for /\n status?: UnstructuredSubDataPointRecommendationStatus[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n /* Include entry and snippet /\n includeEncryptedSnippets?: boolean;\n /* Include encryptedSamplesS3Key /\n includeEncryptedSamplesS3Key?: boolean;\n}\n/\n Pull unstructured subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @param options.dataSiloIds - IDs of data silos to filter down\n * @param options.status - Parent categories to filter down for\n * @param options.subCategories - Sub categories to filter down for\n * @param options.includeEncryptedSnippets - Include entry and snippet\n * @param options.includeEncryptedSamplesS3Key - Include encryptedSamplesS3Key\n * @param options.pageSize - Page size to pull in\n * @returns A promise that resolves to an array of unstructured subdatapoint recommendations\n /\nexport async function pullUnstructuredSubDataPointRecommendations(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n status,\n subCategories = [],\n includeEncryptedSnippets,\n pageSize = 100,\n }: EntryFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<UnstructuredSubDataPointRecommendationCsvPreview[]> {\n const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] =\n [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n ...(status ? { status } : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n unstructuredSubDataPointRecommendations: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* Count /\n totalCount: number;\n };\n }>(client, ENTRY_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n unstructuredSubDataPointRecommendations: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* List of matches */\n nodes: UnstructuredSubDataPointRecommendationCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(\n $filterBy: UnstructuredSubDataPointRecommendationsFilterInput\n $first: Int!\n $offset: Int!\n ) {\n unstructuredSubDataPointRecommendations(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n dataSiloId\n scannedObjectPathId\n scannedObjectId\n ${includeEncryptedSnippets ? 'name' : ''}\n ${includeEncryptedSnippets ? 'contextSnippet' : ''}\n dataSubCategory {\n name\n category\n }\n status\n confidence\n classificationMethod\n classifierVersion\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n unstructuredSubDataPointRecommendations.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n"]}
1	+ {"version":3,"sources":["/home/runner/work/cli/cli/dist/chunk-GDPRENOQ.cjs","../src/lib/data-inventory/pullAllDatapoints.ts","../src/lib/data-inventory/pullUnstructuredSubDataPointRecommendations.ts"],"names":["pullSubDatapoints","client","dataSiloIds","includeGuessedCategories","includeAttributes","parentCategories","subCategories","pageSize","subDataPoints","t0","progressBar","cliProgress","filterBy","SubDataPointDataSubCategoryGuessStatus","totalCount","makeGraphQLRequest","SUB_DATA_POINTS_COUNT","logger","colors","total","shouldContinue","cursor","offset","nodes","gql","err"],"mappings":"AAAA,quBAAqE,wDAAyC,wDAA8D,2DCKrK,qGACiB,iDACJ,gFACD,oCAWO,MAuE1B,SAAeA,CAAAA,CACbC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CAAc,CAAC,CAAA,CACf,wBAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CAAmB,CAAC,CAAA,CACpB,aAAA,CAAAC,CAAAA,CAAgB,CAAC,CAAA,CACjB,QAAA,CAAAC,CAAAA,CAAW,GACb,CAAA,CAGI,CAAC,CAAA,CAC8B,CACnC,IAAMC,CAAAA,CAA0C,CAAC,CAAA,CAG3CC,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CAGxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CAGMC,CAAAA,CAAW,CACf,GAAIP,CAAAA,CAAiB,MAAA,CAAS,CAAA,CAAI,CAAE,QAAA,CAAUA,CAAiB,CAAA,CAAI,CAAC,CAAA,CACpE,GAAIC,CAAAA,CAAc,MAAA,CAAS,CAAA,CAAI,CAAE,cAAA,CAAgBA,CAAc,CAAA,CAAI,CAAC,CAAA,CAEpE,GAAID,CAAAA,CAAiB,MAAA,CAASC,CAAAA,CAAc,MAAA,CAAS,CAAA,EACrD,CAACH,CAAAA,CAEG,CAAE,MAAA,CAAQU,oDAAAA,CAAuC,QAAS,CAAA,CAC1D,CAAC,CAAA,CACL,GAAIX,CAAAA,CAAY,MAAA,CAAS,CAAA,CAAI,CAAE,SAAA,CAAWA,CAAY,CAAA,CAAI,CAAC,CAC7D,CAAA,CAGM,CACJ,aAAA,CAAe,CAAE,UAAA,CAAAY,CAAW,CAC9B,CAAA,CAAI,MAAMC,kCAAAA,CAMPd,CAAQe,mBAAAA,CAAuB,CAChC,QAAA,CAAAJ,CACF,CAAC,CAAA,CAEDK,mBAAAA,CAAO,IAAA,CAAKC,gBAAAA,CAAO,OAAA,CAAQ,yCAAyC,CAAC,CAAA,CAErER,CAAAA,CAAY,KAAA,CAAMI,CAAAA,CAAY,CAAC,CAAA,CAC/B,IAAIK,CAAAA,CAAQ,CAAA,CACRC,CAAAA,CAAiB,CAAA,CAAA,CACjBC,CAAAA,CACAC,CAAAA,CAAS,CAAA,CACb,GACE,GAAI,CACF,GAAM,CACJ,aAAA,CAAe,CAAE,KAAA,CAAAC,CAAM,CACzB,CAAA,CAAI,MAAMR,kCAAAA,CAORd,CACAuB,mBAAAA,CAAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAAA,EA2BUrB,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAQA,EACN,CAAA;AAAA,gBAAA,EAEEC,CAAAA,CACI,CAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAA,CAAA,CAMA,EACN,CAAA;AAAA;AAAA;AAAA;AAAA,QAAA,CAAA,CAKR,CACE,KAAA,CAAOG,CAAAA,CACP,MAAA,CAAAe,CAAAA,CACA,QAAA,CAAU,CACR,GAAGV,CAGL,CACF,CACF,CAAA,CAEAS,CAAAA,iBAASE,CAAAA,qBAAMA,CAAAA,CAAM,MAAA,CAAS,CAAC,CAAA,6BAAG,IAAA,CAClCf,CAAAA,CAAc,IAAA,CAAK,GAAGe,CAAK,CAAA,CAC3BH,CAAAA,CAAiBG,CAAAA,CAAM,MAAA,GAAWhB,CAAAA,CAClCY,CAAAA,EAASI,CAAAA,CAAM,MAAA,CACfD,CAAAA,EAAUC,CAAAA,CAAM,MAAA,CAChBb,CAAAA,CAAY,MAAA,CAAOS,CAAK,CAC1B,CAAA,KAAA,CAASM,CAAAA,CAAK,CACZ,MAAAR,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,2CAAA,EAA8CG,CAAM,CAAA,YAAA,EAAeC,CAAM,CAAA,CAAA;AC7G3E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBgD,gBAAA;AACU,gBAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+BiB,QAAA","file":"/home/runner/work/cli/cli/dist/chunk-GDPRENOQ.cjs","sourcesContent":[null,"/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\nimport {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport { gql } from 'graphql-request';\nimport colors from 'colors';\nimport type { GraphQLClient } from 'graphql-request';\nimport {\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n type DataSiloAttributeValue,\n SUB_DATA_POINTS_COUNT,\n makeGraphQLRequest,\n} from '../graphql';\nimport { logger } from '../../logger';\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs';\nimport { mapSeries } from 'bluebird';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 &&\n !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`,\n ),\n );\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(\n ', ',\n )}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n logger.info(\n colors.magenta(\n `[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`,\n ),\n );\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`,\n ),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines /\n","import type { UnstructuredSubDataPointRecommendationStatus } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\nimport { sortBy } from 'lodash-es';\nimport type { DataCategoryInput } from '../../codecs';\nimport { ENTRY_COUNT, makeGraphQLRequest } from '../graphql';\nimport { logger } from '../../logger';\n\ninterface UnstructuredSubDataPointRecommendationCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Entry or Named Entity recognized by the classifier /\n name: string;\n /* Context snippet including entry /\n contextSnippet: string;\n /* Scanned object ID /\n scannedObjectId: string;\n /* Scanned object path ID /\n scannedObjectPathId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* Personal data category /\n dataSubCategory: DataCategoryInput;\n /* Classification Status /\n status: UnstructuredSubDataPointRecommendationStatus;\n /* Confidence /\n confidence: number;\n /* Classification method /\n classificationMethod: string;\n /* Classifier version /\n classifierVersion: string;\n}\n\ninterface EntryFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Parent categories to filter down for /\n status?: UnstructuredSubDataPointRecommendationStatus[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n /* Include entry and snippet /\n includeEncryptedSnippets?: boolean;\n /* Include encryptedSamplesS3Key /\n includeEncryptedSamplesS3Key?: boolean;\n}\n/\n Pull unstructured subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @param options.dataSiloIds - IDs of data silos to filter down\n * @param options.status - Parent categories to filter down for\n * @param options.subCategories - Sub categories to filter down for\n * @param options.includeEncryptedSnippets - Include entry and snippet\n * @param options.includeEncryptedSamplesS3Key - Include encryptedSamplesS3Key\n * @param options.pageSize - Page size to pull in\n * @returns A promise that resolves to an array of unstructured subdatapoint recommendations\n /\nexport async function pullUnstructuredSubDataPointRecommendations(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n status,\n subCategories = [],\n includeEncryptedSnippets,\n pageSize = 100,\n }: EntryFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<UnstructuredSubDataPointRecommendationCsvPreview[]> {\n const unstructuredSubDataPointRecommendations: UnstructuredSubDataPointRecommendationCsvPreview[] =\n [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Filters\n const filterBy = {\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n ...(status ? { status } : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n unstructuredSubDataPointRecommendations: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* Count /\n totalCount: number;\n };\n }>(client, ENTRY_COUNT, {\n filterBy,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n unstructuredSubDataPointRecommendations: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n unstructuredSubDataPointRecommendations: {\n /* List of matches */\n nodes: UnstructuredSubDataPointRecommendationCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliUnstructuredSubDataPointRecommendationCsvExport(\n $filterBy: UnstructuredSubDataPointRecommendationsFilterInput\n $first: Int!\n $offset: Int!\n ) {\n unstructuredSubDataPointRecommendations(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n dataSiloId\n scannedObjectPathId\n scannedObjectId\n ${includeEncryptedSnippets ? 'name' : ''}\n ${includeEncryptedSnippets ? 'contextSnippet' : ''}\n dataSubCategory {\n name\n category\n }\n status\n confidence\n classificationMethod\n classifierVersion\n }\n }\n }\n `,\n {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n },\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n unstructuredSubDataPointRecommendations.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(\n `An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`,\n ),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(unstructuredSubDataPointRecommendations, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${\n totalTime / 1000\n } seconds!`,\n ),\n );\n return sorted;\n}\n"]}