@transcend-io/cli 10.0.0 → 10.0.1

package/dist/{notifyPrivacyRequestsAdditionalTime-BvXIXZYu.mjs → notifyPrivacyRequestsAdditionalTime-TEHAJe4C.mjs}

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./syncTemplates-BNu1_dmW.mjs";import{r as i}from"./request-CAsR6CMY.mjs";import{r as a,t as o}from"./makeGraphQLRequest-G078PsEL.mjs";import{r as s}from"./fetchAllRequests-DEPTEUbi.mjs";import{RequestAction as c}from"@transcend-io/privacy-types";import l from"colors";import u from"cli-progress";async function d({requestActions:d=Object.values(c),auth:f,requestIds:p,createdAtBefore:m,days:h=45,daysLeft:g=10,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,emailTemplate:b=`Additional Time Needed`,concurrency:x=100,transcendUrl:S=e}){let C=a(S,f),w=new Date().getTime(),T=new u.SingleBar({},u.Presets.shades_classic),E=(await r(C,b)).find(e=>e.title===b);if(!E)throw Error(`Failed to find a template with title: "${b}"`);let D=await s(C,{actions:d,createdAtBefore:m,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,isSilent:!1,isClosed:!1,requestIds:p});D=D.filter(e=>typeof e.daysRemaining==`number`&&e.daysRemaining<g),t.info(l.magenta(`Notifying "${D.length}" that more time is needed.`));let O=0;T.start(D.length,0),await n(D,async e=>{await o(C,i,{input:{requestId:e.id,template:E.template.defaultMessage,subject:E.subject.defaultMessage,additionalTime:h}}),O+=1,T.update(O)},{concurrency:x}),T.stop();let k=new Date().getTime()-w;return t.info(l.green(`Successfully marked ${O} requests as silent mode in "${k/1e3}" seconds!`)),D.length}export{d as t};
-//# sourceMappingURL=notifyPrivacyRequestsAdditionalTime-BvXIXZYu.mjs.map
+import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./syncTemplates-BrH7Yr0V.mjs";import{r as i}from"./request-CAsR6CMY.mjs";import{r as a,t as o}from"./makeGraphQLRequest-Cq26A_Lq.mjs";import{r as s}from"./fetchAllRequests-DNQQsY4s.mjs";import{RequestAction as c}from"@transcend-io/privacy-types";import l from"colors";import u from"cli-progress";async function d({requestActions:d=Object.values(c),auth:f,requestIds:p,createdAtBefore:m,days:h=45,daysLeft:g=10,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,emailTemplate:b=`Additional Time Needed`,concurrency:x=100,transcendUrl:S=e}){let C=a(S,f),w=new Date().getTime(),T=new u.SingleBar({},u.Presets.shades_classic),E=(await r(C,b)).find(e=>e.title===b);if(!E)throw Error(`Failed to find a template with title: "${b}"`);let D=await s(C,{actions:d,createdAtBefore:m,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,isSilent:!1,isClosed:!1,requestIds:p});D=D.filter(e=>typeof e.daysRemaining==`number`&&e.daysRemaining<g),t.info(l.magenta(`Notifying "${D.length}" that more time is needed.`));let O=0;T.start(D.length,0),await n(D,async e=>{await o(C,i,{input:{requestId:e.id,template:E.template.defaultMessage,subject:E.subject.defaultMessage,additionalTime:h}}),O+=1,T.update(O)},{concurrency:x}),T.stop();let k=new Date().getTime()-w;return t.info(l.green(`Successfully marked ${O} requests as silent mode in "${k/1e3}" seconds!`)),D.length}export{d as t};
+//# sourceMappingURL=notifyPrivacyRequestsAdditionalTime-TEHAJe4C.mjs.map

package/dist/{notifyPrivacyRequestsAdditionalTime-BvXIXZYu.mjs.map → notifyPrivacyRequestsAdditionalTime-TEHAJe4C.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"notifyPrivacyRequestsAdditionalTime-BvXIXZYu.mjs","names":[],"sources":["../src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts"],"sourcesContent":["import { RequestAction } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { map } from '../bluebird.js';\nimport {\n  NOTIFY_ADDITIONAL_TIME,\n  fetchAllRequests,\n  makeGraphQLRequest,\n  buildTranscendGraphQLClient,\n  fetchAllTemplates,\n} from '../graphql/index.js';\n\n/**\n * Mark a set of privacy requests to be in silent mode.\n * Note requests in silent mode are ignored\n *\n * @param options - Options\n * @returns The number of requests marked silent\n */\nexport async function notifyPrivacyRequestsAdditionalTime({\n  requestActions = Object.values(RequestAction),\n  auth,\n  requestIds,\n  createdAtBefore,\n  days = 45,\n  daysLeft = 10,\n  createdAtAfter,\n  updatedAtBefore,\n  updatedAtAfter,\n  emailTemplate = 'Additional Time Needed',\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** The request actions that should be restarted */\n  requestActions?: RequestAction[];\n  /** Filter for requests created before this date */\n  createdAtBefore: Date;\n  /** Filter for requests created after this date */\n  createdAtAfter?: Date;\n  /** Filter for requests updated before this date */\n  updatedAtBefore?: Date;\n  /** Filter for requests updated after this date */\n  updatedAtAfter?: Date;\n  /** Email template */\n  emailTemplate?: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Number of days to extend request by */\n  days?: number;\n  /**\n   * Only notify requests that have less than this number of days until they are considered expired.\n   * This allows for re-running the command without notifying the same users multiple times\n   */\n  daysLeft?: number;\n  /** Concurrency limit for approving */\n  concurrency?: number;\n  /** The set of privacy requests to notify */\n  requestIds?: string[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<number> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Grab the template with that title\n  const matchingTemplates = await fetchAllTemplates(client, emailTemplate);\n  const exactTemplateMatch = matchingTemplates.find((template) => template.title === emailTemplate);\n  if (!exactTemplateMatch) {\n    throw new Error(`Failed to find a template with title: \"${emailTemplate}\"`);\n  }\n\n  // Pull in the requests\n  let allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    createdAtBefore,\n    createdAtAfter,\n    updatedAtBefore,\n    updatedAtAfter,\n    isSilent: false,\n    isClosed: false,\n    requestIds,\n  });\n\n  // Filter requests by daysLeft\n  allRequests = allRequests.filter(\n    (request) => typeof request.daysRemaining === 'number' && request.daysRemaining < daysLeft,\n  );\n\n  // Notify Transcend\n  logger.info(colors.magenta(`Notifying \"${allRequests.length}\" that more time is needed.`));\n\n  let total = 0;\n  progressBar.start(allRequests.length, 0);\n  await map(\n    allRequests,\n    async (requestToNotify) => {\n      await makeGraphQLRequest(client, NOTIFY_ADDITIONAL_TIME, {\n        input: {\n          requestId: requestToNotify.id,\n          template: exactTemplateMatch.template.defaultMessage,\n          subject: exactTemplateMatch.subject.defaultMessage,\n          additionalTime: days,\n        },\n      });\n\n      total += 1;\n      progressBar.update(total);\n    },\n    { concurrency },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully marked ${total} requests as silent mode in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n  return allRequests.length;\n}\n"],"mappings":"ybAsBA,eAAsB,EAAoC,CACxD,iBAAiB,OAAO,OAAO,EAAc,CAC7C,OACA,aACA,kBACA,OAAO,GACP,WAAW,GACX,iBACA,kBACA,iBACA,gBAAgB,yBAChB,cAAc,IACd,eAAe,GA6BG,CAElB,IAAM,EAAS,EAA4B,EAAc,EAAK,CAGxD,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAI/E,GADoB,MAAM,EAAkB,EAAQ,EAAc,EAC3B,KAAM,GAAa,EAAS,QAAU,EAAc,CACjG,GAAI,CAAC,EACH,MAAU,MAAM,0CAA0C,EAAc,GAAG,CAI7E,IAAI,EAAc,MAAM,EAAiB,EAAQ,CAC/C,QAAS,EACT,kBACA,iBACA,kBACA,iBACA,SAAU,GACV,SAAU,GACV,aACD,CAAC,CAGF,EAAc,EAAY,OACvB,GAAY,OAAO,EAAQ,eAAkB,UAAY,EAAQ,cAAgB,EACnF,CAGD,EAAO,KAAK,EAAO,QAAQ,cAAc,EAAY,OAAO,6BAA6B,CAAC,CAE1F,IAAI,EAAQ,EACZ,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,MAAM,EACJ,EACA,KAAO,IAAoB,CACzB,MAAM,EAAmB,EAAQ,EAAwB,CACvD,MAAO,CACL,UAAW,EAAgB,GAC3B,SAAU,EAAmB,SAAS,eACtC,QAAS,EAAmB,QAAQ,eACpC,eAAgB,EACjB,CACF,CAAC,CAEF,GAAS,EACT,EAAY,OAAO,EAAM,EAE3B,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAM,+BAA+B,EAAY,IAAK,YAC9E,CACF,CACM,EAAY"}
+{"version":3,"file":"notifyPrivacyRequestsAdditionalTime-TEHAJe4C.mjs","names":[],"sources":["../src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts"],"sourcesContent":["import { RequestAction } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { map } from '../bluebird.js';\nimport {\n  NOTIFY_ADDITIONAL_TIME,\n  fetchAllRequests,\n  makeGraphQLRequest,\n  buildTranscendGraphQLClient,\n  fetchAllTemplates,\n} from '../graphql/index.js';\n\n/**\n * Mark a set of privacy requests to be in silent mode.\n * Note requests in silent mode are ignored\n *\n * @param options - Options\n * @returns The number of requests marked silent\n */\nexport async function notifyPrivacyRequestsAdditionalTime({\n  requestActions = Object.values(RequestAction),\n  auth,\n  requestIds,\n  createdAtBefore,\n  days = 45,\n  daysLeft = 10,\n  createdAtAfter,\n  updatedAtBefore,\n  updatedAtAfter,\n  emailTemplate = 'Additional Time Needed',\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** The request actions that should be restarted */\n  requestActions?: RequestAction[];\n  /** Filter for requests created before this date */\n  createdAtBefore: Date;\n  /** Filter for requests created after this date */\n  createdAtAfter?: Date;\n  /** Filter for requests updated before this date */\n  updatedAtBefore?: Date;\n  /** Filter for requests updated after this date */\n  updatedAtAfter?: Date;\n  /** Email template */\n  emailTemplate?: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Number of days to extend request by */\n  days?: number;\n  /**\n   * Only notify requests that have less than this number of days until they are considered expired.\n   * This allows for re-running the command without notifying the same users multiple times\n   */\n  daysLeft?: number;\n  /** Concurrency limit for approving */\n  concurrency?: number;\n  /** The set of privacy requests to notify */\n  requestIds?: string[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<number> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Grab the template with that title\n  const matchingTemplates = await fetchAllTemplates(client, emailTemplate);\n  const exactTemplateMatch = matchingTemplates.find((template) => template.title === emailTemplate);\n  if (!exactTemplateMatch) {\n    throw new Error(`Failed to find a template with title: \"${emailTemplate}\"`);\n  }\n\n  // Pull in the requests\n  let allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    createdAtBefore,\n    createdAtAfter,\n    updatedAtBefore,\n    updatedAtAfter,\n    isSilent: false,\n    isClosed: false,\n    requestIds,\n  });\n\n  // Filter requests by daysLeft\n  allRequests = allRequests.filter(\n    (request) => typeof request.daysRemaining === 'number' && request.daysRemaining < daysLeft,\n  );\n\n  // Notify Transcend\n  logger.info(colors.magenta(`Notifying \"${allRequests.length}\" that more time is needed.`));\n\n  let total = 0;\n  progressBar.start(allRequests.length, 0);\n  await map(\n    allRequests,\n    async (requestToNotify) => {\n      await makeGraphQLRequest(client, NOTIFY_ADDITIONAL_TIME, {\n        input: {\n          requestId: requestToNotify.id,\n          template: exactTemplateMatch.template.defaultMessage,\n          subject: exactTemplateMatch.subject.defaultMessage,\n          additionalTime: days,\n        },\n      });\n\n      total += 1;\n      progressBar.update(total);\n    },\n    { concurrency },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully marked ${total} requests as silent mode in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n  return allRequests.length;\n}\n"],"mappings":"ybAsBA,eAAsB,EAAoC,CACxD,iBAAiB,OAAO,OAAO,EAAc,CAC7C,OACA,aACA,kBACA,OAAO,GACP,WAAW,GACX,iBACA,kBACA,iBACA,gBAAgB,yBAChB,cAAc,IACd,eAAe,GA6BG,CAElB,IAAM,EAAS,EAA4B,EAAc,EAAK,CAGxD,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAI/E,GADoB,MAAM,EAAkB,EAAQ,EAAc,EAC3B,KAAM,GAAa,EAAS,QAAU,EAAc,CACjG,GAAI,CAAC,EACH,MAAU,MAAM,0CAA0C,EAAc,GAAG,CAI7E,IAAI,EAAc,MAAM,EAAiB,EAAQ,CAC/C,QAAS,EACT,kBACA,iBACA,kBACA,iBACA,SAAU,GACV,SAAU,GACV,aACD,CAAC,CAGF,EAAc,EAAY,OACvB,GAAY,OAAO,EAAQ,eAAkB,UAAY,EAAQ,cAAgB,EACnF,CAGD,EAAO,KAAK,EAAO,QAAQ,cAAc,EAAY,OAAO,6BAA6B,CAAC,CAE1F,IAAI,EAAQ,EACZ,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,MAAM,EACJ,EACA,KAAO,IAAoB,CACzB,MAAM,EAAmB,EAAQ,EAAwB,CACvD,MAAO,CACL,UAAW,EAAgB,GAC3B,SAAU,EAAmB,SAAS,eACtC,QAAS,EAAmB,QAAQ,eACpC,eAAgB,EACjB,CACF,CAAC,CAEF,GAAS,EACT,EAAY,OAAO,EAAM,EAE3B,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAM,+BAA+B,EAAY,IAAK,YAC9E,CACF,CACM,EAAY"}

package/dist/package-C4J38oR1.mjs

@@ -0,0 +1,2 @@
+var e=`10.0.1`,t=`A command line interface for programmatic operations across Transcend.`;export{e as n,t};
+//# sourceMappingURL=package-C4J38oR1.mjs.map

package/dist/package-C4J38oR1.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"package-C4J38oR1.mjs","names":[],"sources":["../package.json"],"sourcesContent":[""],"mappings":""}

package/dist/{pullAllDatapoints-DiMWp8a7.mjs → pullAllDatapoints-Cntwuzw7.mjs}

@@ -1,4 +1,4 @@
-import{t as e}from"./logger-B-LXIf3U.mjs";import{n as t}from"./bluebird-CUitXgsY.mjs";import{i as n,o as r,u as i}from"./dataSilo-DrFetFXw.mjs";import{t as a}from"./makeGraphQLRequest-G078PsEL.mjs";import{SubDataPointDataSubCategoryGuessStatus as o}from"@transcend-io/privacy-types";import{chunk as s,keyBy as c,sortBy as l,uniq as u}from"lodash-es";import d from"colors";import{gql as f}from"graphql-request";import p from"cli-progress";async function m(t,{dataSiloIds:n=[],includeGuessedCategories:r,includeAttributes:s,parentCategories:c=[],subCategories:u=[],pageSize:m=1e3}={}){let h=[],g=new Date().getTime(),_=new p.SingleBar({},p.Presets.shades_classic),v={...c.length>0?{category:c}:{},...u.length>0?{subCategoryIds:u}:{},...c.length+u.length>0&&!r?{status:o.Approved}:{},...n.length>0?{dataSilos:n}:{}},{subDataPoints:{totalCount:y}}=await a(t,i,{filterBy:v});e.info(d.magenta(`[Step 1/3] Pulling in all subdatapoints`)),_.start(y,0);let b=0,x=!1,S,C=0;do try{let{subDataPoints:{nodes:e}}=await a(t,f`
+import{t as e}from"./logger-B-LXIf3U.mjs";import{n as t}from"./bluebird-CUitXgsY.mjs";import{i as n,o as r,u as i}from"./dataSilo-DrFetFXw.mjs";import{t as a}from"./makeGraphQLRequest-Cq26A_Lq.mjs";import{SubDataPointDataSubCategoryGuessStatus as o}from"@transcend-io/privacy-types";import{chunk as s,keyBy as c,sortBy as l,uniq as u}from"lodash-es";import d from"colors";import{gql as f}from"graphql-request";import p from"cli-progress";async function m(t,{dataSiloIds:n=[],includeGuessedCategories:r,includeAttributes:s,parentCategories:c=[],subCategories:u=[],pageSize:m=1e3}={}){let h=[],g=new Date().getTime(),_=new p.SingleBar({},p.Presets.shades_classic),v={...c.length>0?{category:c}:{},...u.length>0?{subCategoryIds:u}:{},...c.length+u.length>0&&!r?{status:o.Approved}:{},...n.length>0?{dataSilos:n}:{}},{subDataPoints:{totalCount:y}}=await a(t,i,{filterBy:v});e.info(d.magenta(`[Step 1/3] Pulling in all subdatapoints`)),_.start(y,0);let b=0,x=!1,S,C=0;do try{let{subDataPoints:{nodes:e}}=await a(t,f`
           query TranscendCliSubDataPointCsvExport(
             $filterBy: SubDataPointFiltersInput
             $first: Int!
@@ -42,4 +42,4 @@ import{t as e}from"./logger-B-LXIf3U.mjs";import{n as t}from"./bluebird-CUitXgsY
             }
           }
         `,{first:m,offset:C,filterBy:{...v}});S=e[e.length-1]?.id,h.push(...e),x=e.length===m,b+=e.length,C+=e.length,_.update(b)}catch(t){throw e.error(d.red(`An error fetching subdatapoints for cursor ${S} and offset ${C}`)),t}while(x);_.stop();let w=new Date().getTime()-g,T=l(h,`name`);return e.info(d.green(`Successfully pulled in ${T.length} subdatapoints in ${w/1e3} seconds!`)),T}async function h(n,{dataPointIds:i=[],pageSize:o=100}){let c=[],l=new Date().getTime(),u=new p.SingleBar({},p.Presets.shades_classic);e.info(d.magenta(`[Step 2/3] Fetching metadata for ${i.length} datapoints`));let f=s(i,o);u.start(i.length,0);let m=0;await t(f,async t=>{try{let{dataPoints:{nodes:e}}=await a(n,r,{first:o,filterBy:{ids:t}});c.push(...e),m+=t.length,u.update(m)}catch(n){throw e.error(d.red(`An error fetching subdatapoints for IDs ${t.join(`, `)}`)),n}}),u.stop();let h=new Date().getTime()-l;return e.info(d.green(`Successfully pulled in ${c.length} dataPoints in ${h/1e3} seconds!`)),c}async function g(r,{dataSiloIds:i=[],pageSize:o=100}){let c=[],l=new Date().getTime(),u=new p.SingleBar({},p.Presets.shades_classic);e.info(d.magenta(`[Step 3/3] Fetching metadata for ${i.length} data silos`));let f=s(i,o);u.start(i.length,0);let m=0;await t(f,async t=>{try{let{dataSilos:{nodes:e}}=await a(r,n,{first:o,filterBy:{ids:t}});c.push(...e),m+=t.length,u.update(m)}catch(n){throw e.error(d.red(`An error fetching data silos for IDs ${t.join(`, `)}`)),n}}),u.stop();let h=new Date().getTime()-l;return e.info(d.green(`Successfully pulled in ${c.length} data silos in ${h/1e3} seconds!`)),c}async function _(e,{dataSiloIds:t=[],includeGuessedCategories:n,includeAttributes:r,parentCategories:i=[],subCategories:a=[],pageSize:o=1e3}={}){let s=await m(e,{dataSiloIds:t,includeGuessedCategories:n,includeAttributes:r,parentCategories:i,subCategories:a,pageSize:o}),l=c(await h(e,{dataPointIds:u(s.map(e=>e.dataPointId))}),`id`),d=c(await g(e,{dataSiloIds:u(s.map(e=>e.dataSiloId))}),`id`);return s.map(e=>({...e,dataPoint:l[e.dataPointId],dataSilo:d[e.dataSiloId]}))}export{_ as t};
-//# sourceMappingURL=pullAllDatapoints-DiMWp8a7.mjs.map
+//# sourceMappingURL=pullAllDatapoints-Cntwuzw7.mjs.map

package/dist/{pullAllDatapoints-DiMWp8a7.mjs.map → pullAllDatapoints-Cntwuzw7.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullAllDatapoints-DiMWp8a7.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n  type DataCategoryType,\n  SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql } from 'graphql-request';\nimport type { GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines */\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { mapSeries } from '../bluebird.js';\nimport {\n  DATAPOINT_EXPORT,\n  DATA_SILO_EXPORT,\n  type DataSiloAttributeValue,\n  SUB_DATA_POINTS_COUNT,\n  makeGraphQLRequest,\n} from '../graphql/index.js';\n\nexport interface DataSiloCsvPreview {\n  /** ID of dataSilo */\n  id: string;\n  /** Name of dataSilo */\n  title: string;\n}\n\nexport interface DataPointCsvPreview {\n  /** ID of dataPoint */\n  id: string;\n  /** The path to this data point */\n  path: string[];\n  /** Description */\n  description: {\n    /** Default message */\n    defaultMessage: string;\n  };\n  /** Name */\n  name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n  /** ID of subDatapoint */\n  id: string;\n  /** Name (or key) of the subdatapoint */\n  name: string;\n  /** The description */\n  description?: string;\n  /** Personal data category */\n  categories: DataCategoryInput[];\n  /** Data point ID */\n  dataPointId: string;\n  /** The data silo ID */\n  dataSiloId: string;\n  /** The processing purpose for this sub datapoint */\n  purposes: ProcessingPurposeInput[];\n  /** Attribute attached to subdatapoint */\n  attributeValues?: DataSiloAttributeValue[];\n  /** Data category guesses that are output by the classifier */\n  pendingCategoryGuesses?: {\n    /** Data category being guessed */\n    category: DataCategoryInput;\n    /** Status of guess */\n    status: SubDataPointDataSubCategoryGuessStatus;\n    /** classifier version that produced the guess */\n    classifierVersion: number;\n  }[];\n}\n\nexport interface DatapointFilterOptions {\n  /** IDs of data silos to filter down */\n  dataSiloIds?: string[];\n  /** Whether to include guessed categories, defaults to only approved categories */\n  includeGuessedCategories?: boolean;\n  /** Whether or not to include attributes */\n  includeAttributes?: boolean;\n  /** Parent categories to filter down for */\n  parentCategories?: DataCategoryType[];\n  /** Sub categories to filter down for */\n  subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/**\n * Pull subdatapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n */\nasync function pullSubDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<SubDataPointCsvPreview[]> {\n  const subDataPoints: SubDataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Filters\n  const filterBy = {\n    ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n    ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n    // if parentCategories or subCategories and not includeGuessedCategories\n    ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n      ? // then only show data points with approved data categories\n        { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n      : {}),\n    ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n  };\n\n  // Build a GraphQL client\n  const {\n    subDataPoints: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Query response */\n    subDataPoints: {\n      /** Count */\n      totalCount: number;\n    };\n  }>(client, SUB_DATA_POINTS_COUNT, {\n    filterBy,\n  });\n\n  logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n  progressBar.start(totalCount, 0);\n  let total = 0;\n  let shouldContinue = false;\n  let cursor: string | undefined;\n  let offset = 0;\n  do {\n    try {\n      const {\n        subDataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        subDataPoints: {\n          /** List of matches */\n          nodes: SubDataPointCsvPreview[];\n        };\n      }>(\n        client,\n        gql`\n          query TranscendCliSubDataPointCsvExport(\n            $filterBy: SubDataPointFiltersInput\n            $first: Int!\n            $offset: Int!\n          ) {\n            subDataPoints(\n              filterBy: $filterBy\n              first: $first\n              offset: $offset\n              useMaster: false\n            ) {\n              nodes {\n                id\n                name\n                description\n                dataPointId\n                dataSiloId\n                purposes {\n                  name\n                  purpose\n                }\n                categories {\n                  name\n                  category\n                }\n                ${\n                  includeGuessedCategories\n                    ? `pendingCategoryGuesses {\n                  category {\n                    name\n                    category\n                  }\n                  status\n                  classifierVersion\n                }`\n                    : ''\n                }\n                ${\n                  includeAttributes\n                    ? `attributeValues {\n                  attributeKey {\n                    name\n                  }\n                  name\n                }`\n                    : ''\n                }\n              }\n            }\n          }\n        `,\n        {\n          first: pageSize,\n          offset,\n          filterBy: {\n            ...filterBy,\n            // TODO: https://transcend.height.app/T-40484 - add cursor support\n            // ...(cursor ? { cursor: { id: cursor } } : {}),\n          },\n        },\n      );\n\n      cursor = nodes[nodes.length - 1]?.id as string;\n      subDataPoints.push(...nodes);\n      shouldContinue = nodes.length === pageSize;\n      total += nodes.length;\n      offset += nodes.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n      );\n      throw err;\n    }\n  } while (shouldContinue);\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  const sorted = sortBy(subDataPoints, 'name');\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return sorted;\n}\n\n/**\n * Pull datapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n */\nasync function pullDatapoints(\n  client: GraphQLClient,\n  {\n    dataPointIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data points to filter down */\n    dataPointIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataPointCsvPreview[]> {\n  const dataPoints: DataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n  // Group by 100\n  const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n  progressBar.start(dataPointIds.length, 0);\n  let total = 0;\n  await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n    try {\n      const {\n        dataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataPoints: {\n          /** List of matches */\n          nodes: DataPointCsvPreview[];\n        };\n      }>(client, DATAPOINT_EXPORT, {\n        first: pageSize,\n        filterBy: {\n          ids: dataPointIdsGroup,\n        },\n      });\n\n      dataPoints.push(...nodes);\n      total += dataPointIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataPoints;\n}\n\n/**\n * Pull data silo information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n */\nasync function pullDataSilos(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data silos to filter down */\n    dataSiloIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataSiloCsvPreview[]> {\n  const dataSilos: DataSiloCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n  // Group by 100\n  const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n  progressBar.start(dataSiloIds.length, 0);\n  let total = 0;\n  await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n    try {\n      const {\n        dataSilos: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataSilos: {\n          /** List of matches */\n          nodes: DataSiloCsvPreview[];\n        };\n      }>(client, DATA_SILO_EXPORT, {\n        first: pageSize,\n        filterBy: {\n          ids: dataSiloIdsGroup,\n        },\n      });\n\n      dataSilos.push(...nodes);\n      total += dataSiloIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataSilos;\n}\n\n/**\n * Pull all datapoints from the data inventory.\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n */\nexport async function pullAllDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<\n  (SubDataPointCsvPreview & {\n    /** Data point information */\n    dataPoint: DataPointCsvPreview;\n    /** Data silo information */\n    dataSilo: DataSiloCsvPreview;\n  })[]\n> {\n  // Subdatapoint information\n  const subDatapoints = await pullSubDatapoints(client, {\n    dataSiloIds,\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories,\n    subCategories,\n    pageSize,\n  });\n\n  // The datapoint ids to grab\n  const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n  const dataPoints = await pullDatapoints(client, {\n    dataPointIds,\n  });\n  const dataPointById = keyBy(dataPoints, 'id');\n\n  // The data silo IDs to grab\n  const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n  const dataSilos = await pullDataSilos(client, {\n    dataSiloIds: allDataSiloIds,\n  });\n  const dataSiloById = keyBy(dataSilos, 'id');\n\n  return subDatapoints.map((subDataPoint) => ({\n    ...subDataPoint,\n    dataPoint: dataPointById[subDataPoint.dataPointId],\n    dataSilo: dataSiloById[subDataPoint.dataSiloId],\n  }));\n}\n/* eslint-enable max-lines */\n"],"mappings":"sbA2FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,WACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAHH,MAAM,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAAC,CAGzE,CAAC,CACsC,KAAK,CAOvC,EAAe,EAHH,MAAM,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAAC,CAG1E,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}
+{"version":3,"file":"pullAllDatapoints-Cntwuzw7.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n  type DataCategoryType,\n  SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql } from 'graphql-request';\nimport type { GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines */\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { mapSeries } from '../bluebird.js';\nimport {\n  DATAPOINT_EXPORT,\n  DATA_SILO_EXPORT,\n  type DataSiloAttributeValue,\n  SUB_DATA_POINTS_COUNT,\n  makeGraphQLRequest,\n} from '../graphql/index.js';\n\nexport interface DataSiloCsvPreview {\n  /** ID of dataSilo */\n  id: string;\n  /** Name of dataSilo */\n  title: string;\n}\n\nexport interface DataPointCsvPreview {\n  /** ID of dataPoint */\n  id: string;\n  /** The path to this data point */\n  path: string[];\n  /** Description */\n  description: {\n    /** Default message */\n    defaultMessage: string;\n  };\n  /** Name */\n  name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n  /** ID of subDatapoint */\n  id: string;\n  /** Name (or key) of the subdatapoint */\n  name: string;\n  /** The description */\n  description?: string;\n  /** Personal data category */\n  categories: DataCategoryInput[];\n  /** Data point ID */\n  dataPointId: string;\n  /** The data silo ID */\n  dataSiloId: string;\n  /** The processing purpose for this sub datapoint */\n  purposes: ProcessingPurposeInput[];\n  /** Attribute attached to subdatapoint */\n  attributeValues?: DataSiloAttributeValue[];\n  /** Data category guesses that are output by the classifier */\n  pendingCategoryGuesses?: {\n    /** Data category being guessed */\n    category: DataCategoryInput;\n    /** Status of guess */\n    status: SubDataPointDataSubCategoryGuessStatus;\n    /** classifier version that produced the guess */\n    classifierVersion: number;\n  }[];\n}\n\nexport interface DatapointFilterOptions {\n  /** IDs of data silos to filter down */\n  dataSiloIds?: string[];\n  /** Whether to include guessed categories, defaults to only approved categories */\n  includeGuessedCategories?: boolean;\n  /** Whether or not to include attributes */\n  includeAttributes?: boolean;\n  /** Parent categories to filter down for */\n  parentCategories?: DataCategoryType[];\n  /** Sub categories to filter down for */\n  subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/**\n * Pull subdatapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n */\nasync function pullSubDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<SubDataPointCsvPreview[]> {\n  const subDataPoints: SubDataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Filters\n  const filterBy = {\n    ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n    ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n    // if parentCategories or subCategories and not includeGuessedCategories\n    ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n      ? // then only show data points with approved data categories\n        { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n      : {}),\n    ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n  };\n\n  // Build a GraphQL client\n  const {\n    subDataPoints: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Query response */\n    subDataPoints: {\n      /** Count */\n      totalCount: number;\n    };\n  }>(client, SUB_DATA_POINTS_COUNT, {\n    filterBy,\n  });\n\n  logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n  progressBar.start(totalCount, 0);\n  let total = 0;\n  let shouldContinue = false;\n  let cursor: string | undefined;\n  let offset = 0;\n  do {\n    try {\n      const {\n        subDataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        subDataPoints: {\n          /** List of matches */\n          nodes: SubDataPointCsvPreview[];\n        };\n      }>(\n        client,\n        gql`\n          query TranscendCliSubDataPointCsvExport(\n            $filterBy: SubDataPointFiltersInput\n            $first: Int!\n            $offset: Int!\n          ) {\n            subDataPoints(\n              filterBy: $filterBy\n              first: $first\n              offset: $offset\n              useMaster: false\n            ) {\n              nodes {\n                id\n                name\n                description\n                dataPointId\n                dataSiloId\n                purposes {\n                  name\n                  purpose\n                }\n                categories {\n                  name\n                  category\n                }\n                ${\n                  includeGuessedCategories\n                    ? `pendingCategoryGuesses {\n                  category {\n                    name\n                    category\n                  }\n                  status\n                  classifierVersion\n                }`\n                    : ''\n                }\n                ${\n                  includeAttributes\n                    ? `attributeValues {\n                  attributeKey {\n                    name\n                  }\n                  name\n                }`\n                    : ''\n                }\n              }\n            }\n          }\n        `,\n        {\n          first: pageSize,\n          offset,\n          filterBy: {\n            ...filterBy,\n            // TODO: https://transcend.height.app/T-40484 - add cursor support\n            // ...(cursor ? { cursor: { id: cursor } } : {}),\n          },\n        },\n      );\n\n      cursor = nodes[nodes.length - 1]?.id as string;\n      subDataPoints.push(...nodes);\n      shouldContinue = nodes.length === pageSize;\n      total += nodes.length;\n      offset += nodes.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n      );\n      throw err;\n    }\n  } while (shouldContinue);\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  const sorted = sortBy(subDataPoints, 'name');\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return sorted;\n}\n\n/**\n * Pull datapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n */\nasync function pullDatapoints(\n  client: GraphQLClient,\n  {\n    dataPointIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data points to filter down */\n    dataPointIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataPointCsvPreview[]> {\n  const dataPoints: DataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n  // Group by 100\n  const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n  progressBar.start(dataPointIds.length, 0);\n  let total = 0;\n  await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n    try {\n      const {\n        dataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataPoints: {\n          /** List of matches */\n          nodes: DataPointCsvPreview[];\n        };\n      }>(client, DATAPOINT_EXPORT, {\n        first: pageSize,\n        filterBy: {\n          ids: dataPointIdsGroup,\n        },\n      });\n\n      dataPoints.push(...nodes);\n      total += dataPointIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataPoints;\n}\n\n/**\n * Pull data silo information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n */\nasync function pullDataSilos(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data silos to filter down */\n    dataSiloIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataSiloCsvPreview[]> {\n  const dataSilos: DataSiloCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n  // Group by 100\n  const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n  progressBar.start(dataSiloIds.length, 0);\n  let total = 0;\n  await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n    try {\n      const {\n        dataSilos: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataSilos: {\n          /** List of matches */\n          nodes: DataSiloCsvPreview[];\n        };\n      }>(client, DATA_SILO_EXPORT, {\n        first: pageSize,\n        filterBy: {\n          ids: dataSiloIdsGroup,\n        },\n      });\n\n      dataSilos.push(...nodes);\n      total += dataSiloIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataSilos;\n}\n\n/**\n * Pull all datapoints from the data inventory.\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n */\nexport async function pullAllDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<\n  (SubDataPointCsvPreview & {\n    /** Data point information */\n    dataPoint: DataPointCsvPreview;\n    /** Data silo information */\n    dataSilo: DataSiloCsvPreview;\n  })[]\n> {\n  // Subdatapoint information\n  const subDatapoints = await pullSubDatapoints(client, {\n    dataSiloIds,\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories,\n    subCategories,\n    pageSize,\n  });\n\n  // The datapoint ids to grab\n  const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n  const dataPoints = await pullDatapoints(client, {\n    dataPointIds,\n  });\n  const dataPointById = keyBy(dataPoints, 'id');\n\n  // The data silo IDs to grab\n  const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n  const dataSilos = await pullDataSilos(client, {\n    dataSiloIds: allDataSiloIds,\n  });\n  const dataSiloById = keyBy(dataSilos, 'id');\n\n  return subDatapoints.map((subDataPoint) => ({\n    ...subDataPoint,\n    dataPoint: dataPointById[subDataPoint.dataPointId],\n    dataSilo: dataSiloById[subDataPoint.dataSiloId],\n  }));\n}\n/* eslint-enable max-lines */\n"],"mappings":"sbA2FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,WACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAHH,MAAM,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAAC,CAGzE,CAAC,CACsC,KAAK,CAOvC,EAAe,EAHH,MAAM,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAAC,CAG1E,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-DgWgggQt.mjs → pullChunkedCustomSiloOutstandingIdentifiers-BT-GZpT1.mjs}

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./createSombraGotInstance-CahOgD6V.mjs";import{n as i}from"./RequestDataSilo-_Iv44M9u.mjs";import{r as a,t as o}from"./makeGraphQLRequest-G078PsEL.mjs";import{decodeCodec as s}from"@transcend-io/type-utils";import c from"colors";import*as l from"io-ts";import u from"cli-progress";async function d(e,{dataSiloId:t}){let{listReducedRequestsForDataSilo:{totalCount:n}}=await o(e,i,{input:{dataSiloId:t,isResolved:!1}});return n}const f=l.type({identifier:l.string,type:l.string,coreIdentifier:l.string,dataSiloId:l.string,requestId:l.string,nonce:l.string,requestCreatedAt:l.string,daysUntilOverdue:l.number,attributes:l.array(l.type({key:l.string,values:l.array(l.string)}))});async function p(e,{dataSiloId:t,limit:n=100,offset:r=0,requestType:i}){try{let a=await e.get(`v1/data-silo/${t}/pending-requests/${i}`,{searchParams:{offset:r,limit:n}}).json(),{items:o}=s(l.type({items:l.array(f)}),a);return o}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function m({dataSiloId:i,auth:o,sombraAuth:s,actions:l,apiPageSize:f=100,savePageSize:m=1e3,onSave:h,transcendUrl:g=e,skipRequestCount:_=!1}){if(m%f!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${m}, apiPageSize: ${f}`);let v=await r(g,o,s),y=a(g,o),b=0;_||(b=await d(y,{dataSiloId:i})),t.info(c.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${i}" for requests of types "${l.join(`", "`)}"`));let x=new Date().getTime(),S=new u.SingleBar({},u.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await n(l,async e=>{let n=0,r=!0;for(;r;){let a=await p(v,{dataSiloId:i,limit:f,offset:n,requestType:e}),o=a.map(t=>(C.add(t.requestId),{...t,action:e})),s=o.map(({attributes:e,...t})=>({...t,...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...o),T.push(...s),T.length>=m&&(await h(T),T=[]),r=a.length===f,n+=f,_?t.info(c.magenta(`Pulled ${a.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await h(T),_||S.stop();let E=new Date().getTime()-x;return t.info(c.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{d as i,f as n,p as r,m as t};
-//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-DgWgggQt.mjs.map
+import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./createSombraGotInstance-D1Il9zUE.mjs";import{n as i}from"./RequestDataSilo-_Iv44M9u.mjs";import{r as a,t as o}from"./makeGraphQLRequest-Cq26A_Lq.mjs";import{decodeCodec as s}from"@transcend-io/type-utils";import c from"colors";import*as l from"io-ts";import u from"cli-progress";async function d(e,{dataSiloId:t}){let{listReducedRequestsForDataSilo:{totalCount:n}}=await o(e,i,{input:{dataSiloId:t,isResolved:!1}});return n}const f=l.type({identifier:l.string,type:l.string,coreIdentifier:l.string,dataSiloId:l.string,requestId:l.string,nonce:l.string,requestCreatedAt:l.string,daysUntilOverdue:l.number,attributes:l.array(l.type({key:l.string,values:l.array(l.string)}))});async function p(e,{dataSiloId:t,limit:n=100,offset:r=0,requestType:i}){try{let a=await e.get(`v1/data-silo/${t}/pending-requests/${i}`,{searchParams:{offset:r,limit:n}}).json(),{items:o}=s(l.type({items:l.array(f)}),a);return o}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function m({dataSiloId:i,auth:o,sombraAuth:s,actions:l,apiPageSize:f=100,savePageSize:m=1e3,onSave:h,transcendUrl:g=e,skipRequestCount:_=!1}){if(m%f!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${m}, apiPageSize: ${f}`);let v=await r(g,o,s),y=a(g,o),b=0;_||(b=await d(y,{dataSiloId:i})),t.info(c.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${i}" for requests of types "${l.join(`", "`)}"`));let x=new Date().getTime(),S=new u.SingleBar({},u.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await n(l,async e=>{let n=0,r=!0;for(;r;){let a=await p(v,{dataSiloId:i,limit:f,offset:n,requestType:e}),o=a.map(t=>(C.add(t.requestId),{...t,action:e})),s=o.map(({attributes:e,...t})=>({...t,...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...o),T.push(...s),T.length>=m&&(await h(T),T=[]),r=a.length===f,n+=f,_?t.info(c.magenta(`Pulled ${a.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await h(T),_||S.stop();let E=new Date().getTime()-x;return t.info(c.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{d as i,f as n,p as r,m as t};
+//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-BT-GZpT1.mjs.map

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-DgWgggQt.mjs.map → pullChunkedCustomSiloOutstandingIdentifiers-BT-GZpT1.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-DgWgggQt.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { GraphQLClient } from 'graphql-request';\n\nimport { REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from './gqls/index.js';\nimport { makeGraphQLRequest } from './makeGraphQLRequest.js';\n\n/**\n * Get number of open requests for a data silo\n *\n * @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n */\nexport async function fetchRequestDataSiloActiveCount(\n  client: GraphQLClient,\n  {\n    dataSiloId,\n  }: {\n    /** Data silo ID */\n    dataSiloId: string;\n  },\n): Promise<number> {\n  const {\n    listReducedRequestsForDataSilo: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Requests */\n    listReducedRequestsForDataSilo: {\n      /** Total count */\n      totalCount: number;\n    };\n  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n    input: {\n      dataSiloId,\n      isResolved: false,\n    },\n  });\n\n  return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nexport const CronIdentifier = t.type({\n  /** The identifier value */\n  identifier: t.string,\n  /** The type of identifier */\n  type: t.string,\n  /** The core identifier of the request */\n  coreIdentifier: t.string,\n  /** The ID of the underlying data silo */\n  dataSiloId: t.string,\n  /** The ID of the underlying request */\n  requestId: t.string,\n  /** The request nonce */\n  nonce: t.string,\n  /** The time the request was created */\n  requestCreatedAt: t.string,\n  /** The number of days until the request is overdue */\n  daysUntilOverdue: t.number,\n  /** Request attributes */\n  attributes: t.array(\n    t.type({\n      key: t.string,\n      values: t.array(t.string),\n    }),\n  ),\n});\n\n/** Type override */\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/**\n * Pull a offset of identifiers for a cron job\n *\n * @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n */\nexport async function pullCronPageOfIdentifiers(\n  sombra: Got,\n  {\n    dataSiloId,\n    limit = 100,\n    offset = 0,\n    requestType,\n  }: {\n    /** Data Silo ID */\n    dataSiloId: string;\n    /** Type of request */\n    requestType: RequestAction;\n    /** Number of identifiers to pull in */\n    limit?: number;\n    /** Page to pull in */\n    offset?: number;\n  },\n): Promise<CronIdentifier[]> {\n  try {\n    // Make the GraphQL request\n    const response = await sombra\n      .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n        searchParams: {\n          offset,\n          limit,\n        },\n      })\n      .json();\n\n    const { items } = decodeCodec(\n      t.type({\n        items: t.array(CronIdentifier),\n      }),\n      response,\n    );\n    return items;\n  } catch (err) {\n    throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n  }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { mapSeries } from '../bluebird.js';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchRequestDataSiloActiveCount,\n} from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/**\n * A CSV formatted identifier\n */\nexport type CsvFormattedIdentifier = {\n  [k in string]: string | null | boolean | number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n  /** The request action that the identifier relates to */\n  action: RequestAction;\n}\n\n/**\n * Pull the set of identifiers outstanding for a cron or AVC integration\n *\n * This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n *\n * @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n */\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n  dataSiloId,\n  auth,\n  sombraAuth,\n  actions,\n  apiPageSize = 100,\n  savePageSize = 1000,\n  onSave,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  skipRequestCount = false,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Data Silo ID to pull down jobs for */\n  dataSiloId: string;\n  /** The request actions to fetch */\n  actions: RequestAction[];\n  /** How many identifiers to pull in a single call to the backend */\n  apiPageSize: number;\n  /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */\n  savePageSize: number;\n  /** Callback function called when a chunk of identifiers is ready to be saved */\n  onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Skip request count */\n  skipRequestCount?: boolean;\n}): Promise<{\n  /** Raw Identifiers */\n  identifiers: CronIdentifierWithAction[];\n}> {\n  // Validate savePageSize\n  if (savePageSize % apiPageSize !== 0) {\n    throw new Error(\n      `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n    );\n  }\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  let totalRequestCount = 0;\n  if (!skipRequestCount) {\n    totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n      dataSiloId,\n    });\n  }\n\n  logger.info(\n    colors.magenta(\n      `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n        `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const foundRequestIds = new Set<string>();\n\n  // identifiers found in total\n  const identifiers: CronIdentifierWithAction[] = [];\n  // current chunk of identifiers to be saved\n  let currentChunk: CsvFormattedIdentifier[] = [];\n\n  // map over each action\n  if (!skipRequestCount) {\n    progressBar.start(totalRequestCount, 0);\n  }\n  await mapSeries(actions, async (action) => {\n    let offset = 0;\n    let shouldContinue = true;\n\n    // Fetch a page of identifiers\n    while (shouldContinue) {\n      const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n        dataSiloId,\n        limit: apiPageSize,\n        offset,\n        requestType: action,\n      });\n\n      const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n        (identifier) => {\n          foundRequestIds.add(identifier.requestId);\n          return {\n            ...identifier,\n            action,\n          };\n        },\n      );\n\n      const csvFormattedIdentifiers = identifiersWithAction.map(\n        ({ attributes, ...identifier }) => ({\n          ...identifier,\n          ...attributes.reduce(\n            (acc, val) =>\n              Object.assign(acc, {\n                [val.key]: val.values.join(','),\n              }),\n            {},\n          ),\n        }),\n      );\n\n      identifiers.push(...identifiersWithAction);\n      currentChunk.push(...csvFormattedIdentifiers);\n\n      // Check if we've reached the savePageSize and call the onSave callback\n      if (currentChunk.length >= savePageSize) {\n        await onSave(currentChunk);\n        currentChunk = [];\n      }\n\n      shouldContinue = pageIdentifiers.length === apiPageSize;\n      offset += apiPageSize;\n      if (!skipRequestCount) {\n        progressBar.update(foundRequestIds.size);\n      } else {\n        logger.info(\n          colors.magenta(\n            `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n          ),\n        );\n      }\n    }\n  });\n\n  // Save any remaining identifiers in the current chunk\n  if (currentChunk.length > 0) {\n    await onSave(currentChunk);\n  }\n\n  if (!skipRequestCount) {\n    progressBar.stop();\n  }\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n        foundRequestIds.size\n      } requests in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n\n  return { identifiers };\n}\n"],"mappings":"qaAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,MAAO,CACL,aACA,WAAY,GACb,CACF,CAAC,CAEF,OAAO,EC/BT,MAAa,EAAiB,EAAE,KAAK,CAEnC,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAEF,IAAM,EAAW,MAAM,EACpB,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CAEH,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC5C5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,EAAW,CAGtE,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,GAAG,MAAkB,CAClC,GAAG,EACH,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}
+{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-BT-GZpT1.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { GraphQLClient } from 'graphql-request';\n\nimport { REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from './gqls/index.js';\nimport { makeGraphQLRequest } from './makeGraphQLRequest.js';\n\n/**\n * Get number of open requests for a data silo\n *\n * @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n */\nexport async function fetchRequestDataSiloActiveCount(\n  client: GraphQLClient,\n  {\n    dataSiloId,\n  }: {\n    /** Data silo ID */\n    dataSiloId: string;\n  },\n): Promise<number> {\n  const {\n    listReducedRequestsForDataSilo: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Requests */\n    listReducedRequestsForDataSilo: {\n      /** Total count */\n      totalCount: number;\n    };\n  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n    input: {\n      dataSiloId,\n      isResolved: false,\n    },\n  });\n\n  return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nexport const CronIdentifier = t.type({\n  /** The identifier value */\n  identifier: t.string,\n  /** The type of identifier */\n  type: t.string,\n  /** The core identifier of the request */\n  coreIdentifier: t.string,\n  /** The ID of the underlying data silo */\n  dataSiloId: t.string,\n  /** The ID of the underlying request */\n  requestId: t.string,\n  /** The request nonce */\n  nonce: t.string,\n  /** The time the request was created */\n  requestCreatedAt: t.string,\n  /** The number of days until the request is overdue */\n  daysUntilOverdue: t.number,\n  /** Request attributes */\n  attributes: t.array(\n    t.type({\n      key: t.string,\n      values: t.array(t.string),\n    }),\n  ),\n});\n\n/** Type override */\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/**\n * Pull a offset of identifiers for a cron job\n *\n * @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n */\nexport async function pullCronPageOfIdentifiers(\n  sombra: Got,\n  {\n    dataSiloId,\n    limit = 100,\n    offset = 0,\n    requestType,\n  }: {\n    /** Data Silo ID */\n    dataSiloId: string;\n    /** Type of request */\n    requestType: RequestAction;\n    /** Number of identifiers to pull in */\n    limit?: number;\n    /** Page to pull in */\n    offset?: number;\n  },\n): Promise<CronIdentifier[]> {\n  try {\n    // Make the GraphQL request\n    const response = await sombra\n      .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n        searchParams: {\n          offset,\n          limit,\n        },\n      })\n      .json();\n\n    const { items } = decodeCodec(\n      t.type({\n        items: t.array(CronIdentifier),\n      }),\n      response,\n    );\n    return items;\n  } catch (err) {\n    throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n  }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { mapSeries } from '../bluebird.js';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchRequestDataSiloActiveCount,\n} from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/**\n * A CSV formatted identifier\n */\nexport type CsvFormattedIdentifier = {\n  [k in string]: string | null | boolean | number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n  /** The request action that the identifier relates to */\n  action: RequestAction;\n}\n\n/**\n * Pull the set of identifiers outstanding for a cron or AVC integration\n *\n * This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n *\n * @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n */\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n  dataSiloId,\n  auth,\n  sombraAuth,\n  actions,\n  apiPageSize = 100,\n  savePageSize = 1000,\n  onSave,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  skipRequestCount = false,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Data Silo ID to pull down jobs for */\n  dataSiloId: string;\n  /** The request actions to fetch */\n  actions: RequestAction[];\n  /** How many identifiers to pull in a single call to the backend */\n  apiPageSize: number;\n  /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */\n  savePageSize: number;\n  /** Callback function called when a chunk of identifiers is ready to be saved */\n  onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Skip request count */\n  skipRequestCount?: boolean;\n}): Promise<{\n  /** Raw Identifiers */\n  identifiers: CronIdentifierWithAction[];\n}> {\n  // Validate savePageSize\n  if (savePageSize % apiPageSize !== 0) {\n    throw new Error(\n      `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n    );\n  }\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  let totalRequestCount = 0;\n  if (!skipRequestCount) {\n    totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n      dataSiloId,\n    });\n  }\n\n  logger.info(\n    colors.magenta(\n      `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n        `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const foundRequestIds = new Set<string>();\n\n  // identifiers found in total\n  const identifiers: CronIdentifierWithAction[] = [];\n  // current chunk of identifiers to be saved\n  let currentChunk: CsvFormattedIdentifier[] = [];\n\n  // map over each action\n  if (!skipRequestCount) {\n    progressBar.start(totalRequestCount, 0);\n  }\n  await mapSeries(actions, async (action) => {\n    let offset = 0;\n    let shouldContinue = true;\n\n    // Fetch a page of identifiers\n    while (shouldContinue) {\n      const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n        dataSiloId,\n        limit: apiPageSize,\n        offset,\n        requestType: action,\n      });\n\n      const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n        (identifier) => {\n          foundRequestIds.add(identifier.requestId);\n          return {\n            ...identifier,\n            action,\n          };\n        },\n      );\n\n      const csvFormattedIdentifiers = identifiersWithAction.map(\n        ({ attributes, ...identifier }) => ({\n          ...identifier,\n          ...attributes.reduce(\n            (acc, val) =>\n              Object.assign(acc, {\n                [val.key]: val.values.join(','),\n              }),\n            {},\n          ),\n        }),\n      );\n\n      identifiers.push(...identifiersWithAction);\n      currentChunk.push(...csvFormattedIdentifiers);\n\n      // Check if we've reached the savePageSize and call the onSave callback\n      if (currentChunk.length >= savePageSize) {\n        await onSave(currentChunk);\n        currentChunk = [];\n      }\n\n      shouldContinue = pageIdentifiers.length === apiPageSize;\n      offset += apiPageSize;\n      if (!skipRequestCount) {\n        progressBar.update(foundRequestIds.size);\n      } else {\n        logger.info(\n          colors.magenta(\n            `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n          ),\n        );\n      }\n    }\n  });\n\n  // Save any remaining identifiers in the current chunk\n  if (currentChunk.length > 0) {\n    await onSave(currentChunk);\n  }\n\n  if (!skipRequestCount) {\n    progressBar.stop();\n  }\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n        foundRequestIds.size\n      } requests in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n\n  return { identifiers };\n}\n"],"mappings":"qaAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,MAAO,CACL,aACA,WAAY,GACb,CACF,CAAC,CAEF,OAAO,EC/BT,MAAa,EAAiB,EAAE,KAAK,CAEnC,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAEF,IAAM,EAAW,MAAM,EACpB,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CAEH,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC5C5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,EAAW,CAGtE,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,GAAG,MAAkB,CAClC,GAAG,EACH,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}

package/dist/{pullConsentManagerMetrics-pFRPXTHJ.mjs → pullConsentManagerMetrics-FnhPEszu.mjs}

@@ -1,2 +1,2 @@
-import{a as e,r as t}from"./fetchConsentManagerId-DHDA5Py9.mjs";async function n(n,{bin:r,start:i,end:a=new Date}){let o=await e(n);if(Math.floor(i.getTime()/1e3)>Math.floor(a.getTime()/1e3))throw Error(`Received "end" date that happened before "start" date`);if(r===`1h`&&a.getTime()-i.getTime()>6048e5*2)throw Error(`When using bin=1h, start and end time can be no more than 2 weeks apart`);let s=i.toISOString(),c=a.toISOString(),[l,u,d]=await Promise.all([t(n,{dataSource:`PRIVACY_SIGNAL_TIMESERIES`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1}),t(n,{dataSource:`CONSENT_CHANGES_TIMESERIES`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1}),t(n,{dataSource:`CONSENT_SESSIONS_BY_REGIME`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1})]);return{PRIVACY_SIGNAL_TIMESERIES:l,CONSENT_CHANGES_TIMESERIES:u,CONSENT_SESSIONS_BY_REGIME:d}}export{n as t};
-//# sourceMappingURL=pullConsentManagerMetrics-pFRPXTHJ.mjs.map
+import{a as e,r as t}from"./fetchConsentManagerId-CFkg3-RS.mjs";async function n(n,{bin:r,start:i,end:a=new Date}){let o=await e(n);if(Math.floor(i.getTime()/1e3)>Math.floor(a.getTime()/1e3))throw Error(`Received "end" date that happened before "start" date`);if(r===`1h`&&a.getTime()-i.getTime()>6048e5*2)throw Error(`When using bin=1h, start and end time can be no more than 2 weeks apart`);let s=i.toISOString(),c=a.toISOString(),[l,u,d]=await Promise.all([t(n,{dataSource:`PRIVACY_SIGNAL_TIMESERIES`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1}),t(n,{dataSource:`CONSENT_CHANGES_TIMESERIES`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1}),t(n,{dataSource:`CONSENT_SESSIONS_BY_REGIME`,startDate:s,endDate:c,forceRefetch:!0,airgapBundleId:o,binInterval:r,smoothTimeseries:!1})]);return{PRIVACY_SIGNAL_TIMESERIES:l,CONSENT_CHANGES_TIMESERIES:u,CONSENT_SESSIONS_BY_REGIME:d}}export{n as t};
+//# sourceMappingURL=pullConsentManagerMetrics-FnhPEszu.mjs.map

package/dist/{pullConsentManagerMetrics-pFRPXTHJ.mjs.map → pullConsentManagerMetrics-FnhPEszu.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullConsentManagerMetrics-pFRPXTHJ.mjs","names":[],"sources":["../src/lib/consent-manager/pullConsentManagerMetrics.ts"],"sourcesContent":["import type { GraphQLClient } from 'graphql-request';\n\nimport {\n  ConsentManagerMetric,\n  ConsentManagerMetricBin,\n  fetchConsentManagerAnalyticsData,\n  fetchConsentManagerId,\n} from '../graphql/index.js';\n\n/**\n * One second of time in ms\n */\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\nconst ONE_DAY = 24 * ONE_HOUR;\nconst ONE_WEEK = 7 * ONE_DAY;\n\n/**\n * Pull consent manager metrics in an organization\n *\n * @param client - GraphQL client\n * @param options - Options\n * @returns The consent manager metrics\n */\nexport async function pullConsentManagerMetrics(\n  client: GraphQLClient,\n  {\n    bin,\n    start,\n    end = new Date(),\n  }: {\n    /** Start date to pull metrics from */\n    start: Date;\n    /** End date to pull metrics from (assumes now) */\n    end?: Date;\n    /** Bin size to pull metrics */\n    bin: ConsentManagerMetricBin;\n  },\n): Promise<{\n  /** Privacy signal data */\n  PRIVACY_SIGNAL_TIMESERIES: ConsentManagerMetric[];\n  /** Consent changes data */\n  CONSENT_CHANGES_TIMESERIES: ConsentManagerMetric[];\n  /** Consent sessions by regime */\n  CONSENT_SESSIONS_BY_REGIME: ConsentManagerMetric[];\n}> {\n  // Grab the bundleId associated with this API key\n  const airgapBundleId = await fetchConsentManagerId(client);\n\n  // convert start and end to times\n  const startTime = Math.floor(start.getTime() / 1000);\n  const endTime = Math.floor(end.getTime() / 1000);\n  if (startTime > endTime) {\n    throw new Error('Received \"end\" date that happened before \"start\" date');\n  }\n\n  // do not allow hourly bins greater than 2 weeks\n  if (bin === '1h' && end.getTime() - start.getTime() > ONE_WEEK * 2) {\n    throw new Error('When using bin=1h, start and end time can be no more than 2 weeks apart');\n  }\n\n  // Pull in the metrics\n  const startDate = start.toISOString();\n  const endDate = end.toISOString();\n  const [privacySignalData, consentChangesData, consentSessionsByRegimeData] = await Promise.all([\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'PRIVACY_SIGNAL_TIMESERIES',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'CONSENT_CHANGES_TIMESERIES',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'CONSENT_SESSIONS_BY_REGIME',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n  ]);\n\n  return {\n    PRIVACY_SIGNAL_TIMESERIES: privacySignalData,\n    CONSENT_CHANGES_TIMESERIES: consentChangesData,\n    CONSENT_SESSIONS_BY_REGIME: consentSessionsByRegimeData,\n  };\n}\n"],"mappings":"gEAyBA,eAAsB,EACpB,EACA,CACE,MACA,QACA,MAAM,IAAI,MAgBX,CAED,IAAM,EAAiB,MAAM,EAAsB,EAAO,CAK1D,GAFkB,KAAK,MAAM,EAAM,SAAS,CAAG,IAAK,CACpC,KAAK,MAAM,EAAI,SAAS,CAAG,IAAK,CAE9C,MAAU,MAAM,wDAAwD,CAI1E,GAAI,IAAQ,MAAQ,EAAI,SAAS,CAAG,EAAM,SAAS,CAAG,OAAW,EAC/D,MAAU,MAAM,0EAA0E,CAI5F,IAAM,EAAY,EAAM,aAAa,CAC/B,EAAU,EAAI,aAAa,CAC3B,CAAC,EAAmB,EAAoB,GAA+B,MAAM,QAAQ,IAAI,CAC7F,EAAiC,EAAQ,CACvC,WAAY,4BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACF,EAAiC,EAAQ,CACvC,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACF,EAAiC,EAAQ,CACvC,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACH,CAAC,CAEF,MAAO,CACL,0BAA2B,EAC3B,2BAA4B,EAC5B,2BAA4B,EAC7B"}
+{"version":3,"file":"pullConsentManagerMetrics-FnhPEszu.mjs","names":[],"sources":["../src/lib/consent-manager/pullConsentManagerMetrics.ts"],"sourcesContent":["import type { GraphQLClient } from 'graphql-request';\n\nimport {\n  ConsentManagerMetric,\n  ConsentManagerMetricBin,\n  fetchConsentManagerAnalyticsData,\n  fetchConsentManagerId,\n} from '../graphql/index.js';\n\n/**\n * One second of time in ms\n */\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\nconst ONE_DAY = 24 * ONE_HOUR;\nconst ONE_WEEK = 7 * ONE_DAY;\n\n/**\n * Pull consent manager metrics in an organization\n *\n * @param client - GraphQL client\n * @param options - Options\n * @returns The consent manager metrics\n */\nexport async function pullConsentManagerMetrics(\n  client: GraphQLClient,\n  {\n    bin,\n    start,\n    end = new Date(),\n  }: {\n    /** Start date to pull metrics from */\n    start: Date;\n    /** End date to pull metrics from (assumes now) */\n    end?: Date;\n    /** Bin size to pull metrics */\n    bin: ConsentManagerMetricBin;\n  },\n): Promise<{\n  /** Privacy signal data */\n  PRIVACY_SIGNAL_TIMESERIES: ConsentManagerMetric[];\n  /** Consent changes data */\n  CONSENT_CHANGES_TIMESERIES: ConsentManagerMetric[];\n  /** Consent sessions by regime */\n  CONSENT_SESSIONS_BY_REGIME: ConsentManagerMetric[];\n}> {\n  // Grab the bundleId associated with this API key\n  const airgapBundleId = await fetchConsentManagerId(client);\n\n  // convert start and end to times\n  const startTime = Math.floor(start.getTime() / 1000);\n  const endTime = Math.floor(end.getTime() / 1000);\n  if (startTime > endTime) {\n    throw new Error('Received \"end\" date that happened before \"start\" date');\n  }\n\n  // do not allow hourly bins greater than 2 weeks\n  if (bin === '1h' && end.getTime() - start.getTime() > ONE_WEEK * 2) {\n    throw new Error('When using bin=1h, start and end time can be no more than 2 weeks apart');\n  }\n\n  // Pull in the metrics\n  const startDate = start.toISOString();\n  const endDate = end.toISOString();\n  const [privacySignalData, consentChangesData, consentSessionsByRegimeData] = await Promise.all([\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'PRIVACY_SIGNAL_TIMESERIES',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'CONSENT_CHANGES_TIMESERIES',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n    fetchConsentManagerAnalyticsData(client, {\n      dataSource: 'CONSENT_SESSIONS_BY_REGIME',\n      startDate,\n      endDate,\n      forceRefetch: true,\n      airgapBundleId,\n      binInterval: bin,\n      smoothTimeseries: false,\n    }),\n  ]);\n\n  return {\n    PRIVACY_SIGNAL_TIMESERIES: privacySignalData,\n    CONSENT_CHANGES_TIMESERIES: consentChangesData,\n    CONSENT_SESSIONS_BY_REGIME: consentSessionsByRegimeData,\n  };\n}\n"],"mappings":"gEAyBA,eAAsB,EACpB,EACA,CACE,MACA,QACA,MAAM,IAAI,MAgBX,CAED,IAAM,EAAiB,MAAM,EAAsB,EAAO,CAK1D,GAFkB,KAAK,MAAM,EAAM,SAAS,CAAG,IAAK,CACpC,KAAK,MAAM,EAAI,SAAS,CAAG,IAAK,CAE9C,MAAU,MAAM,wDAAwD,CAI1E,GAAI,IAAQ,MAAQ,EAAI,SAAS,CAAG,EAAM,SAAS,CAAG,OAAW,EAC/D,MAAU,MAAM,0EAA0E,CAI5F,IAAM,EAAY,EAAM,aAAa,CAC/B,EAAU,EAAI,aAAa,CAC3B,CAAC,EAAmB,EAAoB,GAA+B,MAAM,QAAQ,IAAI,CAC7F,EAAiC,EAAQ,CACvC,WAAY,4BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACF,EAAiC,EAAQ,CACvC,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACF,EAAiC,EAAQ,CACvC,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CAAC,CACH,CAAC,CAEF,MAAO,CACL,0BAA2B,EAC3B,2BAA4B,EAC5B,2BAA4B,EAC7B"}

package/dist/{pullManualEnrichmentIdentifiersToCsv-DA_4rIzW.mjs → pullManualEnrichmentIdentifiersToCsv-B_4REnga.mjs}

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./createSombraGotInstance-CahOgD6V.mjs";import{t as i}from"./fetchAllRequestEnrichers-q34mRuE5.mjs";import{n as a,r as o}from"./fetchAllRequestIdentifiers-YP-geTV4.mjs";import{r as s}from"./makeGraphQLRequest-G078PsEL.mjs";import{r as c}from"./fetchAllRequests-DEPTEUbi.mjs";import{a as l}from"./writeCsv-B51ulrVl.mjs";import{RequestStatus as u}from"@transcend-io/privacy-types";import{groupBy as d,uniq as f}from"lodash-es";import p from"colors";async function m({file:m,auth:h,sombraAuth:g,requestActions:_=[],concurrency:v=100,transcendUrl:y=e}){let b=s(y,h),x=await r(y,h,g);t.info(p.magenta(`Pulling manual enrichment requests, filtered for actions: ${_.join(`,`)}`));let S=await c(b,{actions:_,statuses:[u.Enriching]});await o(b);let C=[];await n(S,async e=>{let t=await i(b,{requestId:e.id});if(t.filter(({status:e})=>e===`ACTION_REQUIRED`)){let n=await a(b,x,{requestId:e.id,skipSombraCheck:!0});C.push({...e,requestIdentifiers:n,requestEnrichers:t})}},{concurrency:v});let w=C.map(({attributeValues:e,requestIdentifiers:t,requestEnrichers:n,...r})=>({...r,...Object.entries(d(t,`name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({value:e})=>e).join(`,`)}),{}),...Object.entries(d(e,`attributeKey.name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({name:e})=>e).join(`,`)}),{})}));return await l(m,w,f(w.map(e=>Object.keys(e)).flat())),t.info(p.green(`Successfully wrote ${C.length} requests to file "${m}"`)),C}export{m as t};
-//# sourceMappingURL=pullManualEnrichmentIdentifiersToCsv-DA_4rIzW.mjs.map
+import{a as e}from"./constants-CeMiHaHx.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./bluebird-CUitXgsY.mjs";import{t as r}from"./createSombraGotInstance-D1Il9zUE.mjs";import{t as i}from"./fetchAllRequestEnrichers-CK-kk5eg.mjs";import{n as a,r as o}from"./fetchAllRequestIdentifiers-DrFFOt0m.mjs";import{r as s}from"./makeGraphQLRequest-Cq26A_Lq.mjs";import{r as c}from"./fetchAllRequests-DNQQsY4s.mjs";import{a as l}from"./writeCsv-B51ulrVl.mjs";import{RequestStatus as u}from"@transcend-io/privacy-types";import{groupBy as d,uniq as f}from"lodash-es";import p from"colors";async function m({file:m,auth:h,sombraAuth:g,requestActions:_=[],concurrency:v=100,transcendUrl:y=e}){let b=s(y,h),x=await r(y,h,g);t.info(p.magenta(`Pulling manual enrichment requests, filtered for actions: ${_.join(`,`)}`));let S=await c(b,{actions:_,statuses:[u.Enriching]});await o(b);let C=[];await n(S,async e=>{let t=await i(b,{requestId:e.id});if(t.filter(({status:e})=>e===`ACTION_REQUIRED`)){let n=await a(b,x,{requestId:e.id,skipSombraCheck:!0});C.push({...e,requestIdentifiers:n,requestEnrichers:t})}},{concurrency:v});let w=C.map(({attributeValues:e,requestIdentifiers:t,requestEnrichers:n,...r})=>({...r,...Object.entries(d(t,`name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({value:e})=>e).join(`,`)}),{}),...Object.entries(d(e,`attributeKey.name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({name:e})=>e).join(`,`)}),{})}));return await l(m,w,f(w.map(e=>Object.keys(e)).flat())),t.info(p.green(`Successfully wrote ${C.length} requests to file "${m}"`)),C}export{m as t};
+//# sourceMappingURL=pullManualEnrichmentIdentifiersToCsv-B_4REnga.mjs.map

package/dist/{pullManualEnrichmentIdentifiersToCsv-DA_4rIzW.mjs.map → pullManualEnrichmentIdentifiersToCsv-B_4REnga.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullManualEnrichmentIdentifiersToCsv-DA_4rIzW.mjs","names":[],"sources":["../src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts"],"sourcesContent":["import { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport colors from 'colors';\nimport { groupBy, uniq } from 'lodash-es';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { map } from '../bluebird.js';\nimport {\n  PrivacyRequest,\n  RequestEnricher,\n  RequestIdentifier,\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllRequestEnrichers,\n  fetchAllRequestIdentifiers,\n  fetchAllRequests,\n  validateSombraVersion,\n} from '../graphql/index.js';\nimport { writeCsv } from '../helpers/writeCsv.js';\n\nexport interface PrivacyRequestWithIdentifiers extends PrivacyRequest {\n  /** Request Enrichers */\n  requestEnrichers: RequestEnricher[];\n  /** Request Identifiers */\n  requestIdentifiers: RequestIdentifier[];\n}\n\n/**\n * Pull the set of manual enrichment jobs to CSV\n *\n * @param options - Options\n * @returns List of requests with identifiers\n */\nexport async function pullManualEnrichmentIdentifiersToCsv({\n  file,\n  auth,\n  sombraAuth,\n  requestActions = [],\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** CSV file path */\n  file: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Sombra API key */\n  sombraAuth?: string;\n  /** Concurrency */\n  concurrency?: number;\n  /** The request actions to fetch */\n  requestActions?: RequestAction[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<PrivacyRequestWithIdentifiers[]> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);\n\n  logger.info(\n    colors.magenta(\n      `Pulling manual enrichment requests, filtered for actions: ${requestActions.join(',')}`,\n    ),\n  );\n\n  // Pull all privacy requests\n  const allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    statuses: [RequestStatus.Enriching],\n  });\n\n  await validateSombraVersion(client);\n\n  // Requests to save\n  const savedRequests: PrivacyRequestWithIdentifiers[] = [];\n\n  // Filter down requests to what is needed\n  await map(\n    allRequests,\n    async (request) => {\n      // Fetch enrichers\n      const requestEnrichers = await fetchAllRequestEnrichers(client, {\n        requestId: request.id,\n      });\n\n      // Check if manual enrichment exists for that request\n      const hasManualEnrichment = requestEnrichers.filter(\n        ({ status }) => status === 'ACTION_REQUIRED',\n      );\n\n      // Save request to queue\n      if (hasManualEnrichment) {\n        const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, {\n          requestId: request.id,\n          skipSombraCheck: true,\n        });\n        savedRequests.push({\n          ...request,\n          requestIdentifiers,\n          requestEnrichers,\n        });\n      }\n    },\n    {\n      concurrency,\n    },\n  );\n\n  const data = savedRequests.map(\n    ({\n      attributeValues,\n      requestIdentifiers,\n      requestEnrichers, // eslint-disable-line @typescript-eslint/no-unused-vars\n      ...request\n    }) => ({\n      ...request,\n      // flatten identifiers\n      ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ value }) => value).join(','),\n          }),\n        {},\n      ),\n      // flatten attributes\n      ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ name }) => name).join(','),\n          }),\n        {},\n      ),\n    }),\n  );\n\n  // Write out to CSV\n  const headers = uniq(data.map((d) => Object.keys(d)).flat());\n  await writeCsv(file, data, headers);\n\n  logger.info(\n    colors.green(`Successfully wrote ${savedRequests.length} requests to file \"${file}\"`),\n  );\n\n  return savedRequests;\n}\n"],"mappings":"glBAiCA,eAAsB,EAAqC,CACzD,OACA,OACA,aACA,iBAAiB,EAAE,CACnB,cAAc,IACd,eAAe,GAc4B,CAE3C,IAAM,EAAS,EAA4B,EAAc,EAAK,CACxD,EAAS,MAAM,EAAwB,EAAc,EAAM,EAAW,CAE5E,EAAO,KACL,EAAO,QACL,6DAA6D,EAAe,KAAK,IAAI,GACtF,CACF,CAGD,IAAM,EAAc,MAAM,EAAiB,EAAQ,CACjD,QAAS,EACT,SAAU,CAAC,EAAc,UAAU,CACpC,CAAC,CAEF,MAAM,EAAsB,EAAO,CAGnC,IAAM,EAAiD,EAAE,CAGzD,MAAM,EACJ,EACA,KAAO,IAAY,CAEjB,IAAM,EAAmB,MAAM,EAAyB,EAAQ,CAC9D,UAAW,EAAQ,GACpB,CAAC,CAQF,GAL4B,EAAiB,QAC1C,CAAE,YAAa,IAAW,kBAC5B,CAGwB,CACvB,IAAM,EAAqB,MAAM,EAA2B,EAAQ,EAAQ,CAC1E,UAAW,EAAQ,GACnB,gBAAiB,GAClB,CAAC,CACF,EAAc,KAAK,CACjB,GAAG,EACH,qBACA,mBACD,CAAC,GAGN,CACE,cACD,CACF,CAED,IAAM,EAAO,EAAc,KACxB,CACC,kBACA,qBACA,mBACA,GAAG,MACE,CACL,GAAG,EAEH,GAAG,OAAO,QAAQ,EAAQ,EAAoB,OAAO,CAAC,CAAC,QACpD,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,WAAY,EAAM,CAAC,KAAK,IAAI,CAClD,CAAC,CACJ,EAAE,CACH,CAED,GAAG,OAAO,QAAQ,EAAQ,EAAiB,oBAAoB,CAAC,CAAC,QAC9D,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,UAAW,EAAK,CAAC,KAAK,IAAI,CAChD,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAUD,OANA,MAAM,EAAS,EAAM,EADL,EAAK,EAAK,IAAK,GAAM,OAAO,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CACzB,CAEnC,EAAO,KACL,EAAO,MAAM,sBAAsB,EAAc,OAAO,qBAAqB,EAAK,GAAG,CACtF,CAEM"}
+{"version":3,"file":"pullManualEnrichmentIdentifiersToCsv-B_4REnga.mjs","names":[],"sources":["../src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts"],"sourcesContent":["import { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport colors from 'colors';\nimport { groupBy, uniq } from 'lodash-es';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { map } from '../bluebird.js';\nimport {\n  PrivacyRequest,\n  RequestEnricher,\n  RequestIdentifier,\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllRequestEnrichers,\n  fetchAllRequestIdentifiers,\n  fetchAllRequests,\n  validateSombraVersion,\n} from '../graphql/index.js';\nimport { writeCsv } from '../helpers/writeCsv.js';\n\nexport interface PrivacyRequestWithIdentifiers extends PrivacyRequest {\n  /** Request Enrichers */\n  requestEnrichers: RequestEnricher[];\n  /** Request Identifiers */\n  requestIdentifiers: RequestIdentifier[];\n}\n\n/**\n * Pull the set of manual enrichment jobs to CSV\n *\n * @param options - Options\n * @returns List of requests with identifiers\n */\nexport async function pullManualEnrichmentIdentifiersToCsv({\n  file,\n  auth,\n  sombraAuth,\n  requestActions = [],\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** CSV file path */\n  file: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Sombra API key */\n  sombraAuth?: string;\n  /** Concurrency */\n  concurrency?: number;\n  /** The request actions to fetch */\n  requestActions?: RequestAction[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<PrivacyRequestWithIdentifiers[]> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const sombra = await createSombraGotInstance(transcendUrl, auth, sombraAuth);\n\n  logger.info(\n    colors.magenta(\n      `Pulling manual enrichment requests, filtered for actions: ${requestActions.join(',')}`,\n    ),\n  );\n\n  // Pull all privacy requests\n  const allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    statuses: [RequestStatus.Enriching],\n  });\n\n  await validateSombraVersion(client);\n\n  // Requests to save\n  const savedRequests: PrivacyRequestWithIdentifiers[] = [];\n\n  // Filter down requests to what is needed\n  await map(\n    allRequests,\n    async (request) => {\n      // Fetch enrichers\n      const requestEnrichers = await fetchAllRequestEnrichers(client, {\n        requestId: request.id,\n      });\n\n      // Check if manual enrichment exists for that request\n      const hasManualEnrichment = requestEnrichers.filter(\n        ({ status }) => status === 'ACTION_REQUIRED',\n      );\n\n      // Save request to queue\n      if (hasManualEnrichment) {\n        const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, {\n          requestId: request.id,\n          skipSombraCheck: true,\n        });\n        savedRequests.push({\n          ...request,\n          requestIdentifiers,\n          requestEnrichers,\n        });\n      }\n    },\n    {\n      concurrency,\n    },\n  );\n\n  const data = savedRequests.map(\n    ({\n      attributeValues,\n      requestIdentifiers,\n      requestEnrichers, // eslint-disable-line @typescript-eslint/no-unused-vars\n      ...request\n    }) => ({\n      ...request,\n      // flatten identifiers\n      ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ value }) => value).join(','),\n          }),\n        {},\n      ),\n      // flatten attributes\n      ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ name }) => name).join(','),\n          }),\n        {},\n      ),\n    }),\n  );\n\n  // Write out to CSV\n  const headers = uniq(data.map((d) => Object.keys(d)).flat());\n  await writeCsv(file, data, headers);\n\n  logger.info(\n    colors.green(`Successfully wrote ${savedRequests.length} requests to file \"${file}\"`),\n  );\n\n  return savedRequests;\n}\n"],"mappings":"glBAiCA,eAAsB,EAAqC,CACzD,OACA,OACA,aACA,iBAAiB,EAAE,CACnB,cAAc,IACd,eAAe,GAc4B,CAE3C,IAAM,EAAS,EAA4B,EAAc,EAAK,CACxD,EAAS,MAAM,EAAwB,EAAc,EAAM,EAAW,CAE5E,EAAO,KACL,EAAO,QACL,6DAA6D,EAAe,KAAK,IAAI,GACtF,CACF,CAGD,IAAM,EAAc,MAAM,EAAiB,EAAQ,CACjD,QAAS,EACT,SAAU,CAAC,EAAc,UAAU,CACpC,CAAC,CAEF,MAAM,EAAsB,EAAO,CAGnC,IAAM,EAAiD,EAAE,CAGzD,MAAM,EACJ,EACA,KAAO,IAAY,CAEjB,IAAM,EAAmB,MAAM,EAAyB,EAAQ,CAC9D,UAAW,EAAQ,GACpB,CAAC,CAQF,GAL4B,EAAiB,QAC1C,CAAE,YAAa,IAAW,kBAC5B,CAGwB,CACvB,IAAM,EAAqB,MAAM,EAA2B,EAAQ,EAAQ,CAC1E,UAAW,EAAQ,GACnB,gBAAiB,GAClB,CAAC,CACF,EAAc,KAAK,CACjB,GAAG,EACH,qBACA,mBACD,CAAC,GAGN,CACE,cACD,CACF,CAED,IAAM,EAAO,EAAc,KACxB,CACC,kBACA,qBACA,mBACA,GAAG,MACE,CACL,GAAG,EAEH,GAAG,OAAO,QAAQ,EAAQ,EAAoB,OAAO,CAAC,CAAC,QACpD,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,WAAY,EAAM,CAAC,KAAK,IAAI,CAClD,CAAC,CACJ,EAAE,CACH,CAED,GAAG,OAAO,QAAQ,EAAQ,EAAiB,oBAAoB,CAAC,CAAC,QAC9D,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,UAAW,EAAK,CAAC,KAAK,IAAI,CAChD,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAUD,OANA,MAAM,EAAS,EAAM,EADL,EAAK,EAAK,IAAK,GAAM,OAAO,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CACzB,CAEnC,EAAO,KACL,EAAO,MAAM,sBAAsB,EAAc,OAAO,qBAAqB,EAAK,GAAG,CACtF,CAEM"}

package/dist/{pullTranscendConfiguration-D2cYlu6V.mjs → pullTranscendConfiguration-CqsgEf9A.mjs}

@@ -1,4 +1,4 @@
-import{o as e}from"./enums-CyFTrzXY.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./fetchIdentifiers-DjqjUnaw.mjs";import{n as r}from"./fetchApiKeys-DkBco7W0.mjs";import{t as i}from"./syncTemplates-BNu1_dmW.mjs";import{Bt as a,C as o,D as s,F as c,I as l,M as u,N as d,O as f,P as p,S as m,T as h,_ as g,b as _,f as v,g as y,h as b,i as x,k as S,l as C,n as w,ot as T,p as ee,t as te,w as ne,x as re}from"./fetchAllActions-C0l3wjQV.mjs";import{n as ie,t as E}from"./assessment-BDywVaGR.mjs";import{a as D,i as ae,l as O,n as oe,o as se}from"./fetchConsentManagerId-DHDA5Py9.mjs";import{t as k}from"./makeGraphQLRequest-G078PsEL.mjs";import{t as ce}from"./fetchAllDataFlows-AQ9j_NRa.mjs";import{t as le}from"./fetchAllPurposesAndPreferences-Dog6N9L2.mjs";import{ActionItemCode as ue,AssessmentsDisplayLogicAction as de,ComparisonOperator as A,ConsentTrackerStatus as j,LogicOperator as M,RequestAction as fe,RetentionType as pe}from"@transcend-io/privacy-types";import{flatten as me,keyBy as he,mapValues as ge}from"lodash-es";import{decodeCodec as N,valuesOf as P}from"@transcend-io/type-utils";import _e from"colors";import*as F from"io-ts";import{gql as I}from"graphql-request";const L=I`
+import{o as e}from"./enums-CyFTrzXY.mjs";import{t}from"./logger-B-LXIf3U.mjs";import{t as n}from"./fetchIdentifiers-pjQV4vUg.mjs";import{n as r}from"./fetchApiKeys-DjOr44xA.mjs";import{t as i}from"./syncTemplates-BrH7Yr0V.mjs";import{Bt as a,C as o,D as s,F as c,I as l,M as u,N as d,O as f,P as p,S as m,T as h,_ as g,b as _,f as v,g as y,h as b,i as x,k as S,l as C,n as w,ot as T,p as ee,t as te,w as ne,x as re}from"./fetchAllActions-BJsPdnxy.mjs";import{n as ie,t as E}from"./assessment-BDywVaGR.mjs";import{a as D,i as ae,l as O,n as oe,o as se}from"./fetchConsentManagerId-CFkg3-RS.mjs";import{t as k}from"./makeGraphQLRequest-Cq26A_Lq.mjs";import{t as ce}from"./fetchAllDataFlows-D248lO6_.mjs";import{t as le}from"./fetchAllPurposesAndPreferences-DD6OyA5t.mjs";import{ActionItemCode as ue,AssessmentsDisplayLogicAction as de,ComparisonOperator as A,ConsentTrackerStatus as j,LogicOperator as M,RequestAction as fe,RetentionType as pe}from"@transcend-io/privacy-types";import{flatten as me,keyBy as he,mapValues as ge}from"lodash-es";import{decodeCodec as N,valuesOf as P}from"@transcend-io/type-utils";import _e from"colors";import*as F from"io-ts";import{gql as I}from"graphql-request";const L=I`
   query TranscendCliAssessmentTemplates(
     $first: Int!
     $offset: Int!
@@ -77,4 +77,4 @@ import{o as e}from"./enums-CyFTrzXY.mjs";import{t}from"./logger-B-LXIf3U.mjs";im
     }
   }
 `;async function z(e){let t=[],n=0,r=!1;do{let{assessmentForms:{nodes:i}}=await k(e,E,{first:20,offset:n});t.push(...i),n+=20,r=i.length===20}while(r);return t.sort((e,t)=>e.title.localeCompare(t.title))}async function ve(e){let t=[],n=0,r=!1;do{let{assessmentFormTemplates:{nodes:i}}=await k(e,L,{first:20,offset:n});t.push(...i),n+=20,r=i.length===20}while(r);return t.sort((e,t)=>e.title.localeCompare(t.title))}async function B(e,t=j.Live){let n=[],r=0,i=await D(e),a=!1;do{let{cookies:{nodes:o}}=await k(e,O,{first:20,offset:r,airgapBundleId:i,status:t});n.push(...o),r+=20,a=o.length===20}while(a);return n.sort((e,t)=>e.name.localeCompare(t.name))}async function ye(e){let{translatedMessages:t}=await k(e,a,{});return t}function V(e){let t=[];return e.map(e=>{let n=t.find(t=>t.key===e.attributeKey.name);return n===void 0?(n={key:e.attributeKey.name,values:[e.name]},t.push(n)):n.values.push(e.name),t}),t}async function be(e){let{privacyCenter:{themeStr:t,...n}}=await k(e,T,{url:await s(e)});return[{...n,theme:JSON.parse(t)}]}const H=F.type({dependsOnQuestionReferenceId:F.string,comparisonOperator:F.union([F.literal(A.IsEqualTo),F.literal(A.IsNotEqualTo),F.literal(A.IsOneOf),F.literal(A.IsNotOneOf),F.literal(A.Contains)]),comparisonOperands:F.array(F.string)}),U=F.type({dependsOnQuestionReferenceId:F.string,comparisonOperator:F.union([F.literal(A.IsNotShown),F.literal(A.IsShown)])}),W=F.union([H,U]),G=F.recursion(`AssessmentNestedRule`,e=>F.intersection([F.type({logicOperator:P(M)}),F.partial({rules:F.array(W),nestedRules:F.array(e)})])),K=F.partial({action:P(de),rule:W,nestedRule:G});function q(e){return N(K,e)}const J=F.intersection([F.partial({riskAssignment:F.partial({riskLevelId:F.string,riskMatrixRowId:F.string,riskMatrixColumnId:F.string})}),F.type({comparisonOperands:F.array(F.string),comparisonOperator:P(A)})]);function Y(e){return N(J,e)}async function xe(e){let t=[],n=0,r=!1;do{let{siloDiscoveryResults:{nodes:i}}=await k(e,R,{first:30,offset:n,input:{},filterBy:{}}),a=i.map(e=>e.title===null&&e.suggestedCatalog?.title?{...e,title:e.suggestedCatalog.title}:e);t.push(...a),n+=30,r=i.length===30}while(r);return t}function Se(e){return e.map(({country:e,countrySubDivision:t})=>({country:e,...t?{countrySubDivision:t}:{}}))}const Ce=[e.DataSilos,e.Enrichers,e.Templates,e.ApiKeys];async function we(a,{dataSiloIds:s,integrationNames:T,debug:ie,resources:E=Ce,pageSize:D,skipDatapoints:O,includeGuessedCategories:k,skipSubDatapoints:de,trackerStatuses:A=Object.values(j)}){if(s.length>0&&T.length>0)throw Error(`Only 1 of integrationNames OR dataSiloIds can be provided`);t.info(_e.magenta(`Fetching data with page size ${D}...`));let[M,N,P,F,I,L,R,H,U,W,G,K,J,we,Te,Ee,De,Oe,ke,Ae,je,Me,Ne,Pe,Fe,Ie,Le,X,Re,ze,Be,Ve,He,Ue]=await Promise.all([E.includes(e.DataSilos)||E.includes(e.DataSubjects)?_(a):[],E.includes(e.ApiKeys)?r({},a,!0):[],E.includes(e.DataSilos)?C(a,{ids:s,integrationNames:T,pageSize:D,debug:ie,includeGuessedCategories:k,skipDatapoints:O,skipSubDatapoints:de}):[],E.includes(e.Enrichers)?x(a):[],E.includes(e.DataFlows)?[...A.includes(j.Live)?await ce(a,j.Live):[],...A.includes(j.NeedsReview)?await ce(a,j.NeedsReview):[]]:[],E.includes(e.Cookies)?[...A.includes(j.Live)?await B(a,j.Live):[],...A.includes(j.NeedsReview)?await B(a,j.NeedsReview):[]]:[],E.includes(e.Attributes)?u(a):[],E.includes(e.Templates)?i(a):[],E.includes(e.Identifiers)?n(a):[],E.includes(e.Actions)?te(a):[],E.includes(e.BusinessEntities)?S(a):[],E.includes(e.ProcessingActivities)?ne(a):[],E.includes(e.ConsentManager)?oe(a):void 0,E.includes(e.ConsentManager)?ae(a):[],E.includes(e.Prompts)?ee(a):[],E.includes(e.PromptPartials)?b(a):[],E.includes(e.PromptGroups)?y(a):[],E.includes(e.Agents)?d(a):[],E.includes(e.AgentFunctions)?p(a):[],E.includes(e.AgentFiles)?c(a):[],E.includes(e.Vendors)?re(a):[],E.includes(e.DataCategories)?f(a):[],E.includes(e.ProcessingPurposes)?o(a):[],E.includes(e.ActionItems)?l(a,{type:[ue.Onboarding]}):[],E.includes(e.ActionItemCollections)?v(a):[],E.includes(e.Teams)?m(a):[],E.includes(e.Policies)?h(a):[],E.includes(e.PrivacyCenters)?be(a):[],E.includes(e.Messages)?ye(a):[],E.includes(e.Partitions)?w(a):[],E.includes(e.Assessments)?z(a):[],E.includes(e.AssessmentTemplates)?ve(a):[],E.includes(e.Purposes)?le(a):[],E.includes(e.SystemDiscovery)?xe(a):[]]),Z=E.includes(e.ConsentManager)&&J?await se(a,J.id):void 0,Q={},We=me(P.map(([{apiKeys:e}])=>e.map(({title:e})=>e))),$=Object.values(N).filter(({title:t})=>E.includes(e.ApiKeys)?!0:We.includes(t));if($.length>0&&E.includes(e.ApiKeys)&&(Q[`api-keys`]=$.map(({title:e})=>({title:e}))),ze.length>0&&E.includes(e.Partitions)&&(Q.partitions=ze.map(({name:e,partition:t})=>({name:e,partition:t}))),J&&E.includes(e.ConsentManager)&&(Q[`consent-manager`]={bundleUrls:{TEST:J.testBundleURL,PRODUCTION:J.bundleURL},domains:J.configuration.domains||void 0,partition:J.configuration.partition||void 0,consentPrecedence:J.configuration.consentPrecedence||void 0,unknownRequestPolicy:J.configuration.unknownRequestPolicy||void 0,unknownCookiePolicy:J.configuration.unknownCookiePolicy||void 0,syncEndpoint:J.configuration.syncEndpoint||void 0,telemetryPartitioning:J.configuration.telemetryPartitioning||void 0,signedIabAgreement:J.configuration.signedIabAgreement||void 0,syncGroups:J.configuration.syncGroups||void 0,theme:Z?{primaryColor:Z.primaryColor||void 0,fontColor:Z.fontColor||void 0,privacyPolicy:Z.privacyPolicy||void 0,prompt:Z.prompt}:void 0,experiences:we.map(e=>({name:e.name,displayName:e.displayName||void 0,regions:e.regions.map(e=>({countrySubDivision:e.countrySubDivision||void 0,country:e.country||void 0})),onConsentExpiry:e.onConsentExpiry,consentExpiry:e.consentExpiry,operator:e.operator,displayPriority:e.displayPriority,viewState:e.viewState,purposes:e.purposes.map(e=>({trackingType:e.trackingType})),optedOutPurposes:e.optedOutPurposes.map(e=>({trackingType:e.trackingType})),browserLanguages:e.browserLanguages,browserTimeZones:e.browserTimeZones}))}),Be.length>0&&E.includes(e.Assessments)&&(Q.assessments=Be.map(({title:e,assessmentGroup:t,sections:n,creator:r,description:i,status:a,assignees:o,externalAssignees:s,reviewers:c,isLocked:l,isArchived:u,isExternallyCreated:d,dueDate:f,createdAt:p,assignedAt:m,submittedAt:h,approvedAt:g,rejectedAt:_,titleIsInternal:v,retentionSchedule:y,attributeValues:b,resources:x,syncedRows:S})=>({title:e,group:t.title,sections:n.map(({title:e,status:t,questions:n,assignees:r,isReviewed:i,externalAssignees:a})=>({title:e,status:t,questions:n.map(({title:e,type:t,subType:n,placeholder:r,description:i,isRequired:a,referenceId:o,displayLogic:s,riskLogic:c,riskCategories:l,riskFramework:u,answerOptions:d,selectedAnswers:f,allowedMimeTypes:p,allowSelectOther:m,syncModel:h,syncColumn:g,attributeKey:_,requireRiskEvaluation:v,requireRiskMatrixEvaluation:y})=>{let b=s?q(s):void 0;return{title:e,type:t,"sub-type":n,placeholder:r,description:i,"is-required":a,"reference-id":o,"display-logic":b&&Object.keys(b).length>0?{action:b.action,rule:b.rule?{"depends-on-question-reference-id":b.rule.dependsOnQuestionReferenceId,"comparison-operator":b.rule.comparisonOperator,"comparison-operands":`comparisonOperands`in b.rule?b.rule.comparisonOperands:void 0}:void 0,"nested-rule":b.nestedRule?{"logic-operator":b.nestedRule.logicOperator,rules:(b.nestedRule.rules||[]).map(e=>({"depends-on-question-reference-id":e.dependsOnQuestionReferenceId,"comparison-operator":e.comparisonOperator,"comparison-operands":`comparisonOperands`in e?e.comparisonOperands:void 0}))}:void 0}:void 0,"risk-logic":c.map(e=>{let t=Y(e);return{"risk-level":t.riskAssignment?.riskLevelId,"comparison-operands":t.comparisonOperands,"comparison-operator":t.comparisonOperator}}),"risk-categories":l.map(({title:e})=>e),"risk-framework":u?.title,"answer-options":d.map(({value:e})=>({value:e})),"selected-answers":f.map(({value:e})=>e),"allowed-mime-types":p,"allow-select-other":m,"sync-model":h||void 0,"sync-column":g||void 0,"attribute-key":_?.name,"require-risk-evaluation":v,"require-risk-matrix-evaluation":y}}),assignees:r.map(({email:e})=>e),"external-assignees":a.map(({email:e})=>e),"is-reviewed":i})),creator:r?.email,description:i,status:a,assignees:o.map(({email:e})=>e),"external-assignees":s.map(({email:e})=>e),reviewers:c.map(({email:e})=>e),locked:l,archived:u,external:d,"title-is-internal":v,"due-date":f||void 0,"created-at":p||void 0,"assigned-at":m||void 0,"submitted-at":h||void 0,"approved-at":g||void 0,"rejected-at":_||void 0,"retention-schedule":y?{type:y.type,"duration-days":y.durationDays,operand:y.operation}:void 0,attributes:b!==void 0&&b.length>0?V(b):void 0,resources:x.map(({resourceType:e,title:t,name:n,category:r,type:i,purpose:a})=>({type:e,title:r?`${r} - ${n}`:a?`${a} - ${n}`:t||n||i||``})),rows:S.map(({resourceType:e,title:t,name:n,category:r,type:i,purpose:a})=>({type:e,title:r?`${r} - ${n}`:a?`${a} - ${n}`:t||n||i||``}))}))),Ve.length>0&&E.includes(e.AssessmentTemplates)&&(Q[`assessment-templates`]=Ve.map(({title:e,description:t,sections:n,status:r,source:i,creator:a,isLocked:o,isArchived:s,createdAt:c,retentionSchedule:l})=>({title:e,description:t,sections:n.map(({title:e,questions:t})=>({title:e,questions:t.map(({title:e,type:t,subType:n,placeholder:r,description:i,isRequired:a,referenceId:o,displayLogic:s,riskLogic:c,riskCategories:l,riskFramework:u,answerOptions:d,allowedMimeTypes:f,allowSelectOther:p,syncModel:m,syncColumn:h,attributeKey:g,requireRiskEvaluation:_,requireRiskMatrixEvaluation:v})=>{let y=s?q(s):void 0;return{title:e,type:t,"sub-type":n,placeholder:r,description:i,"is-required":a,"reference-id":o,"display-logic":y&&Object.keys(y).length>0?{action:y.action,rule:y.rule?{"depends-on-question-reference-id":y.rule.dependsOnQuestionReferenceId,"comparison-operator":y.rule.comparisonOperator,"comparison-operands":`comparisonOperands`in y.rule?y.rule.comparisonOperands:void 0}:void 0,"nested-rule":y.nestedRule?{"logic-operator":y.nestedRule.logicOperator,rules:(y.nestedRule.rules||[]).map(e=>({"depends-on-question-reference-id":e.dependsOnQuestionReferenceId,"comparison-operator":e.comparisonOperator,"comparison-operands":`comparisonOperands`in e?e.comparisonOperands:void 0}))}:void 0}:void 0,"risk-logic":c.map(e=>{let t=Y(e);return{"risk-level":t.riskAssignment?.riskLevelId,"risk-matrix-row":t.riskAssignment?.riskMatrixRowId,"risk-matrix-column":t.riskAssignment?.riskMatrixColumnId,"comparison-operands":t.comparisonOperands,"comparison-operator":t.comparisonOperator}}),"risk-categories":l.map(({title:e})=>e),"risk-framework":u?.title,"answer-options":d.map(({value:e})=>({value:e})),"allowed-mime-types":f,"allow-select-other":p,"sync-model":m||void 0,"sync-column":h||void 0,"attribute-key":g?.name,"require-risk-evaluation":_,"require-risk-matrix-evaluation":v}})})),status:r,source:i,creator:a?.email,locked:o,archived:s,"created-at":c||void 0,"retention-schedule":l?{type:l.type,"duration-days":l.durationDays,operand:l.operation}:void 0}))),Ue.length>0&&E.includes(e.SystemDiscovery)&&(Q[`system-discovery`]=Ue.map(({title:e,resourceId:t,suggestedCatalog:{title:n},plugin:{dataSilo:{title:r}},country:i,countrySubDivision:a,plaintextContext:o,containsSensitiveData:s,status:c})=>({title:e,resourceId:t,suggestedCatalog:n,plugin:r,country:i||void 0,countrySubDivision:a||void 0,plaintextContext:o,containsSensitiveData:s,status:c}))),Te.length>0&&E.includes(e.Prompts)&&(Q.prompts=Te.map(({title:e,content:t})=>({title:e,content:t}))),Ee.length>0&&E.includes(e.PromptPartials)&&(Q[`prompt-partials`]=Ee.map(({title:e,content:t})=>({title:e,content:t}))),De.length>0&&E.includes(e.PromptGroups)&&(Q[`prompt-groups`]=De.map(({title:e,description:t,prompts:n})=>({title:e,description:t,prompts:n.map(({title:e})=>e)}))),Ie.length>0&&E.includes(e.Teams)&&(Q.teams=Ie.map(({name:e,description:t,ssoDepartment:n,ssoGroup:r,ssoTitle:i,users:a,scopes:o})=>({name:e,description:t,"sso-department":n||void 0,"sso-group":r||void 0,"sso-title":i||void 0,users:a.map(({email:e})=>e),scopes:o.map(({name:e})=>e)}))),M.length>0&&E.includes(e.DataSubjects)&&(Q[`data-subjects`]=M.map(({type:e,title:t,active:n,adminDashboardDefaultSilentMode:r,actions:i})=>({type:e,title:t?.defaultMessage,active:n,adminDashboardDefaultSilentMode:r,actions:i.map(({type:e})=>e)}))),Le.length>0&&(Q.policies=Le.map(({title:e,versions:t,disabledLocales:n})=>({title:e?.defaultMessage,content:t?.[0]?.content?.defaultMessage,disabledLocales:n}))),Re.length>0&&(Q.messages=Re.map(({id:e,defaultMessage:t,targetReactIntlId:n,description:r,translations:i})=>({id:e,defaultMessage:t,description:r,targetReactIntlId:n||void 0,translations:i.reduce((e,{locale:t,value:n})=>Object.assign(e,{[t]:n}),{})}))),X.length>0){let e=X[0];Q[`privacy-center`]={isDisabled:e.isDisabled,showPrivacyRequestButton:e.showPrivacyRequestButton,showPolicies:e.showPolicies,showTrackingTechnologies:e.showTrackingTechnologies,showCookies:e.showCookies,showDataFlows:e.showDataFlows,showConsentManager:e.showConsentManager,showManageYourPrivacy:e.showManageYourPrivacy,showMarketingPreferences:e.showMarketingPreferences,locales:e.locales,defaultLocale:e.defaultLocale,preferBrowserDefaultLocale:e.preferBrowserDefaultLocale,supportEmail:e.supportEmail||void 0,replyToEmail:e.replyToEmail||void 0,useNoReplyEmailAddress:e.useNoReplyEmailAddress,useCustomEmailDomain:e.useCustomEmailDomain,transformAccessReportJsonToCsv:e.transformAccessReportJsonToCsv,theme:e.theme}}if(G.length>0&&E.includes(e.BusinessEntities)&&(Q[`business-entities`]=G.map(({title:e,description:t,address:n,headquarterCountry:r,headquarterSubDivision:i,dataProtectionOfficerName:a,dataProtectionOfficerEmail:o,attributeValues:s})=>({title:e,description:t||void 0,address:n||void 0,headquarterCountry:r||void 0,headquarterSubDivision:i||void 0,dataProtectionOfficerName:a||void 0,dataProtectionOfficerEmail:o||void 0,attributes:s!==void 0&&s.length>0?V(s):void 0}))),K.length>0&&E.includes(e.ProcessingActivities)&&(Q[`processing-activities`]=K.map(({title:e,description:t,securityMeasureDetails:n,controllerships:r,storageRegions:i,transferRegions:a,retentionType:o,retentionPeriod:s,dataProtectionImpactAssessmentLink:c,dataProtectionImpactAssessmentStatus:l,attributeValues:u,dataSilos:d,dataSubjects:f,teams:p,owners:m,processingPurposeSubCategories:h,dataSubCategories:g,saaSCategories:_})=>({title:e,description:t,securityMeasureDetails:n??void 0,controllerships:r.length>0?r:void 0,storageRegions:i.length>0?Se(i):void 0,transferRegions:a.length>0?Se(a):void 0,retentionType:o,retentionPeriod:o===pe.StatedPeriod?s:void 0,dataProtectionImpactAssessmentLink:c??void 0,dataProtectionImpactAssessmentStatus:l,attributes:u!==void 0&&u.length>0?V(u):void 0,dataSiloTitles:d.length>0?d.map(({title:e})=>e):void 0,dataSubjectTypes:f.length>0?f.map(({type:e})=>e):void 0,teamNames:p.length>0?p.map(({name:e})=>e):void 0,ownerEmails:m.length>0?m.map(({email:e})=>e):void 0,processingSubPurposes:h.length>0?h.map(({name:e,purpose:t})=>({purpose:t,...e?{name:e}:{}})):void 0,dataSubCategories:g.length>0?g.map(({name:e,category:t})=>({category:t,...e?{name:e}:{}})):void 0,saaSCategories:_.length>0?_.map(({title:e})=>e):void 0}))),W.length>0&&E.includes(e.Actions)&&(Q.actions=W.map(({type:e,skipSecondaryIfNoFiles:t,skipDownloadableStep:n,requiresReview:r,regionList:i,regionDetectionMethod:a,waitingPeriod:o})=>({type:e,...e===fe.Erasure?{skipSecondaryIfNoFiles:t,skipDownloadableStep:n}:{},requiresReview:r,waitingPeriod:o,regionDetectionMethod:a,regionList:i.length>0?i:void 0}))),U.length>0&&E.includes(e.Identifiers)&&(Q.identifiers=U.map(({name:e,type:t,regex:n,selectOptions:r,privacyCenterVisibility:i,isRequiredInForm:a,placeholder:o,displayTitle:s,dataSubjects:c,displayDescription:l,displayOrder:u,isUniqueOnPreferenceStore:d})=>({name:e,type:t,regex:n,selectOptions:r.length>0?r:void 0,privacyCenterVisibility:i.length>0?i:void 0,isRequiredInForm:a,placeholder:o||void 0,dataSubjects:c.length>0?c.map(({type:e})=>e):void 0,displayTitle:s?.defaultMessage,displayDescription:l?.defaultMessage,displayOrder:u,isUniqueOnPreferenceStore:d}))),Oe.length>0&&E.includes(e.Agents)&&(Q.agents=Oe.map(({name:e,agentId:t,description:n,instructions:r,codeInterpreterEnabled:i,retrievalEnabled:a,prompt:o,largeLanguageModel:s,teams:c,owners:l,agentFunctions:u,agentFiles:d})=>({name:e,agentId:t,description:n||void 0,instructions:r,codeInterpreterEnabled:i,retrievalEnabled:a,prompt:o?.title,"large-language-model":{name:s.name,client:s.client},teams:c&&c.length>0?c.map(({name:e})=>e):void 0,owners:l&&l.length>0?l.map(({email:e})=>e):void 0,"agent-functions":u&&u.length>0?u.map(({name:e})=>e):void 0,"agent-files":d&&d.length>0?d.map(({name:e})=>e):void 0}))),Pe.length>0&&E.includes(e.ActionItems)&&(Q[`action-items`]=Pe.map(({teams:e,users:t,customerExperienceActionItemIds:[n],dueDate:r,priority:i,resolved:a,collections:o,notes:s,link:c,title:l,type:u,attributeValues:d})=>({teams:e.map(({name:e})=>e),users:t.map(({email:e})=>e),dueDate:r||void 0,title:l,notes:s,customerExperienceActionItemId:n,collections:o.map(({title:e})=>e),link:c,priority:i||void 0,resolved:a,type:u,attributes:d!==void 0&&d.length>0?V(d):void 0}))),Fe.length>0&&E.includes(e.ActionItemCollections)&&(Q[`action-item-collections`]=Fe.map(({title:e,description:t,hidden:n,productLine:r})=>({title:e,description:t||void 0,hidden:n,productLine:r}))),ke.length>0&&E.includes(e.AgentFunctions)&&(Q[`agent-functions`]=ke.map(({name:e,description:t,parameters:n})=>({name:e,description:t,parameters:JSON.stringify(n)}))),Ae.length>0&&E.includes(e.AgentFiles)&&(Q[`agent-files`]=Ae.map(({name:e,description:t,fileId:n,size:r,purpose:i})=>({name:e,description:t,fileId:n,size:r,purpose:i}))),je.length>0&&E.includes(e.Vendors)&&(Q.vendors=je.map(({title:e,description:t,dataProcessingAgreementLink:n,contactName:r,contactPhone:i,address:a,headquarterCountry:o,headquarterSubDivision:s,websiteUrl:c,businessEntity:l,teams:u,owners:d,attributeValues:f})=>({title:e,description:t||void 0,dataProcessingAgreementLink:n||void 0,contactName:r||void 0,contactPhone:i||void 0,address:a||void 0,headquarterCountry:o||void 0,headquarterSubDivision:s||void 0,websiteUrl:c||void 0,businessEntity:l?.title,teams:u&&u.length>0?u.map(({name:e})=>e):void 0,owners:d&&d.length>0?d.map(({email:e})=>e):void 0,attributes:f!==void 0&&f.length>0?V(f):void 0}))),Me.length>0&&E.includes(e.DataCategories)&&(Q[`data-categories`]=Me.map(({name:e,category:t,description:n,regex:r,owners:i,teams:a,attributeValues:o})=>({name:e,category:t,description:n||void 0,regex:r||void 0,owners:i&&i.length>0?i.map(({email:e})=>e):void 0,teams:a&&a.length>0?a.map(({name:e})=>e):void 0,attributes:o!==void 0&&o.length>0?V(o):void 0}))),Ne.length>0&&E.includes(e.ProcessingPurposes)&&(Q[`processing-purposes`]=Ne.map(({name:e,purpose:t,description:n,owners:r,teams:i,attributeValues:a})=>({name:e,purpose:t,description:n||void 0,owners:r&&r.length>0?r.map(({email:e})=>e):void 0,teams:i&&i.length>0?i.map(({name:e})=>e):void 0,attributes:a!==void 0&&a.length>0?V(a):void 0}))),I.length>0&&E.includes(e.DataFlows)&&(Q[`data-flows`]=I.map(({value:e,type:t,description:n,trackingType:r,service:i,status:a,owners:o,teams:s,attributeValues:c})=>({value:e,type:t,description:n||void 0,trackingPurposes:r,status:a,service:i?.integrationName,owners:o.map(({email:e})=>e),teams:s.map(({name:e})=>e),attributes:c!==void 0&&c.length>0?V(c):void 0}))),L.length>0&&E.includes(e.Cookies)&&(Q.cookies=L.map(({name:e,isRegex:t,description:n,trackingPurposes:r,service:i,status:a,owners:o,teams:s,attributeValues:c})=>({name:e,isRegex:t,description:n||void 0,trackingPurposes:r,status:a,service:i?.integrationName,owners:o.map(({email:e})=>e),teams:s.map(({name:e})=>e),attributes:c!==void 0&&c.length>0?V(c):void 0}))),R.length>0&&E.includes(e.Attributes)&&(Q.attributes=R.map(({description:e,name:t,type:n,values:r,enabledOn:i=[]})=>({description:e||void 0,resources:i,name:t,type:n,values:r.map(({name:e,color:t,description:n})=>({name:e,color:t||void 0,description:n}))}))),He.length>0&&(Q.purposes=He.map(({name:e,description:t,trackingType:n,defaultConsent:r,configurable:i,showInConsentManager:a,isActive:o,displayOrder:s,optOutSignals:c,authLevel:l,topics:u,showInPrivacyCenter:d,title:f})=>({name:e,title:f,description:t||void 0,trackingType:n,"default-consent":r,configurable:i,"show-in-consent-manager":a,"show-in-privacy-center":d,"is-active":o,"display-order":s,"opt-out-signals":c.length>0?c:void 0,"auth-level":l||void 0,"preference-topics":u.map(({title:e,type:t,displayDescription:n,defaultConfiguration:r,showInPrivacyCenter:i,preferenceOptionValues:a})=>({title:e.defaultMessage,type:t,description:n.defaultMessage,"default-configuration":r,"show-in-privacy-center":i,...a.length>0?{options:a.map(({title:e,slug:t})=>({title:e.defaultMessage,slug:t}))}:{}}))}))),s.length===0&&H.length>0&&E.includes(e.Templates)&&(Q.templates=H.map(({title:e})=>({title:e}))),F.length>0&&E.includes(e.Enrichers)&&(Q.enrichers=F.map(({title:e,url:t,type:n,inputIdentifier:r,identifiers:i,actions:a,testRegex:o,dataSubjects:s,expirationDuration:c,lookerQueryTitle:l,transitionRequestStatus:u,phoneNumbers:d,regionList:f})=>({title:e,url:t||void 0,type:n,"input-identifier":r?.name,"output-identifiers":i.map(({name:e})=>e),"privacy-actions":Object.values(fe).length===a.length?void 0:a,testRegex:o||void 0,lookerQueryTitle:l||void 0,expirationDuration:parseInt(c,10),transitionRequestStatus:u||void 0,phoneNumbers:d&&d.length>0?d:void 0,regionList:f&&f.length>0?f:void 0,"data-subjects":s.map(({type:e})=>e)}))),P.length>0&&E.includes(e.DataSilos)){let e=he(M,`type`);Q[`data-silos`]=P.map(([{title:t,description:n,url:r,type:i,outerType:a,apiKeys:o,notifyEmailAddress:s,identifiers:c,dependentDataSilos:l,owners:u,country:d,countrySubDivision:f,teams:p,subjectBlocklist:m,isLive:h,promptAVendorEmailSendFrequency:_,promptAVendorEmailSendType:v,promptAVendorEmailIncludeIdentifiersAttachment:y,promptAVendorEmailCompletionLinkType:b,manualWorkRetryFrequency:x,catalog:S,attributeValues:C,discoveredBy:w,businessEntities:T},ee])=>({title:t,description:n,integrationName:i,"outer-type":a||void 0,url:r||void 0,"api-key-title":o[0]?.title,"identity-keys":c.filter(({isConnected:e})=>e).map(({name:e})=>e),...l.length>0?{"deletion-dependencies":l.map(({title:e})=>e)}:{},...u.length>0?{owners:u.map(({email:e})=>e)}:{},...p.length>0?{teams:p.map(({name:e})=>e)}:{},...w.length>0?{discoveredBy:w.map(({title:e})=>e)}:{},...T.length>0?{businessEntities:T.map(({title:e})=>e)}:{},country:d||void 0,countrySubDivision:f||void 0,disabled:!h,"data-subjects":m.length>0?g(m.map(({type:e})=>e),e):void 0,...S.hasAvcFunctionality?{"email-settings":{"notify-email-address":s||void 0,"send-frequency":_,"send-type":v,"include-identifiers-attachment":y,"completion-link-type":b,"manual-work-retry-frequency":x}}:{},attributes:C!==void 0&&C.length>0?V(C):void 0,datapoints:ee.map(e=>({key:e.name,title:e.title?.defaultMessage,description:e.description?.defaultMessage,owners:e.owners.map(({email:e})=>e),teams:e.teams.map(({name:e})=>e),...e.path.length>0?{path:e.path}:{},...e.dataCollection?.title?{"data-collection-tag":e.dataCollection.title.defaultMessage}:{},...e.dbIntegrationQueries.length>0?{"privacy-action-queries":ge(he(e.dbIntegrationQueries,`requestType`),e=>e.suggestedQuery||e.query||void 0)}:{},...e.subDataPoints.length>0?{fields:e.subDataPoints.map(e=>({key:e.name,description:e.description,purposes:e.purposes,categories:e.categories,...k&&e.pendingCategoryGuesses?{"guessed-categories":e.pendingCategoryGuesses.filter(e=>e.status===`PENDING`).map(e=>({category:{name:e.category.name,category:e.category.category},status:e.status,confidence:e.confidence,classifierVersion:e.classifierVersion||void 0}))}:{},"access-request-visibility-enabled":e.accessRequestVisibilityEnabled,"erasure-request-redaction-enabled":e.erasureRequestRedactionEnabled,attributes:e.attributeValues!==void 0&&e.attributeValues.length>0?V(e.attributeValues):void 0})).sort((e,t)=>e.key.localeCompare(t.key))}:{},"privacy-actions":e.actionSettings.filter(({active:e})=>e).map(({type:e})=>e)})).sort((e,t)=>[...e.path??[],e.key].join(`.`).localeCompare([...t.path??[],t.key].join(`.`)))}))}return Q}export{z as _,Y as a,W as c,q as d,be as f,ve as g,B as h,J as i,H as l,ye as m,we as n,K as o,V as p,xe as r,G as s,Ce as t,U as u,R as v,L as y};
-//# sourceMappingURL=pullTranscendConfiguration-D2cYlu6V.mjs.map
+//# sourceMappingURL=pullTranscendConfiguration-CqsgEf9A.mjs.map