@transcend-io/cli 10.2.3 → 10.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transcend-io/cli",
-  "version": "10.2.3",
+  "version": "10.2.5",
   "description": "A command line interface for programmatic operations across Transcend.",
   "homepage": "https://github.com/transcend-io/tools/tree/main/packages/cli",
   "license": "Apache-2.0",
@@ -106,9 +106,9 @@
     "yargs-parser": "^21.1.1",
     "@transcend-io/internationalization": "4.0.1",
     "@transcend-io/privacy-types": "5.1.3",
-    "@transcend-io/sdk": "1.0.3",
+    "@transcend-io/sdk": "1.1.1",
     "@transcend-io/type-utils": "3.0.1",
-    "@transcend-io/utils": "0.1.2"
+    "@transcend-io/utils": "0.1.3"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.18.2",

package/dist/impl-fZQxhZRu.mjs

@@ -1,12 +0,0 @@
-import{t as e}from"./logger-Bj782ZYD.mjs";import{t}from"./readCsv-C4TyEs-r.mjs";import{a as n}from"./writeCsv-C4pjXGsD.mjs";import{t as r}from"./done-input-validation-BcNBxhEs.mjs";import{chunk as i}from"lodash-es";import{decodeCodec as a}from"@transcend-io/type-utils";import{readdirSync as o}from"node:fs";import{join as s}from"node:path";import c from"colors";import{DeletePreferenceRecordCliCsvRow as l,DeletePreferenceRecordsResponse as u,createSombraGotInstance as d,withPreferenceRetry as f}from"@transcend-io/sdk";import{map as p}from"@transcend-io/utils";import m from"cli-progress";async function h(t,{partition:n,identifierChunk:r,timestamp:i}){try{let{failures:o}=a(u,await f(`Delete Preference Records`,()=>t.post(`v1/preferences/${n}/delete`,{json:{records:r.map(e=>({anchorIdentifier:e,timestamp:i.toISOString()}))}}).json(),{logger:e,maxAttempts:3,onRetry:(t,n,r)=>{e.warn(c.yellow(`Attempt ${t} to delete preference records failed: ${r}`))}}));return o.length>0?o.map(({index:e,error:t})=>({...r[e],error:t})):[]}catch(e){return r.map(t=>({...t,error:e.message}))}}async function g(e,{partition:n,filePath:r,timestamp:a,maxItemsInChunk:o,maxConcurrency:s}){return(await p(i(t(r,l),o),async t=>await h(e,{partition:n,identifierChunk:t,timestamp:a}),{concurrency:s})).flat()}async function _({auth:t,partition:i,sombraAuth:a,file:l=``,directory:u,transcendUrl:f,timestamp:h,maxConcurrency:_,maxItemsInChunk:v,receiptDirectory:y,fileConcurrency:b}){u&&l&&(e.error(c.red(`Cannot provide both a directory and a file. Please provide only one.`)),this.process.exit(1)),!l&&!u&&(e.error(c.red(`A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences`)),this.process.exit(1)),r(this.process.exit);let x=[];if(u)try{let t=o(u).filter(e=>e.endsWith(`.csv`));t.length===0&&(e.error(c.red(`No CSV files found in directory: ${u}`)),this.process.exit(1)),x.push(...t.map(e=>s(u,e)))}catch(t){e.error(c.red(`Failed to read directory: ${u}`)),e.error(c.red(t.message)),this.process.exit(1)}else try{l.endsWith(`.csv`)||(e.error(c.red(`File must be a CSV file`)),this.process.exit(1)),x.push(l)}catch(t){e.error(c.red(`Failed to access file: ${l}`)),e.error(c.red(t.message)),this.process.exit(1)}e.debug(c.green(`Processing ${x.length} consent preferences files for partition: ${i}`)),e.debug(`\nFiles to process: ${x.join(`, `)}\n`);let S=await d(f,t,{logger:e,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),C=new m.SingleBar({format:`Deletion Progress |${c.cyan(`[{bar}]`)}| Duration: ${c.red(`{duration_formatted}`)} | {value}/{total} Files Processed `},m.Presets.shades_classic);C.start(x.length,0);let w=await p(x,async e=>{let t=await g(S,{partition:i,filePath:e,timestamp:h,maxItemsInChunk:v,maxConcurrency:_});return C.increment(),t},{concurrency:b});C.stop();let T=w.flat(),E=``;T.length>0&&(E=s(y,`deletion-failures-${Date.now()}.csv`),n(E,T,!0)),e.info(c.green(`
-
- ================================== 
-
-`)),e.info(c.green(`
-#### Deletion Summary Report #####
-`)),e.info(c.green(`📁 Total Files Processed: ${x.length} \n❌ Errors: ${T.length} \n📝 Receipt Path: ${E||`N/A`}`)),e.info(c.green(`
-
-==================================
-
-`))}export{_ as deletePreferenceRecords};
-//# sourceMappingURL=impl-fZQxhZRu.mjs.map

package/dist/impl-fZQxhZRu.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"impl-fZQxhZRu.mjs","names":[],"sources":["../src/lib/preference-management/bulkDeletePreferenceRecords.ts","../src/commands/consent/delete-preference-records/impl.ts"],"sourcesContent":["import {\n  DeletePreferenceRecordCliCsvRow,\n  DeletePreferenceRecordsResponse,\n  withPreferenceRetry,\n} from '@transcend-io/sdk';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport type { Got } from 'got';\nimport { chunk } from 'lodash-es';\n\nimport { logger } from '../../logger.js';\nimport { readCsv } from '../requests/index.js';\n\ninterface FailedResult extends DeletePreferenceRecordCliCsvRow {\n  /** Error message describing the failure */\n  error: string;\n}\n\ninterface DeletePreferenceRecordsRepositoryOptions {\n  /** The partition to delete from */\n  partition: string;\n  /** Chunk of identifiers to delete */\n  identifierChunk: DeletePreferenceRecordCliCsvRow[];\n  /** the timestamp for the deletion operation */\n  timestamp: Date;\n}\n\n/**\n * Options for deleting preference records\n */\ntype DeletePreferenceRecordsOptions = Omit<\n  DeletePreferenceRecordsRepositoryOptions,\n  'identifierChunk'\n> & {\n  /** The file path to read CSV rows from */\n  filePath: string;\n  /** Maximum items to include in each deletion chunk */\n  maxItemsInChunk: number;\n  /** Maximum concurrency for deletion requests */\n  maxConcurrency: number;\n};\n\n/**\n *\n * Delete a chunk of preference records\n *\n * @param sombra - Sombra instance (must include auth headers)\n * @param options - Options for deletion\n * @param options.partition - The partition to delete from\n * @param options.identifierChunk - Chunk of identifiers to delete\n * @param options.timestamp - The timestamp for the deletion operation\n * @returns List of failed deletions\n */\nasync function deletePreferenceRecordsRepository(\n  sombra: Got,\n  { partition, identifierChunk: chunk, timestamp }: DeletePreferenceRecordsRepositoryOptions,\n): Promise<FailedResult[]> {\n  try {\n    const response = await withPreferenceRetry(\n      'Delete Preference Records',\n      () =>\n        sombra\n          .post(`v1/preferences/${partition}/delete`, {\n            json: {\n              records: chunk.map((record) => ({\n                anchorIdentifier: record,\n                timestamp: timestamp.toISOString(),\n              })),\n            },\n          })\n          .json(),\n      {\n        logger,\n        maxAttempts: 3,\n        onRetry: (attempt, _err, msg) => {\n          logger.warn(\n            colors.yellow(`Attempt ${attempt} to delete preference records failed: ${msg}`),\n          );\n        },\n      },\n    );\n    const { failures } = decodeCodec(DeletePreferenceRecordsResponse, response);\n    if (failures.length > 0) {\n      return failures.map(({ index, error }) => ({\n        ...chunk[index],\n        error,\n      }));\n    }\n    return [];\n  } catch (err) {\n    return chunk.map((record) => ({\n      ...record,\n      error: (err as Error).message,\n    }));\n  }\n}\n\n/**\n * Delete consent preferences for the managed consent database (delete endpoint)\n *\n * Uses POST /v1/preferences/{partition}/delete.\n *\n *\n * @param sombra - Sombra instance (must include auth headers)\n * @param options - Query options\n * @returns All nodes (only when onItems is not provided)\n */\nexport async function bulkDeletePreferenceRecords(\n  sombra: Got,\n  {\n    partition,\n    filePath,\n    timestamp,\n    maxItemsInChunk,\n    maxConcurrency,\n  }: DeletePreferenceRecordsOptions,\n): Promise<FailedResult[]> {\n  const anchorIdentifiers = readCsv(filePath, DeletePreferenceRecordCliCsvRow);\n  const chunks = chunk(anchorIdentifiers, maxItemsInChunk);\n\n  const failedResults = await map(\n    chunks,\n    async (identifierChunk) => {\n      const failedResults = await deletePreferenceRecordsRepository(sombra, {\n        partition,\n        identifierChunk,\n        timestamp,\n      });\n      return failedResults;\n    },\n    { concurrency: maxConcurrency },\n  );\n  return failedResults.flat();\n}\n","import { readdirSync } from 'node:fs';\nimport { join } from 'node:path';\n\nimport { createSombraGotInstance } from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport type { LocalContext } from '../../../context.js';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation.js';\nimport { writeCsv } from '../../../lib/helpers/index.js';\nimport { bulkDeletePreferenceRecords } from '../../../lib/preference-management/index.js';\nimport { logger } from '../../../logger.js';\n\nexport interface DeletePreferenceRecordsCommandFlags {\n  /** Transcend API key for authentication */\n  auth: string;\n  /** Partition ID to delete preference records from */\n  partition: string;\n  /** Optional Sombra internal key for self-hosted instances */\n  sombraAuth?: string;\n  /** Path to the CSV file used to identify preference records to delete */\n  file?: string;\n  /** Path to the directory of CSV files to load preferences from */\n  directory?: string;\n  /** Base URL for the Transcend API */\n  transcendUrl: string;\n  /** The timestamp when the deletion operation is made. Used for logging purposes. */\n  timestamp: Date;\n  /** Maximum items to include in each deletion chunk */\n  maxItemsInChunk: number;\n  /** Maximum concurrency for deletion requests */\n  maxConcurrency: number;\n  /** Directory to write receipts of failed deletions to */\n  receiptDirectory: string;\n  /** Number of files to process concurrently when deleting preference records from multiple files */\n  fileConcurrency: number;\n}\n\nexport async function deletePreferenceRecords(\n  this: LocalContext,\n  {\n    auth,\n    partition,\n    sombraAuth,\n    file = '',\n    directory,\n    transcendUrl,\n    timestamp,\n    maxConcurrency,\n    maxItemsInChunk,\n    receiptDirectory,\n    fileConcurrency,\n  }: DeletePreferenceRecordsCommandFlags,\n): Promise<void> {\n  if (!!directory && !!file) {\n    logger.error(\n      colors.red('Cannot provide both a directory and a file. Please provide only one.'),\n    );\n    this.process.exit(1);\n  }\n\n  if (!file && !directory) {\n    logger.error(\n      colors.red(\n        'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences',\n      ),\n    );\n    this.process.exit(1);\n  }\n  doneInputValidation(this.process.exit);\n\n  const files: string[] = [];\n\n  if (directory) {\n    try {\n      const filesInDirectory = readdirSync(directory);\n      const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv'));\n\n      if (csvFiles.length === 0) {\n        logger.error(colors.red(`No CSV files found in directory: ${directory}`));\n        this.process.exit(1);\n      }\n\n      // Add full paths for each CSV file\n      files.push(...csvFiles.map((file) => join(directory, file)));\n    } catch (err) {\n      logger.error(colors.red(`Failed to read directory: ${directory}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  } else {\n    try {\n      // Verify file exists and is a CSV\n      if (!file.endsWith('.csv')) {\n        logger.error(colors.red('File must be a CSV file'));\n        this.process.exit(1);\n      }\n      files.push(file);\n    } catch (err) {\n      logger.error(colors.red(`Failed to access file: ${file}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  }\n\n  logger.debug(\n    colors.green(\n      `Processing ${files.length} consent preferences files for partition: ${partition}`,\n    ),\n  );\n  logger.debug(`\\nFiles to process: ${files.join(', ')}\\n`);\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n  const globalProgressBar = new cliProgress.SingleBar(\n    {\n      format: `Deletion Progress |${colors.cyan('[{bar}]')}| Duration: ${colors.red(\n        '{duration_formatted}',\n      )} | {value}/{total} Files Processed `,\n    },\n    cliProgress.Presets.shades_classic,\n  );\n  globalProgressBar.start(files.length, 0);\n\n  // Process batch of files with concurrency\n  const failedResultsArrays = await map(\n    files,\n    async (filePath) => {\n      const result = await bulkDeletePreferenceRecords(sombra, {\n        partition,\n        filePath,\n        timestamp,\n        maxItemsInChunk,\n        maxConcurrency,\n      });\n      globalProgressBar.increment();\n      return result;\n    },\n    { concurrency: fileConcurrency },\n  );\n  globalProgressBar.stop();\n  const failedResults = failedResultsArrays.flat();\n\n  // Check for failed results and write receipt if any\n  let receiptPath = '';\n  if (failedResults.length > 0) {\n    receiptPath = join(receiptDirectory, `deletion-failures-${Date.now()}.csv`);\n    writeCsv(receiptPath, failedResults, true);\n  }\n\n  logger.info(colors.green('\\n\\n ================================== \\n\\n'));\n  logger.info(colors.green('\\n#### Deletion Summary Report #####\\n'));\n  logger.info(\n    colors.green(\n      `📁 Total Files Processed: ${files.length} \\n` +\n        `❌ Errors: ${failedResults.length} \\n` +\n        `📝 Receipt Path: ${receiptPath || 'N/A'}`,\n    ),\n  );\n  logger.info(colors.green('\\n\\n==================================\\n\\n'));\n}\n"],"mappings":"glBAsDA,eAAe,EACb,EACA,CAAE,YAAW,gBAAiB,EAAO,aACZ,CACzB,GAAI,CAwBF,GAAM,CAAE,YAAa,EAAY,EAvBhB,MAAM,EACrB,gCAEE,EACG,KAAK,kBAAkB,EAAU,SAAU,CAC1C,KAAM,CACJ,QAAS,EAAM,IAAK,IAAY,CAC9B,iBAAkB,EAClB,UAAW,EAAU,aAAa,CACnC,EAAE,CACJ,CACF,CAAC,CACD,MAAM,CACX,CACE,SACA,YAAa,EACb,SAAU,EAAS,EAAM,IAAQ,CAC/B,EAAO,KACL,EAAO,OAAO,WAAW,EAAQ,wCAAwC,IAAM,CAChF,EAEJ,CACF,CAC0E,CAO3E,OANI,EAAS,OAAS,EACb,EAAS,KAAK,CAAE,QAAO,YAAa,CACzC,GAAG,EAAM,GACT,QACD,EAAE,CAEE,EAAE,OACF,EAAK,CACZ,OAAO,EAAM,IAAK,IAAY,CAC5B,GAAG,EACH,MAAQ,EAAc,QACvB,EAAE,EAcP,eAAsB,EACpB,EACA,CACE,YACA,WACA,YACA,kBACA,kBAEuB,CAgBzB,OAZsB,MAAM,EAFb,EADW,EAAQ,EAAU,EAAgC,CACpC,EAAgB,CAItD,KAAO,IACiB,MAAM,EAAkC,EAAQ,CACpE,YACA,kBACA,YACD,CAAC,CAGJ,CAAE,YAAa,EAAgB,CAChC,EACoB,MAAM,CC9F7B,eAAsB,EAEpB,CACE,OACA,YACA,aACA,OAAO,GACP,YACA,eACA,YACA,iBACA,kBACA,mBACA,mBAEa,CACT,GAAe,IACnB,EAAO,MACL,EAAO,IAAI,uEAAuE,CACnF,CACD,KAAK,QAAQ,KAAK,EAAE,EAGlB,CAAC,GAAQ,CAAC,IACZ,EAAO,MACL,EAAO,IACL,uHACD,CACF,CACD,KAAK,QAAQ,KAAK,EAAE,EAEtB,EAAoB,KAAK,QAAQ,KAAK,CAEtC,IAAM,EAAkB,EAAE,CAE1B,GAAI,EACF,GAAI,CAEF,IAAM,EADmB,EAAY,EAAU,CACb,OAAQ,GAAS,EAAK,SAAS,OAAO,CAAC,CAErE,EAAS,SAAW,IACtB,EAAO,MAAM,EAAO,IAAI,oCAAoC,IAAY,CAAC,CACzE,KAAK,QAAQ,KAAK,EAAE,EAItB,EAAM,KAAK,GAAG,EAAS,IAAK,GAAS,EAAK,EAAW,EAAK,CAAC,CAAC,OACrD,EAAK,CACZ,EAAO,MAAM,EAAO,IAAI,6BAA6B,IAAY,CAAC,CAClE,EAAO,MAAM,EAAO,IAAK,EAAc,QAAQ,CAAC,CAChD,KAAK,QAAQ,KAAK,EAAE,MAGtB,GAAI,CAEG,EAAK,SAAS,OAAO,GACxB,EAAO,MAAM,EAAO,IAAI,0BAA0B,CAAC,CACnD,KAAK,QAAQ,KAAK,EAAE,EAEtB,EAAM,KAAK,EAAK,OACT,EAAK,CACZ,EAAO,MAAM,EAAO,IAAI,0BAA0B,IAAO,CAAC,CAC1D,EAAO,MAAM,EAAO,IAAK,EAAc,QAAQ,CAAC,CAChD,KAAK,QAAQ,KAAK,EAAE,CAIxB,EAAO,MACL,EAAO,MACL,cAAc,EAAM,OAAO,4CAA4C,IACxE,CACF,CACD,EAAO,MAAM,uBAAuB,EAAM,KAAK,KAAK,CAAC,IAAI,CAGzD,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CACI,EAAoB,IAAI,EAAY,UACxC,CACE,OAAQ,sBAAsB,EAAO,KAAK,UAAU,CAAC,cAAc,EAAO,IACxE,uBACD,CAAC,qCACH,CACD,EAAY,QAAQ,eACrB,CACD,EAAkB,MAAM,EAAM,OAAQ,EAAE,CAGxC,IAAM,EAAsB,MAAM,EAChC,EACA,KAAO,IAAa,CAClB,IAAM,EAAS,MAAM,EAA4B,EAAQ,CACvD,YACA,WACA,YACA,kBACA,iBACD,CAAC,CAEF,OADA,EAAkB,WAAW,CACtB,GAET,CAAE,YAAa,EAAiB,CACjC,CACD,EAAkB,MAAM,CACxB,IAAM,EAAgB,EAAoB,MAAM,CAG5C,EAAc,GACd,EAAc,OAAS,IACzB,EAAc,EAAK,EAAkB,qBAAqB,KAAK,KAAK,CAAC,MAAM,CAC3E,EAAS,EAAa,EAAe,GAAK,EAG5C,EAAO,KAAK,EAAO,MAAM;;;;EAA+C,CAAC,CACzE,EAAO,KAAK,EAAO,MAAM;;EAAyC,CAAC,CACnE,EAAO,KACL,EAAO,MACL,6BAA6B,EAAM,OAAO,eAC3B,EAAc,OAAO,sBACd,GAAe,QACtC,CACF,CACD,EAAO,KAAK,EAAO,MAAM;;;;EAA6C,CAAC"}

package/dist/pullChunkedCustomSiloOutstandingIdentifiers-DRKraWKp.mjs

@@ -1,2 +0,0 @@
-import{a as e}from"./constants-3jGxMMU0.mjs";import{t}from"./logger-Bj782ZYD.mjs";import{decodeCodec as n}from"@transcend-io/type-utils";import r from"colors";import*as i from"io-ts";import{REDUCED_REQUESTS_FOR_DATA_SILO_COUNT as a,buildTranscendGraphQLClient as o,createSombraGotInstance as s,makeGraphQLRequest as c}from"@transcend-io/sdk";import{mapSeries as l}from"@transcend-io/utils";import u from"cli-progress";async function d(e,{dataSiloId:n}){let{listReducedRequestsForDataSilo:{totalCount:r}}=await c(e,a,{variables:{input:{dataSiloId:n,isResolved:!1}},logger:t});return r}const f=i.type({identifier:i.string,type:i.string,coreIdentifier:i.string,dataSiloId:i.string,requestId:i.string,nonce:i.string,requestCreatedAt:i.string,daysUntilOverdue:i.number,attributes:i.array(i.type({key:i.string,values:i.array(i.string)}))});async function p(e,{dataSiloId:t,limit:r=100,offset:a=0,requestType:o}){try{let s=await e.get(`v1/data-silo/${t}/pending-requests/${o}`,{searchParams:{offset:a,limit:r}}).json(),{items:c}=n(i.type({items:i.array(f)}),s);return c}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function m({dataSiloId:n,auth:i,sombraAuth:a,actions:c,apiPageSize:f=100,savePageSize:m=1e3,onSave:h,transcendUrl:g=e,skipRequestCount:_=!1}){if(m%f!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${m}, apiPageSize: ${f}`);let v=await s(g,i,{logger:t,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),y=o(g,i),b=0;_||(b=await d(y,{dataSiloId:n})),t.info(r.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${n}" for requests of types "${c.join(`", "`)}"`));let x=new Date().getTime(),S=new u.SingleBar({},u.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await l(c,async e=>{let i=0,a=!0;for(;a;){let o=await p(v,{dataSiloId:n,limit:f,offset:i,requestType:e}),s=o.map(t=>(C.add(t.requestId),{...t,action:e})),c=s.map(({attributes:e,...t})=>({...t,...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...s),T.push(...c),T.length>=m&&(await h(T),T=[]),a=o.length===f,i+=f,_?t.info(r.magenta(`Pulled ${o.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await h(T),_||S.stop();let E=new Date().getTime()-x;return t.info(r.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{d as i,f as n,p as r,m as t};
-//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-DRKraWKp.mjs.map

package/dist/pullChunkedCustomSiloOutstandingIdentifiers-DRKraWKp.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-DRKraWKp.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { makeGraphQLRequest, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from '@transcend-io/sdk';\nimport { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/**\n * Get number of open requests for a data silo\n *\n * @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n */\nexport async function fetchRequestDataSiloActiveCount(\n  client: GraphQLClient,\n  {\n    dataSiloId,\n  }: {\n    /** Data silo ID */\n    dataSiloId: string;\n  },\n): Promise<number> {\n  const {\n    listReducedRequestsForDataSilo: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Requests */\n    listReducedRequestsForDataSilo: {\n      /** Total count */\n      totalCount: number;\n    };\n  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n    variables: {\n      input: {\n        dataSiloId,\n        isResolved: false,\n      },\n    },\n    logger,\n  });\n\n  return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nexport const CronIdentifier = t.type({\n  /** The identifier value */\n  identifier: t.string,\n  /** The type of identifier */\n  type: t.string,\n  /** The core identifier of the request */\n  coreIdentifier: t.string,\n  /** The ID of the underlying data silo */\n  dataSiloId: t.string,\n  /** The ID of the underlying request */\n  requestId: t.string,\n  /** The request nonce */\n  nonce: t.string,\n  /** The time the request was created */\n  requestCreatedAt: t.string,\n  /** The number of days until the request is overdue */\n  daysUntilOverdue: t.number,\n  /** Request attributes */\n  attributes: t.array(\n    t.type({\n      key: t.string,\n      values: t.array(t.string),\n    }),\n  ),\n});\n\n/** Type override */\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/**\n * Pull a offset of identifiers for a cron job\n *\n * @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n */\nexport async function pullCronPageOfIdentifiers(\n  sombra: Got,\n  {\n    dataSiloId,\n    limit = 100,\n    offset = 0,\n    requestType,\n  }: {\n    /** Data Silo ID */\n    dataSiloId: string;\n    /** Type of request */\n    requestType: RequestAction;\n    /** Number of identifiers to pull in */\n    limit?: number;\n    /** Page to pull in */\n    offset?: number;\n  },\n): Promise<CronIdentifier[]> {\n  try {\n    // Make the GraphQL request\n    const response = await sombra\n      .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n        searchParams: {\n          offset,\n          limit,\n        },\n      })\n      .json();\n\n    const { items } = decodeCodec(\n      t.type({\n        items: t.array(CronIdentifier),\n      }),\n      response,\n    );\n    return items;\n  } catch (err) {\n    throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n  }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { buildTranscendGraphQLClient, createSombraGotInstance } from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchRequestDataSiloActiveCount } from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/**\n * A CSV formatted identifier\n */\nexport type CsvFormattedIdentifier = {\n  [k in string]: string | null | boolean | number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n  /** The request action that the identifier relates to */\n  action: RequestAction;\n}\n\n/**\n * Pull the set of identifiers outstanding for a cron or AVC integration\n *\n * This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n *\n * @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n */\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n  dataSiloId,\n  auth,\n  sombraAuth,\n  actions,\n  apiPageSize = 100,\n  savePageSize = 1000,\n  onSave,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  skipRequestCount = false,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Data Silo ID to pull down jobs for */\n  dataSiloId: string;\n  /** The request actions to fetch */\n  actions: RequestAction[];\n  /** How many identifiers to pull in a single call to the backend */\n  apiPageSize: number;\n  /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */\n  savePageSize: number;\n  /** Callback function called when a chunk of identifiers is ready to be saved */\n  onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Skip request count */\n  skipRequestCount?: boolean;\n}): Promise<{\n  /** Raw Identifiers */\n  identifiers: CronIdentifierWithAction[];\n}> {\n  // Validate savePageSize\n  if (savePageSize % apiPageSize !== 0) {\n    throw new Error(\n      `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n    );\n  }\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  let totalRequestCount = 0;\n  if (!skipRequestCount) {\n    totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n      dataSiloId,\n    });\n  }\n\n  logger.info(\n    colors.magenta(\n      `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n        `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const foundRequestIds = new Set<string>();\n\n  // identifiers found in total\n  const identifiers: CronIdentifierWithAction[] = [];\n  // current chunk of identifiers to be saved\n  let currentChunk: CsvFormattedIdentifier[] = [];\n\n  // map over each action\n  if (!skipRequestCount) {\n    progressBar.start(totalRequestCount, 0);\n  }\n  await mapSeries(actions, async (action) => {\n    let offset = 0;\n    let shouldContinue = true;\n\n    // Fetch a page of identifiers\n    while (shouldContinue) {\n      const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n        dataSiloId,\n        limit: apiPageSize,\n        offset,\n        requestType: action,\n      });\n\n      const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n        (identifier) => {\n          foundRequestIds.add(identifier.requestId);\n          return {\n            ...identifier,\n            action,\n          };\n        },\n      );\n\n      const csvFormattedIdentifiers = identifiersWithAction.map(\n        ({ attributes, ...identifier }) => ({\n          ...identifier,\n          ...attributes.reduce(\n            (acc, val) =>\n              Object.assign(acc, {\n                [val.key]: val.values.join(','),\n              }),\n            {},\n          ),\n        }),\n      );\n\n      identifiers.push(...identifiersWithAction);\n      currentChunk.push(...csvFormattedIdentifiers);\n\n      // Check if we've reached the savePageSize and call the onSave callback\n      if (currentChunk.length >= savePageSize) {\n        await onSave(currentChunk);\n        currentChunk = [];\n      }\n\n      shouldContinue = pageIdentifiers.length === apiPageSize;\n      offset += apiPageSize;\n      if (!skipRequestCount) {\n        progressBar.update(foundRequestIds.size);\n      } else {\n        logger.info(\n          colors.magenta(\n            `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n          ),\n        );\n      }\n    }\n  });\n\n  // Save any remaining identifiers in the current chunk\n  if (currentChunk.length > 0) {\n    await onSave(currentChunk);\n  }\n\n  if (!skipRequestCount) {\n    progressBar.stop();\n  }\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n        foundRequestIds.size\n      } requests in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n\n  return { identifiers };\n}\n"],"mappings":"kaAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,UAAW,CACT,MAAO,CACL,aACA,WAAY,GACb,CACF,CACD,SACD,CAAC,CAEF,OAAO,EClCT,MAAa,EAAiB,EAAE,KAAK,CAEnC,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAEF,IAAM,EAAW,MAAM,EACpB,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CAEH,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC/C5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGI,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,GAAG,MAAkB,CAClC,GAAG,EACH,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}