@transcend-io/cli 8.24.0 → 8.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/bash-complete.cjs +1 -1
- package/dist/bin/cli.cjs +1 -1
- package/dist/bin/deprecated-command.cjs +2 -2
- package/dist/{chunk-OKWH7Y2Z.cjs → chunk-AOO3U7W3.cjs} +3 -3
- package/dist/{chunk-OKWH7Y2Z.cjs.map → chunk-AOO3U7W3.cjs.map} +1 -1
- package/dist/{chunk-CYGC567Y.cjs → chunk-KKAJPI6O.cjs} +4 -4
- package/dist/{chunk-CYGC567Y.cjs.map → chunk-KKAJPI6O.cjs.map} +1 -1
- package/dist/{chunk-UJPJAASL.cjs → chunk-MWZABOIC.cjs} +21 -21
- package/dist/{chunk-UJPJAASL.cjs.map → chunk-MWZABOIC.cjs.map} +1 -1
- package/dist/{chunk-C5AAGXZE.cjs → chunk-NA6JTKNJ.cjs} +2 -2
- package/dist/{chunk-C5AAGXZE.cjs.map → chunk-NA6JTKNJ.cjs.map} +1 -1
- package/dist/{chunk-DCLHMZWV.cjs → chunk-NNHHVBQP.cjs} +4 -4
- package/dist/{chunk-DCLHMZWV.cjs.map → chunk-NNHHVBQP.cjs.map} +1 -1
- package/dist/{chunk-7IN5X2S4.cjs → chunk-O3EACWMC.cjs} +2 -2
- package/dist/{chunk-7IN5X2S4.cjs.map → chunk-O3EACWMC.cjs.map} +1 -1
- package/dist/{chunk-7A6YW7SF.cjs → chunk-T5QZTOZC.cjs} +2 -2
- package/dist/{chunk-7A6YW7SF.cjs.map → chunk-T5QZTOZC.cjs.map} +1 -1
- package/dist/{chunk-AVZBE2R3.cjs → chunk-TRQBFWZP.cjs} +2 -2
- package/dist/{chunk-AVZBE2R3.cjs.map → chunk-TRQBFWZP.cjs.map} +1 -1
- package/dist/{chunk-CFFB56DK.cjs → chunk-VTYBZZQ5.cjs} +2 -2
- package/dist/{chunk-CFFB56DK.cjs.map → chunk-VTYBZZQ5.cjs.map} +1 -1
- package/dist/{chunk-4DYEWT53.cjs → chunk-VY55L6P5.cjs} +2 -2
- package/dist/{chunk-4DYEWT53.cjs.map → chunk-VY55L6P5.cjs.map} +1 -1
- package/dist/{chunk-LFAJZQET.cjs → chunk-WAXPOCXW.cjs} +4 -4
- package/dist/{chunk-LFAJZQET.cjs.map → chunk-WAXPOCXW.cjs.map} +1 -1
- package/dist/{chunk-IWZ2HIHK.cjs → chunk-YFF6A56N.cjs} +2 -2
- package/dist/{chunk-IWZ2HIHK.cjs.map → chunk-YFF6A56N.cjs.map} +1 -1
- package/dist/{chunk-4Y2OJ475.cjs → chunk-ZUT7UIQU.cjs} +2 -2
- package/dist/{chunk-4Y2OJ475.cjs.map → chunk-ZUT7UIQU.cjs.map} +1 -1
- package/dist/{impl-OGY3N3JL.cjs → impl-2TS5SALN.cjs} +2 -2
- package/dist/{impl-OGY3N3JL.cjs.map → impl-2TS5SALN.cjs.map} +1 -1
- package/dist/{impl-H2HELZXJ.cjs → impl-2Y7WAQGJ.cjs} +2 -2
- package/dist/{impl-H2HELZXJ.cjs.map → impl-2Y7WAQGJ.cjs.map} +1 -1
- package/dist/{impl-3PVAU2SZ.cjs → impl-2YVWUXEC.cjs} +2 -2
- package/dist/{impl-3PVAU2SZ.cjs.map → impl-2YVWUXEC.cjs.map} +1 -1
- package/dist/{impl-WV34A76M.cjs → impl-3PH5TSFY.cjs} +3 -3
- package/dist/{impl-WV34A76M.cjs.map → impl-3PH5TSFY.cjs.map} +1 -1
- package/dist/{impl-55HR73CO.cjs → impl-4EEEQLZS.cjs} +3 -3
- package/dist/{impl-55HR73CO.cjs.map → impl-4EEEQLZS.cjs.map} +1 -1
- package/dist/{impl-XPH46QYI.cjs → impl-4RXGRXXS.cjs} +2 -2
- package/dist/{impl-XPH46QYI.cjs.map → impl-4RXGRXXS.cjs.map} +1 -1
- package/dist/{impl-NTMCDOS7.cjs → impl-6HWW3LAY.cjs} +2 -2
- package/dist/{impl-NTMCDOS7.cjs.map → impl-6HWW3LAY.cjs.map} +1 -1
- package/dist/{impl-2ZXK4G3X.cjs → impl-6OZ2ZIVM.cjs} +2 -2
- package/dist/{impl-2ZXK4G3X.cjs.map → impl-6OZ2ZIVM.cjs.map} +1 -1
- package/dist/{impl-JX55JUEK.cjs → impl-7YED2CRU.cjs} +2 -2
- package/dist/{impl-JX55JUEK.cjs.map → impl-7YED2CRU.cjs.map} +1 -1
- package/dist/{impl-ODHDBZ66.cjs → impl-AKKWJJRW.cjs} +2 -2
- package/dist/{impl-ODHDBZ66.cjs.map → impl-AKKWJJRW.cjs.map} +1 -1
- package/dist/{impl-PDZTOIGB.cjs → impl-CUNZ7YNN.cjs} +2 -2
- package/dist/{impl-PDZTOIGB.cjs.map → impl-CUNZ7YNN.cjs.map} +1 -1
- package/dist/{impl-BTCB55I2.cjs → impl-DDDGYCKH.cjs} +2 -2
- package/dist/{impl-BTCB55I2.cjs.map → impl-DDDGYCKH.cjs.map} +1 -1
- package/dist/{impl-CACU53DW.cjs → impl-DJCQNX56.cjs} +2 -2
- package/dist/{impl-CACU53DW.cjs.map → impl-DJCQNX56.cjs.map} +1 -1
- package/dist/{impl-KGR5OOIL.cjs → impl-FGLNPBNN.cjs} +2 -2
- package/dist/{impl-KGR5OOIL.cjs.map → impl-FGLNPBNN.cjs.map} +1 -1
- package/dist/{impl-EMPK5SP4.cjs → impl-FO3XHC3T.cjs} +2 -2
- package/dist/{impl-EMPK5SP4.cjs.map → impl-FO3XHC3T.cjs.map} +1 -1
- package/dist/{impl-TYNSXOHT.cjs → impl-IBGULX7G.cjs} +2 -2
- package/dist/{impl-TYNSXOHT.cjs.map → impl-IBGULX7G.cjs.map} +1 -1
- package/dist/{impl-F7TSRMCF.cjs → impl-IESXEP2S.cjs} +2 -2
- package/dist/{impl-F7TSRMCF.cjs.map → impl-IESXEP2S.cjs.map} +1 -1
- package/dist/{impl-YE7X5MSV.cjs → impl-IZTOX7X3.cjs} +2 -2
- package/dist/{impl-YE7X5MSV.cjs.map → impl-IZTOX7X3.cjs.map} +1 -1
- package/dist/{impl-EGBXUZEX.cjs → impl-JIVQZGAO.cjs} +2 -2
- package/dist/{impl-EGBXUZEX.cjs.map → impl-JIVQZGAO.cjs.map} +1 -1
- package/dist/{impl-QPOOCXTZ.cjs → impl-KUZ5CZKZ.cjs} +2 -2
- package/dist/{impl-QPOOCXTZ.cjs.map → impl-KUZ5CZKZ.cjs.map} +1 -1
- package/dist/{impl-R2KE5BYZ.cjs → impl-L6F4PTY3.cjs} +2 -2
- package/dist/{impl-R2KE5BYZ.cjs.map → impl-L6F4PTY3.cjs.map} +1 -1
- package/dist/{impl-DEJ7Q25E.cjs → impl-MAHW2W43.cjs} +2 -2
- package/dist/{impl-DEJ7Q25E.cjs.map → impl-MAHW2W43.cjs.map} +1 -1
- package/dist/{impl-PWMPWKRC.cjs → impl-MCE7B6NW.cjs} +2 -2
- package/dist/{impl-PWMPWKRC.cjs.map → impl-MCE7B6NW.cjs.map} +1 -1
- package/dist/{impl-H7PE22UA.cjs → impl-OCLMWYMP.cjs} +2 -2
- package/dist/{impl-H7PE22UA.cjs.map → impl-OCLMWYMP.cjs.map} +1 -1
- package/dist/{impl-SKGYIPGE.cjs → impl-OOQNMHTX.cjs} +2 -2
- package/dist/{impl-SKGYIPGE.cjs.map → impl-OOQNMHTX.cjs.map} +1 -1
- package/dist/{impl-DD246CUP.cjs → impl-OPBD2KMY.cjs} +5 -5
- package/dist/{impl-DD246CUP.cjs.map → impl-OPBD2KMY.cjs.map} +1 -1
- package/dist/impl-R3RC2YP7.cjs +2 -0
- package/dist/{impl-CVF67V6C.cjs.map → impl-R3RC2YP7.cjs.map} +1 -1
- package/dist/{impl-NSQG5ZE2.cjs → impl-RHAYM5BX.cjs} +4 -4
- package/dist/{impl-NSQG5ZE2.cjs.map → impl-RHAYM5BX.cjs.map} +1 -1
- package/dist/{impl-2RF3CQYJ.cjs → impl-SMK2SUTD.cjs} +2 -2
- package/dist/{impl-2RF3CQYJ.cjs.map → impl-SMK2SUTD.cjs.map} +1 -1
- package/dist/{impl-7Q7N7MEL.cjs → impl-TJIMDEBN.cjs} +8 -8
- package/dist/{impl-7Q7N7MEL.cjs.map → impl-TJIMDEBN.cjs.map} +1 -1
- package/dist/{impl-FS3K2DMH.cjs → impl-TXQ7EFQE.cjs} +2 -2
- package/dist/{impl-FS3K2DMH.cjs.map → impl-TXQ7EFQE.cjs.map} +1 -1
- package/dist/{impl-DUFE2SD3.cjs → impl-VKEICKZ7.cjs} +2 -2
- package/dist/{impl-DUFE2SD3.cjs.map → impl-VKEICKZ7.cjs.map} +1 -1
- package/dist/{impl-VE2OJ6VL.cjs → impl-VMV24ZJH.cjs} +2 -2
- package/dist/{impl-VE2OJ6VL.cjs.map → impl-VMV24ZJH.cjs.map} +1 -1
- package/dist/{impl-5YHZWUVM.cjs → impl-VXHUL55C.cjs} +2 -2
- package/dist/{impl-5YHZWUVM.cjs.map → impl-VXHUL55C.cjs.map} +1 -1
- package/dist/{impl-U4XXZKTL.cjs → impl-WIGDO7DS.cjs} +2 -2
- package/dist/{impl-U4XXZKTL.cjs.map → impl-WIGDO7DS.cjs.map} +1 -1
- package/dist/{impl-5273O2PM.cjs → impl-WKBZDQYY.cjs} +2 -2
- package/dist/{impl-5273O2PM.cjs.map → impl-WKBZDQYY.cjs.map} +1 -1
- package/dist/{impl-A5WFSC3U.cjs → impl-WLYOQTIL.cjs} +2 -2
- package/dist/{impl-A5WFSC3U.cjs.map → impl-WLYOQTIL.cjs.map} +1 -1
- package/dist/{impl-GWQDOEQM.cjs → impl-YEXHKEWQ.cjs} +2 -2
- package/dist/{impl-GWQDOEQM.cjs.map → impl-YEXHKEWQ.cjs.map} +1 -1
- package/dist/{impl-YQ6LBC3S.cjs → impl-YHIIZRG6.cjs} +2 -2
- package/dist/{impl-YQ6LBC3S.cjs.map → impl-YHIIZRG6.cjs.map} +1 -1
- package/dist/{impl-MYKXOHZB.cjs → impl-YPKYWIYP.cjs} +2 -2
- package/dist/{impl-MYKXOHZB.cjs.map → impl-YPKYWIYP.cjs.map} +1 -1
- package/dist/{impl-M353HLO7.cjs → impl-ZVMYU6PT.cjs} +2 -2
- package/dist/{impl-M353HLO7.cjs.map → impl-ZVMYU6PT.cjs.map} +1 -1
- package/dist/index.cjs +3 -3
- package/dist/index.d.cts +1 -1
- package/package.json +1 -1
- package/dist/impl-CVF67V6C.cjs +0 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-7Q7N7MEL.cjs","../src/commands/consent/upload-preferences/impl.ts","../src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts","../src/lib/preference-management/parsePreferenceManagementCsv.ts","../src/lib/preference-management/getPreferencesForIdentifiers.ts","../src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts","../src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts","../src/lib/preference-management/codecs.ts"],"names":["PreferenceRecordsQueryResponse","PreferenceQueryResponseItem","MSGS","getPreferencesForIdentifiers","sombra","identifiers","partitionKey","skipLogging","results","groupedIdentifiers","chunk_default","t0","progressBar","cliProgress","total","map","group","attempts","maxAttempts","rawResult","result","decodeCodec","err","msg","errorMessage"],"mappings":"AAAA,u/BAAyC,wDAAwG,gCAA6B,wDAAyC,wDAAuE,gCAA6B,gFCCxS,oCCQC,qGAGI,+DAEO,qJCXZ,2DCHyB,qDAKhB,IAKtBA,EAAAA,CAAmC,CAAA,CAAA,YAAA,CAAa,CAClD,CAAA,CAAA,IAAA,CAAK,CACL,KAAA,CAAS,CAAA,CAAA,KAAA,CAAMC,yCAA2B,CAC5C,CAAC,CAAA,CACC,CAAA,CAAA,OAAA,CAAQ,CAER,MAAA,CAAU,CAAA,CAAA,MACZ,CAAC,CACH,CAAC,CAAA,CAEKC,EAAAA,CAAO,CACX,WAAA,CACA,WAAA,CACA,sBAAA,CACA,sBACF,CAAA,CASA,MAAA,SAAsBC,EAAAA,CACpBC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,CAAA,CAChB,CAAA,CAWwC,CACxC,IAAMC,CAAAA,CAAyC,CAAC,CAAA,CAC1CC,CAAAA,CAAqBC,iCAAAA,CAAML,CAAa,GAAG,CAAA,CAG3CM,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CACxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CACKN,CAAAA,EACHK,CAAAA,CAAY,KAAA,CAAMP,CAAAA,CAAY,MAAA,CAAQ,CAAC,CAAA,CAGzC,IAAIS,CAAAA,CAAQ,CAAA,CACZ,MAAMC,2BAAAA,CACJN,CACA,MAAOO,CAAAA,EAAU,CAEf,IAAIC,CAAAA,CAAW,CAAA,CACTC,CAAAA,CAAc,CAAA,CACpB,GAAA,CAAA,CAAOD,CAAAA,CAAWC,CAAAA,CAAAA,CAChB,GAAI,CACF,IAAMC,CAAAA,CAAY,MAAMf,CAAAA,CACrB,IAAA,CAAK,CAAA,eAAA,EAAkBE,CAAY,CAAA,MAAA,CAAA,CAAU,CAC5C,IAAA,CAAM,CACJ,MAAA,CAAQ,CACN,WAAA,CAAaU,CACf,CAAA,CACA,KAAA,CAAOA,CAAAA,CAAM,MACf,CACF,CAAC,CAAA,CACA,IAAA,CAAK,CAAA,CAEFI,CAAAA,CAASC,oCAAAA,EAAYrB,CAAgCmB,CAAS,CAAA,CACpEX,CAAAA,CAAQ,IAAA,CAAK,GAAGY,CAAAA,CAAO,KAAK,CAAA,CAC5BN,CAAAA,EAASE,CAAAA,CAAM,MAAA,CACfJ,CAAAA,CAAY,MAAA,CAAOE,CAAK,CAAA,CACxB,KACF,CAAA,KAAA,CAASQ,CAAAA,CAAK,CACZL,CAAAA,EAAY,CAAA,CACZ,IAAMM,CAAAA,iBAAMD,CAAAA,6BAAK,QAAA,6BAAU,MAAA,kBAAQA,CAAAA,6BAAK,SAAA,EAAW,EAAA,CACnD,EAAA,CACEL,CAAAA,EAAYC,CAAAA,EACZ,CAAChB,EAAAA,CAAK,IAAA,CAAMsB,CAAAA,EAAiBD,CAAAA,CAAI,QAAA,CAASC,CAAY,CAAC,CAAA,CAEvD,MAAM,IAAI,KAAA,CACR,CAAA,oCAAA,EAAuCP,CAAQ,CAAA,WAAA,EAAcM,CAAG,CAAA,CAAA;ACclE;ACHS;ACkFjB,oGAAA;ALrGI;AAGA;AAAA;AAKQ;AAAK;AACgC,sCAAA;AAmIpB","file":"/home/runner/work/cli/cli/dist/impl-7Q7N7MEL.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport colors from 'colors';\n\nimport { logger } from '../../../logger';\nimport { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management';\nimport { splitCsvToList } from '../../../lib/requests';\nimport { readdirSync } from 'node:fs';\nimport { map } from 'bluebird';\nimport { basename, join } from 'node:path';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadPreferencesCommandFlags {\n auth: string;\n partition: string;\n sombraAuth?: string;\n transcendUrl: string;\n file?: string;\n directory?: string;\n dryRun: boolean;\n skipExistingRecordCheck: boolean;\n receiptFileDir: string;\n skipWorkflowTriggers: boolean;\n forceTriggerWorkflows: boolean;\n skipConflictUpdates: boolean;\n isSilent: boolean;\n attributes: string;\n receiptFilepath: string;\n concurrency: number;\n}\n\nexport async function uploadPreferences(\n this: LocalContext,\n {\n auth,\n partition,\n sombraAuth,\n transcendUrl,\n file = '',\n directory,\n dryRun,\n skipExistingRecordCheck,\n receiptFileDir,\n skipWorkflowTriggers,\n forceTriggerWorkflows,\n skipConflictUpdates,\n isSilent,\n attributes,\n concurrency,\n }: UploadPreferencesCommandFlags,\n): Promise<void> {\n if (!!directory && !!file) {\n logger.error(\n colors.red(\n 'Cannot provide both a directory and a file. Please provide only one.',\n ),\n );\n this.process.exit(1);\n }\n\n if (!file && !directory) {\n logger.error(\n colors.red(\n 'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences',\n ),\n );\n this.process.exit(1);\n }\n\n doneInputValidation(this.process.exit);\n\n const files: string[] = [];\n\n if (directory) {\n try {\n const filesInDirectory = readdirSync(directory);\n const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv'));\n\n if (csvFiles.length === 0) {\n logger.error(\n colors.red(`No CSV files found in directory: ${directory}`),\n );\n this.process.exit(1);\n }\n\n // Add full paths for each CSV file\n files.push(...csvFiles.map((file) => join(directory, file)));\n } catch (err) {\n logger.error(colors.red(`Failed to read directory: ${directory}`));\n logger.error(colors.red((err as Error).message));\n this.process.exit(1);\n }\n } else {\n try {\n // Verify file exists and is a CSV\n if (!file.endsWith('.csv')) {\n logger.error(colors.red('File must be a CSV file'));\n this.process.exit(1);\n }\n files.push(file);\n } catch (err) {\n logger.error(colors.red(`Failed to access file: ${file}`));\n logger.error(colors.red((err as Error).message));\n this.process.exit(1);\n }\n }\n\n logger.info(\n colors.green(\n `Processing ${files.length} consent preferences files for partition: ${partition}`,\n ),\n );\n logger.debug(`Files to process: ${files.join(', ')}`);\n\n if (skipExistingRecordCheck) {\n logger.info(\n colors.bgYellow(\n `Skipping existing record check: ${skipExistingRecordCheck}`,\n ),\n );\n }\n\n await map(\n files,\n async (filePath) => {\n const fileName = basename(filePath).replace('.csv', '');\n await uploadPreferenceManagementPreferencesInteractive({\n receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`),\n auth,\n sombraAuth,\n file: filePath,\n partition,\n transcendUrl,\n skipConflictUpdates,\n skipWorkflowTriggers,\n skipExistingRecordCheck,\n isSilent,\n dryRun,\n attributes: splitCsvToList(attributes),\n forceTriggerWorkflows,\n });\n },\n { concurrency },\n );\n}\n","import {\n buildTranscendGraphQLClient,\n createSombraGotInstance,\n fetchAllPurposes,\n fetchAllPreferenceTopics,\n PreferenceTopic,\n Purpose,\n} from '../graphql';\nimport colors from 'colors';\nimport { map } from 'bluebird';\nimport { chunk } from 'lodash-es';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { parseAttributesFromString } from '../requests';\nimport { PersistedState } from '@transcend-io/persisted-state';\nimport { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv';\nimport { PreferenceState } from './codecs';\nimport { PreferenceUpdateItem } from '@transcend-io/privacy-types';\nimport { apply } from '@transcend-io/type-utils';\nimport { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\n\n/**\n * Upload a set of consent preferences\n *\n * @param options - Options\n */\nexport async function uploadPreferenceManagementPreferencesInteractive({\n auth,\n sombraAuth,\n receiptFilepath,\n file,\n partition,\n isSilent = true,\n dryRun = false,\n skipWorkflowTriggers = false,\n skipConflictUpdates = false,\n skipExistingRecordCheck = false,\n attributes = [],\n transcendUrl,\n forceTriggerWorkflows = false,\n}: {\n /** The Transcend API key */\n auth: string;\n /** Sombra API key authentication */\n sombraAuth?: string;\n /** Partition key */\n partition: string;\n /** File where to store receipt and continue from where left off */\n receiptFilepath: string;\n /** The file to process */\n file: string;\n /** API URL for Transcend backend */\n transcendUrl: string;\n /** Whether to do a dry run */\n dryRun?: boolean;\n /** Whether to upload as isSilent */\n isSilent?: boolean;\n /** Attributes string pre-parse. In format Key:Value */\n attributes?: string[];\n /** Skip workflow triggers */\n skipWorkflowTriggers?: boolean;\n /**\n * When true, only update preferences that do not conflict with existing\n * preferences. When false, update all preferences in CSV based on timestamp.\n */\n skipConflictUpdates?: boolean;\n /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n skipExistingRecordCheck?: boolean;\n /** Whether to force trigger workflows */\n forceTriggerWorkflows?: boolean;\n}): Promise<void> {\n // Parse out the extra attributes to apply to all requests uploaded\n const parsedAttributes = parseAttributesFromString(attributes);\n\n // Create a new state file to store the requests from this run\n const preferenceState = new PersistedState(receiptFilepath, PreferenceState, {\n fileMetadata: {},\n failingUpdates: {},\n pendingUpdates: {},\n });\n const failingRequests = preferenceState.getValue('failingUpdates');\n const pendingRequests = preferenceState.getValue('pendingUpdates');\n let fileMetadata = preferenceState.getValue('fileMetadata');\n\n logger.info(\n colors.magenta(\n 'Restored cache, there are: \\n' +\n `${\n Object.values(failingRequests).length\n } failing requests to be retried\\n` +\n `${\n Object.values(pendingRequests).length\n } pending requests to be processed\\n` +\n `The following files are stored in cache and will be used:\\n${Object.keys(\n fileMetadata,\n )\n .map((x) => x)\n .join('\\n')}\\n` +\n `The following file will be processed: ${file}\\n`,\n ),\n );\n\n // Create GraphQL client to connect to Transcend backend\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n const [sombra, purposes, preferenceTopics] = await Promise.all([\n // Create sombra instance to communicate with\n createSombraGotInstance(transcendUrl, auth, sombraAuth),\n // get all purposes and topics\n forceTriggerWorkflows\n ? Promise.resolve([] as Purpose[])\n : fetchAllPurposes(client),\n forceTriggerWorkflows\n ? Promise.resolve([] as PreferenceTopic[])\n : fetchAllPreferenceTopics(client),\n ]);\n\n // Process the file\n await parsePreferenceManagementCsvWithCache(\n {\n file,\n purposeSlugs: purposes.map((x) => x.trackingType),\n preferenceTopics,\n sombra,\n partitionKey: partition,\n skipExistingRecordCheck,\n forceTriggerWorkflows,\n },\n preferenceState,\n );\n\n // Construct the pending updates\n const pendingUpdates: Record<string, PreferenceUpdateItem> = {};\n fileMetadata = preferenceState.getValue('fileMetadata');\n const metadata = fileMetadata[file];\n\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.pendingSafeUpdates).length\n } safe updates in ${file}`,\n ),\n );\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.pendingConflictUpdates).length\n } conflict updates in ${file}`,\n ),\n );\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.skippedUpdates).length\n } skipped updates in ${file}`,\n ),\n );\n\n // Update either safe updates only or safe + conflict\n Object.entries({\n ...metadata.pendingSafeUpdates,\n ...(skipConflictUpdates\n ? {}\n : apply(metadata.pendingConflictUpdates, ({ row }) => row)),\n }).forEach(([userId, update]) => {\n // Determine timestamp\n const timestamp =\n metadata.timestampColum === NONE_PREFERENCE_MAP\n ? new Date()\n : new Date(update[metadata.timestampColum!]);\n\n // Determine updates\n const updates = getPreferenceUpdatesFromRow({\n row: update,\n columnToPurposeName: metadata.columnToPurposeName,\n preferenceTopics,\n purposeSlugs: purposes.map((x) => x.trackingType),\n });\n pendingUpdates[userId] = {\n userId,\n partition,\n timestamp: timestamp.toISOString(),\n purposes: Object.entries(updates).map(([purpose, value]) => ({\n ...value,\n purpose,\n workflowSettings: {\n attributes: parsedAttributes,\n isSilent,\n skipWorkflowTrigger: skipWorkflowTriggers,\n },\n })),\n };\n });\n await preferenceState.setValue(pendingUpdates, 'pendingUpdates');\n await preferenceState.setValue({}, 'failingUpdates');\n\n // Exist early if dry run\n if (dryRun) {\n logger.info(\n colors.green(\n `Dry run complete, exiting. ${\n Object.values(pendingUpdates).length\n } pending updates. Check file: ${receiptFilepath}`,\n ),\n );\n return;\n }\n\n logger.info(\n colors.magenta(\n `Uploading ${\n Object.values(pendingUpdates).length\n } preferences to partition: ${partition}`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Build a GraphQL client\n let total = 0;\n const updatesToRun = Object.entries(pendingUpdates);\n const chunkedUpdates = chunk(updatesToRun, skipWorkflowTriggers ? 100 : 10);\n progressBar.start(updatesToRun.length, 0);\n await map(\n chunkedUpdates,\n async (currentChunk) => {\n // Make the request\n try {\n await sombra\n .put('v1/preferences', {\n json: {\n records: currentChunk.map(([, update]) => update),\n skipWorkflowTriggers,\n forceTriggerWorkflows,\n },\n })\n .json();\n } catch (err) {\n try {\n const parsed = JSON.parse(err?.response?.body || '{}');\n if (parsed.error) {\n logger.error(colors.red(`Error: ${parsed.error}`));\n }\n } catch (e) {\n // continue\n }\n logger.error(\n colors.red(\n `Failed to upload ${\n currentChunk.length\n } user preferences to partition ${partition}: ${\n err?.response?.body || err?.message\n }`,\n ),\n );\n const failingUpdates = preferenceState.getValue('failingUpdates');\n currentChunk.forEach(([userId, update]) => {\n failingUpdates[userId] = {\n uploadedAt: new Date().toISOString(),\n update,\n error: err?.response?.body || err?.message || 'Unknown error',\n };\n });\n await preferenceState.setValue(failingUpdates, 'failingUpdates');\n }\n\n total += currentChunk.length;\n progressBar.update(total);\n },\n {\n concurrency: 40,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n logger.info(\n colors.green(\n `Successfully uploaded ${\n updatesToRun.length\n } user preferences to partition ${partition} in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n}\n","import { PersistedState } from '@transcend-io/persisted-state';\nimport type { Got } from 'got';\nimport { keyBy } from 'lodash-es';\nimport * as t from 'io-ts';\nimport colors from 'colors';\nimport { FileMetadataState, PreferenceState } from './codecs';\nimport { logger } from '../../logger';\nimport { readCsv } from '../requests';\nimport { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers';\nimport { PreferenceTopic } from '../graphql';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\nimport { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv';\nimport { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv';\nimport { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv';\nimport { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp';\nimport { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict';\n\n/**\n * Parse a file into the cache\n *\n *\n * @param options - Options\n * @param cache - The cache to store the parsed file in\n * @returns The cache with the parsed file\n */\nexport async function parsePreferenceManagementCsvWithCache(\n {\n file,\n sombra,\n purposeSlugs,\n preferenceTopics,\n partitionKey,\n skipExistingRecordCheck,\n forceTriggerWorkflows,\n }: {\n /** File to parse */\n file: string;\n /** The purpose slugs that are allowed to be updated */\n purposeSlugs: string[];\n /** The preference topics */\n preferenceTopics: PreferenceTopic[];\n /** Sombra got instance */\n sombra: Got;\n /** Partition key */\n partitionKey: string;\n /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n skipExistingRecordCheck: boolean;\n /** Wheather to force workflow triggers */\n forceTriggerWorkflows: boolean;\n },\n cache: PersistedState<typeof PreferenceState>,\n): Promise<void> {\n // Start the timer\n const t0 = new Date().getTime();\n\n // Get the current metadata\n const fileMetadata = cache.getValue('fileMetadata');\n\n // Read in the file\n logger.info(colors.magenta(`Reading in file: \"${file}\"`));\n let preferences = readCsv(file, t.record(t.string, t.string));\n\n // start building the cache, can use previous cache as well\n let currentState: FileMetadataState = {\n columnToPurposeName: {},\n pendingSafeUpdates: {},\n pendingConflictUpdates: {},\n skippedUpdates: {},\n // Load in the last fetched time\n ...((fileMetadata[file] || {}) as Partial<FileMetadataState>),\n lastFetchedAt: new Date().toISOString(),\n };\n\n // Validate that all timestamps are present in the file\n currentState = await parsePreferenceTimestampsFromCsv(\n preferences,\n currentState,\n );\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Validate that all identifiers are present and unique\n const result = await parsePreferenceIdentifiersFromCsv(\n preferences,\n currentState,\n );\n currentState = result.currentState;\n preferences = result.preferences;\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Ensure all other columns are mapped to purpose and preference\n // slug values\n currentState = await parsePreferenceAndPurposeValuesFromCsv(\n preferences,\n currentState,\n {\n preferenceTopics,\n purposeSlugs,\n forceTriggerWorkflows,\n },\n );\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Grab existing preference store records\n const identifiers = preferences.map(\n (pref) => pref[currentState.identifierColumn!],\n );\n const existingConsentRecords = skipExistingRecordCheck\n ? []\n : await getPreferencesForIdentifiers(sombra, {\n identifiers: identifiers.map((x) => ({ value: x })),\n partitionKey,\n });\n const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId');\n\n // Clear out previous updates\n currentState.pendingConflictUpdates = {};\n currentState.pendingSafeUpdates = {};\n currentState.skippedUpdates = {};\n\n // Process each row\n preferences.forEach((pref) => {\n // Grab unique Id for the user\n const userId = pref[currentState.identifierColumn!];\n\n // determine updates for user\n const pendingUpdates = getPreferenceUpdatesFromRow({\n row: pref,\n columnToPurposeName: currentState.columnToPurposeName,\n preferenceTopics,\n purposeSlugs,\n });\n\n // Grab current state of the update\n const currentConsentRecord = consentRecordByIdentifier[userId];\n if (forceTriggerWorkflows && !currentConsentRecord) {\n throw new Error(\n `No existing consent record found for user with id: ${userId}. \n When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`,\n );\n }\n // Check if the update can be skipped\n // this is the case if a record exists, and the purpose\n // and preference values are all in sync\n if (\n currentConsentRecord &&\n checkIfPendingPreferenceUpdatesAreNoOp({\n currentConsentRecord,\n pendingUpdates,\n preferenceTopics,\n }) &&\n !forceTriggerWorkflows\n ) {\n currentState.skippedUpdates[userId] = pref;\n return;\n }\n\n // Determine if there are any conflicts\n if (\n currentConsentRecord &&\n checkIfPendingPreferenceUpdatesCauseConflict({\n currentConsentRecord,\n pendingUpdates,\n preferenceTopics,\n })\n ) {\n currentState.pendingConflictUpdates[userId] = {\n row: pref,\n record: currentConsentRecord,\n };\n return;\n }\n\n // Add to pending updates\n currentState.pendingSafeUpdates[userId] = pref;\n });\n\n // Read in the file\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n const t1 = new Date().getTime();\n logger.info(\n colors.green(\n `Successfully pre-processed file: \"${file}\" in ${(t1 - t0) / 1000}s`,\n ),\n );\n}\n","import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\nimport colors from 'colors';\nimport cliProgress from 'cli-progress';\nimport { chunk } from 'lodash-es';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport * as t from 'io-ts';\nimport { map } from 'bluebird';\nimport { logger } from '../../logger';\n\nconst PreferenceRecordsQueryResponse = t.intersection([\n t.type({\n nodes: t.array(PreferenceQueryResponseItem),\n }),\n t.partial({\n /** The base64 encoded(PreferenceStorePaginationKey) cursor for pagination */\n cursor: t.string,\n }),\n]);\n\nconst MSGS = [\n 'ENOTFOUND',\n 'ETIMEDOUT',\n '504 Gateway Time-out',\n 'Task timed out after',\n];\n\n/**\n * Grab the current consent preference values for a list of identifiers\n *\n * @param sombra - Backend to make API call to\n * @param options - Options\n * @returns Plaintext context information\n */\nexport async function getPreferencesForIdentifiers(\n sombra: Got,\n {\n identifiers,\n partitionKey,\n skipLogging = false,\n }: {\n /** The list of identifiers to look up */\n identifiers: {\n /** The value of the identifier */\n value: string;\n }[];\n /** The partition key to look up */\n partitionKey: string;\n /** Whether to skip logging */\n skipLogging?: boolean;\n },\n): Promise<PreferenceQueryResponseItem[]> {\n const results: PreferenceQueryResponseItem[] = [];\n const groupedIdentifiers = chunk(identifiers, 100);\n\n // create a new progress bar instance and use shades_classic theme\n const t0 = new Date().getTime();\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n if (!skipLogging) {\n progressBar.start(identifiers.length, 0);\n }\n\n let total = 0;\n await map(\n groupedIdentifiers,\n async (group) => {\n // Make the request with retry logic\n let attempts = 0;\n const maxAttempts = 3;\n while (attempts < maxAttempts) {\n try {\n const rawResult = await sombra\n .post(`v1/preferences/${partitionKey}/query`, {\n json: {\n filter: {\n identifiers: group,\n },\n limit: group.length,\n },\n })\n .json();\n\n const result = decodeCodec(PreferenceRecordsQueryResponse, rawResult);\n results.push(...result.nodes);\n total += group.length;\n progressBar.update(total);\n break; // Exit loop if successful\n } catch (err) {\n attempts += 1;\n const msg = err?.response?.body || err?.message || '';\n if (\n attempts >= maxAttempts ||\n !MSGS.some((errorMessage) => msg.includes(errorMessage))\n ) {\n throw new Error(\n `Received an error from server after ${attempts} attempts: ${msg}`,\n );\n }\n\n logger.warn(\n colors.yellow(\n `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` +\n `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`,\n ),\n );\n }\n }\n },\n {\n concurrency: 40,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n if (!skipLogging) {\n // Log completion time\n logger.info(\n colors.green(`Completed download in \"${totalTime / 1000}\" seconds.`),\n );\n }\n\n return results;\n}\n","import { uniq, groupBy, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { inquirerConfirmBoolean } from '../helpers';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse identifiers from a CSV list of preferences\n *\n * Ensures that all rows have a valid identifier\n * and that all identifiers are unique.\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceIdentifiersFromCsv(\n preferences: Record<string, string>[],\n currentState: FileMetadataState,\n): Promise<{\n /** The updated state */\n currentState: FileMetadataState;\n /** The updated preferences */\n preferences: Record<string, string>[];\n}> {\n // Determine columns to map\n const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n // Determine the columns that could potentially be used for identifier\n const remainingColumnsForIdentifier = difference(columnNames, [\n ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n ...Object.keys(currentState.columnToPurposeName),\n ]);\n\n // Determine the identifier column to work off of\n if (!currentState.identifierColumn) {\n const { identifierName } = await inquirer.prompt<{\n /** Identifier name */\n identifierName: string;\n }>([\n {\n name: 'identifierName',\n message:\n 'Choose the column that will be used as the identifier to upload consent preferences by',\n type: 'list',\n default:\n remainingColumnsForIdentifier.find((col) =>\n col.toLowerCase().includes('email'),\n ) || remainingColumnsForIdentifier[0],\n choices: remainingColumnsForIdentifier,\n },\n ]);\n currentState.identifierColumn = identifierName;\n }\n logger.info(\n colors.magenta(\n `Using identifier column \"${currentState.identifierColumn}\"`,\n ),\n );\n\n // Validate that the identifier column is present for all rows and unique\n const identifierColumnsMissing = preferences\n .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind]))\n .filter((x): x is number[] => !!x)\n .flat();\n if (identifierColumnsMissing.length > 0) {\n const msg = `The identifier column \"${\n currentState.identifierColumn\n }\" is missing a value for the following rows: ${identifierColumnsMissing.join(\n ', ',\n )}`;\n logger.warn(colors.yellow(msg));\n\n // Ask user if they would like to skip rows missing an identifier\n const skip = await inquirerConfirmBoolean({\n message: 'Would you like to skip rows missing an identifier?',\n });\n if (!skip) {\n throw new Error(msg);\n }\n\n // Filter out rows missing an identifier\n const previous = preferences.length;\n preferences = preferences.filter(\n (pref) => pref[currentState.identifierColumn!],\n );\n logger.info(\n colors.yellow(\n `Skipped ${previous - preferences.length} rows missing an identifier`,\n ),\n );\n }\n logger.info(\n colors.magenta(\n `The identifier column \"${currentState.identifierColumn}\" is present for all rows`,\n ),\n );\n\n // Validate that all identifiers are unique\n const rowsByUserId = groupBy(preferences, currentState.identifierColumn);\n const duplicateIdentifiers = Object.entries(rowsByUserId).filter(\n ([, rows]) => rows.length > 1,\n );\n if (duplicateIdentifiers.length > 0) {\n const msg = `The identifier column \"${\n currentState.identifierColumn\n }\" has duplicate values for the following rows: ${duplicateIdentifiers\n .slice(0, 10)\n .map(([userId, rows]) => `${userId} (${rows.length})`)\n .join('\\n')}`;\n logger.warn(colors.yellow(msg));\n\n // Ask user if they would like to take the most recent update\n // for each duplicate identifier\n const skip = await inquirerConfirmBoolean({\n message: 'Would you like to automatically take the latest update?',\n });\n if (!skip) {\n throw new Error(msg);\n }\n preferences = Object.entries(rowsByUserId)\n .map(([, rows]) => {\n const sorted = rows.sort(\n (a, b) =>\n new Date(b[currentState.timestampColum!]).getTime() -\n new Date(a[currentState.timestampColum!]).getTime(),\n );\n return sorted[0];\n })\n .filter((x) => x);\n }\n\n return { currentState, preferences };\n}\n/* eslint-enable no-param-reassign */\n","import { uniq, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { mapSeries } from 'bluebird';\nimport { PreferenceTopic } from '../graphql';\nimport { PreferenceTopicType } from '@transcend-io/privacy-types';\nimport { splitCsvToList } from '../requests';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse out the purpose.enabled and preference values from a CSV file\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @param options - Options\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceAndPurposeValuesFromCsv(\n preferences: Record<string, string>[],\n currentState: FileMetadataState,\n {\n purposeSlugs,\n preferenceTopics,\n forceTriggerWorkflows,\n }: {\n /** The purpose slugs that are allowed to be updated */\n purposeSlugs: string[];\n /** The preference topics */\n preferenceTopics: PreferenceTopic[];\n /** Force workflow triggers */\n forceTriggerWorkflows: boolean;\n },\n): Promise<FileMetadataState> {\n // Determine columns to map\n const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n // Determine the columns that could potentially be used for identifier\n const otherColumns = difference(columnNames, [\n ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n ...(currentState.timestampColum ? [currentState.timestampColum] : []),\n ]);\n if (otherColumns.length === 0) {\n if (forceTriggerWorkflows) {\n return currentState;\n }\n throw new Error('No other columns to process');\n }\n\n // The purpose and preferences to map to\n const purposeNames = [\n ...purposeSlugs,\n ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`),\n ];\n\n // Ensure all columns are accounted for\n await mapSeries(otherColumns, async (col) => {\n // Determine the unique values to map in this column\n const uniqueValues = uniq(preferences.map((x) => x[col]));\n\n // Map the column to a purpose\n let purposeMapping = currentState.columnToPurposeName[col];\n if (purposeMapping) {\n logger.info(\n colors.magenta(\n `Column \"${col}\" is associated with purpose \"${purposeMapping.purpose}\"`,\n ),\n );\n } else {\n const { purposeName } = await inquirer.prompt<{\n /** purpose name */\n purposeName: string;\n }>([\n {\n name: 'purposeName',\n message: `Choose the purpose that column ${col} is associated with`,\n type: 'list',\n default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])),\n choices: purposeNames,\n },\n ]);\n const [purposeSlug, preferenceSlug] = purposeName.split('->');\n purposeMapping = {\n purpose: purposeSlug,\n preference: preferenceSlug || null,\n valueMapping: {},\n };\n }\n\n // map each value to the purpose value\n await mapSeries(uniqueValues, async (value) => {\n if (purposeMapping.valueMapping[value] !== undefined) {\n logger.info(\n colors.magenta(\n `Value \"${value}\" is associated with purpose value \"${purposeMapping.valueMapping[value]}\"`,\n ),\n );\n return;\n }\n // if preference is null, this column is just for the purpose\n if (purposeMapping.preference === null) {\n const { purposeValue } = await inquirer.prompt<{\n /** purpose value */\n purposeValue: boolean;\n }>([\n {\n name: 'purposeValue',\n message: `Choose the purpose value for value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'confirm',\n default: value !== 'false',\n },\n ]);\n purposeMapping.valueMapping[value] = purposeValue;\n }\n\n // if preference is not null, this column is for a specific preference\n if (purposeMapping.preference !== null) {\n const preferenceTopic = preferenceTopics.find(\n (x) => x.slug === purposeMapping.preference,\n );\n if (!preferenceTopic) {\n logger.error(\n colors.red(\n `Preference topic \"${purposeMapping.preference}\" not found`,\n ),\n );\n return;\n }\n const preferenceOptions = preferenceTopic.preferenceOptionValues.map(\n ({ slug }) => slug,\n );\n\n if (preferenceTopic.type === PreferenceTopicType.Boolean) {\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n message:\n // eslint-disable-next-line max-len\n `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'confirm',\n default: value !== 'false',\n },\n ]);\n purposeMapping.valueMapping[value] = preferenceValue;\n return;\n }\n\n if (preferenceTopic.type === PreferenceTopicType.Select) {\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n // eslint-disable-next-line max-len\n message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'list',\n choices: preferenceOptions,\n default: preferenceOptions.find((x) => x === value),\n },\n ]);\n purposeMapping.valueMapping[value] = preferenceValue;\n return;\n }\n\n if (preferenceTopic.type === PreferenceTopicType.MultiSelect) {\n const parsedValues = splitCsvToList(value);\n // need to do this serially\n await mapSeries(parsedValues, async (parsedValue) => {\n // if we already have a value, skip re-processing it again\n if (purposeMapping.valueMapping[parsedValue] !== undefined) {\n return;\n }\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n // eslint-disable-next-line max-len\n message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${parsedValue}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'list',\n choices: preferenceOptions,\n default: preferenceOptions.find((x) => x === parsedValue),\n },\n ]);\n purposeMapping.valueMapping[parsedValue] = preferenceValue;\n });\n return;\n }\n\n throw new Error(\n `Unknown preference topic type: ${preferenceTopic.type}`,\n );\n }\n });\n\n currentState.columnToPurposeName[col] = purposeMapping;\n });\n\n return currentState;\n}\n/* eslint-enable no-param-reassign */\n","import {\n PreferenceQueryResponseItem,\n PreferenceUpdateItem,\n} from '@transcend-io/privacy-types';\nimport * as t from 'io-ts';\n\nexport const PurposeRowMapping = t.type({\n /**\n * The slug or trackingType of the purpose to map to\n *\n * e.g. `Marketing`\n */\n purpose: t.string,\n /**\n * If the column maps to a preference instead of a purpose\n * this is the slug of the purpose.\n *\n * null value indicates that this column maps to the true/false\n * value of the purpose\n */\n preference: t.union([t.string, t.null]),\n /**\n * The mapping between each row value and purpose/preference value.\n *\n * e.g. for a boolean preference or purpose\n * {\n * 'true': true,\n * 'false': false,\n * '': true,\n * }\n *\n * or for a single or multi select preference\n * {\n * '': true,\n * 'value1': 'Value1',\n * 'value2': 'Value2',\n * }\n */\n valueMapping: t.record(\n t.string,\n t.union([t.string, t.boolean, t.null, t.undefined]),\n ),\n});\n\n/** Override type */\nexport type PurposeRowMapping = t.TypeOf<typeof PurposeRowMapping>;\n\n/**\n * Mapping of column name to purpose row mapping.\n * This is used to map each column in the CSV to the relevant purpose and preference definitions in\n * transcend.\n */\nexport const ColumnPurposeMap = t.record(t.string, PurposeRowMapping);\n\n/** Override type */\nexport type ColumnPurposeMap = t.TypeOf<typeof ColumnPurposeMap>;\n\nexport const IdentifierMetadataForPreference = t.type({\n /** The identifier name */\n name: t.string,\n /** Is unique on preference store */\n isUniqueOnPreferenceStore: t.boolean,\n});\n\n/** Override type */\nexport type IdentifierMetadataForPreference = t.TypeOf<\n typeof IdentifierMetadataForPreference\n>;\n\n/**\n * Mapping of identifier name to the column name in the CSV file.\n * This is used to map each identifier name to the column in the CSV file.\n */\nexport const ColumnIdentifierMap = t.record(\n t.string,\n IdentifierMetadataForPreference,\n);\n\n/** Override type */\nexport type ColumnIdentifierMap = t.TypeOf<typeof ColumnIdentifierMap>;\n\nexport const FileMetadataState = t.intersection([\n t.type({\n /**\n * Definition of how to map each column in the CSV to\n * the relevant purpose and preference definitions in transcend\n */\n columnToPurposeName: t.record(t.string, PurposeRowMapping),\n /** Last time the file was last parsed at */\n lastFetchedAt: t.string,\n /**\n * Mapping of userId to the rows in the file that need to be uploaded\n * These uploads are overwriting non-existent preferences and are safe\n */\n pendingSafeUpdates: t.record(t.string, t.record(t.string, t.string)),\n /**\n * Mapping of userId to the rows in the file that need to be uploaded\n * these records have conflicts with existing consent preferences\n */\n pendingConflictUpdates: t.record(\n t.string,\n t.type({\n record: PreferenceQueryResponseItem,\n row: t.record(t.string, t.string),\n }),\n ),\n /**\n * Mapping of userId to the rows in the file that can be skipped because\n * their preferences are already in the store\n */\n skippedUpdates: t.record(t.string, t.record(t.string, t.string)),\n }),\n t.partial({\n /** Determine which column name in file maps to consent record identifier to upload on */\n identifierColumn: t.string,\n /** Determine which column name in file maps to the timestamp */\n timestampColum: t.string,\n }),\n]);\n\n/** Override type */\nexport type FileMetadataState = t.TypeOf<typeof FileMetadataState>;\n\n/**\n * This is the type of the receipts that are stored in the file\n * that is used to track the state of the upload process.\n * It is used to resume the upload process from where it left off.\n * It is used to persist the state of the upload process across multiple runs.\n */\nexport const PreferenceUpdateMap = t.record(\n t.string,\n // This can either be true to indicate the record is pending\n // or it can be an object showing the object\n // We only return a fixed number of results to avoid\n // making the JSON file too large\n t.union([t.boolean, PreferenceUpdateItem]),\n);\n\n/** Override type */\nexport type PreferenceUpdateMap = t.TypeOf<typeof PreferenceUpdateMap>;\n\n/**\n * This is the type of the pending updates that are safe to run without\n * conflicts with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is safe to upload.\n */\nexport const PendingSafePreferenceUpdates = t.record(\n t.string,\n // This can either be true to indicate the record is safe\n // or it can be an object showing the object\n // We only return a fixed number of results to avoid\n // making the JSON file too large\n t.union([t.boolean, t.record(t.string, t.string)]),\n);\n\n/** Override type */\nexport type PendingSafePreferenceUpdates = t.TypeOf<\n typeof PendingSafePreferenceUpdates\n>;\n\n/**\n * These are the updates that failed to be uploaded to the API.\n */\nexport const FailingPreferenceUpdates = t.record(\n t.string,\n t.type({\n /** Time upload ran at */\n uploadedAt: t.string,\n /** Attempts to upload that resulted in an error */\n error: t.string,\n /** The update body */\n update: PreferenceUpdateItem,\n }),\n);\n\n/** Override type */\nexport type FailingPreferenceUpdates = t.TypeOf<\n typeof FailingPreferenceUpdates\n>;\n\n/**\n * This is the type of the pending updates that are in conflict with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is pending upload.\n */\nexport const PendingWithConflictPreferenceUpdates = t.record(\n t.string,\n // We always return the conflicts for investigation\n t.type({\n /** Record to be inserted to transcend v1/preferences API */\n record: PreferenceQueryResponseItem,\n /** The row in the file that is pending upload */\n row: t.record(t.string, t.string),\n }),\n);\n\n/** Override type */\nexport type PendingWithConflictPreferenceUpdates = t.TypeOf<\n typeof PendingWithConflictPreferenceUpdates\n>;\n\n/**\n * The set of preference updates that are skipped\n * Key is primaryKey and value is the row in the CSV\n * that is skipped.\n *\n * This is usually because the preferences are already in the store\n * or there are duplicate rows in the CSV file that are identical.\n */\nexport const SkippedPreferenceUpdates = t.record(\n t.string,\n t.record(t.string, t.string),\n);\n\n/** Override type */\nexport type SkippedPreferenceUpdates = t.TypeOf<\n typeof SkippedPreferenceUpdates\n>;\n\n/** Persist this data between runs of the script */\nexport const PreferenceState = t.type({\n /**\n * Store a cache of previous files read in\n */\n fileMetadata: t.record(t.string, FileMetadataState),\n /**\n * The set of successful uploads to Transcend\n * Mapping from userId to the upload metadata\n */\n failingUpdates: t.record(\n t.string,\n t.type({\n /** Time upload ran at */\n uploadedAt: t.string,\n /** Attempts to upload that resulted in an error */\n error: t.string,\n /** The update body */\n update: PreferenceUpdateItem,\n }),\n ),\n /**\n * The set of pending uploads to Transcend\n * Mapping from userId to the upload metadata\n */\n pendingUpdates: t.record(t.string, PreferenceUpdateItem),\n});\n\n/** Override type */\nexport type PreferenceState = t.TypeOf<typeof PreferenceState>;\n"]}
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-TJIMDEBN.cjs","../src/commands/consent/upload-preferences/impl.ts","../src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts","../src/lib/preference-management/parsePreferenceManagementCsv.ts","../src/lib/preference-management/getPreferencesForIdentifiers.ts","../src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts","../src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts","../src/lib/preference-management/codecs.ts"],"names":["PreferenceRecordsQueryResponse","PreferenceQueryResponseItem","MSGS","getPreferencesForIdentifiers","sombra","identifiers","partitionKey","skipLogging","results","groupedIdentifiers","chunk_default","t0","progressBar","cliProgress","total","map","group","attempts","maxAttempts","rawResult","result","decodeCodec","err","msg","errorMessage"],"mappings":"AAAA,u/BAAyC,wDAAwG,gCAA6B,wDAAyC,wDAAuE,gCAA6B,gFCCxS,oCCQC,qGAGI,+DAEO,qJCXZ,2DCHyB,qDAKhB,IAKtBA,EAAAA,CAAmC,CAAA,CAAA,YAAA,CAAa,CAClD,CAAA,CAAA,IAAA,CAAK,CACL,KAAA,CAAS,CAAA,CAAA,KAAA,CAAMC,yCAA2B,CAC5C,CAAC,CAAA,CACC,CAAA,CAAA,OAAA,CAAQ,CAER,MAAA,CAAU,CAAA,CAAA,MACZ,CAAC,CACH,CAAC,CAAA,CAEKC,EAAAA,CAAO,CACX,WAAA,CACA,WAAA,CACA,sBAAA,CACA,sBACF,CAAA,CASA,MAAA,SAAsBC,EAAAA,CACpBC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,CAAA,CAChB,CAAA,CAWwC,CACxC,IAAMC,CAAAA,CAAyC,CAAC,CAAA,CAC1CC,CAAAA,CAAqBC,iCAAAA,CAAML,CAAa,GAAG,CAAA,CAG3CM,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CACxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CACKN,CAAAA,EACHK,CAAAA,CAAY,KAAA,CAAMP,CAAAA,CAAY,MAAA,CAAQ,CAAC,CAAA,CAGzC,IAAIS,CAAAA,CAAQ,CAAA,CACZ,MAAMC,2BAAAA,CACJN,CACA,MAAOO,CAAAA,EAAU,CAEf,IAAIC,CAAAA,CAAW,CAAA,CACTC,CAAAA,CAAc,CAAA,CACpB,GAAA,CAAA,CAAOD,CAAAA,CAAWC,CAAAA,CAAAA,CAChB,GAAI,CACF,IAAMC,CAAAA,CAAY,MAAMf,CAAAA,CACrB,IAAA,CAAK,CAAA,eAAA,EAAkBE,CAAY,CAAA,MAAA,CAAA,CAAU,CAC5C,IAAA,CAAM,CACJ,MAAA,CAAQ,CACN,WAAA,CAAaU,CACf,CAAA,CACA,KAAA,CAAOA,CAAAA,CAAM,MACf,CACF,CAAC,CAAA,CACA,IAAA,CAAK,CAAA,CAEFI,CAAAA,CAASC,oCAAAA,EAAYrB,CAAgCmB,CAAS,CAAA,CACpEX,CAAAA,CAAQ,IAAA,CAAK,GAAGY,CAAAA,CAAO,KAAK,CAAA,CAC5BN,CAAAA,EAASE,CAAAA,CAAM,MAAA,CACfJ,CAAAA,CAAY,MAAA,CAAOE,CAAK,CAAA,CACxB,KACF,CAAA,KAAA,CAASQ,CAAAA,CAAK,CACZL,CAAAA,EAAY,CAAA,CACZ,IAAMM,CAAAA,iBAAMD,CAAAA,6BAAK,QAAA,6BAAU,MAAA,kBAAQA,CAAAA,6BAAK,SAAA,EAAW,EAAA,CACnD,EAAA,CACEL,CAAAA,EAAYC,CAAAA,EACZ,CAAChB,EAAAA,CAAK,IAAA,CAAMsB,CAAAA,EAAiBD,CAAAA,CAAI,QAAA,CAASC,CAAY,CAAC,CAAA,CAEvD,MAAM,IAAI,KAAA,CACR,CAAA,oCAAA,EAAuCP,CAAQ,CAAA,WAAA,EAAcM,CAAG,CAAA,CAAA;ACclE;ACHS;ACkFjB,oGAAA;ALrGI;AAGA;AAAA;AAKQ;AAAK;AACgC,sCAAA;AAmIpB","file":"/home/runner/work/cli/cli/dist/impl-TJIMDEBN.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport colors from 'colors';\n\nimport { logger } from '../../../logger';\nimport { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management';\nimport { splitCsvToList } from '../../../lib/requests';\nimport { readdirSync } from 'node:fs';\nimport { map } from 'bluebird';\nimport { basename, join } from 'node:path';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadPreferencesCommandFlags {\n auth: string;\n partition: string;\n sombraAuth?: string;\n transcendUrl: string;\n file?: string;\n directory?: string;\n dryRun: boolean;\n skipExistingRecordCheck: boolean;\n receiptFileDir: string;\n skipWorkflowTriggers: boolean;\n forceTriggerWorkflows: boolean;\n skipConflictUpdates: boolean;\n isSilent: boolean;\n attributes: string;\n receiptFilepath: string;\n concurrency: number;\n}\n\nexport async function uploadPreferences(\n this: LocalContext,\n {\n auth,\n partition,\n sombraAuth,\n transcendUrl,\n file = '',\n directory,\n dryRun,\n skipExistingRecordCheck,\n receiptFileDir,\n skipWorkflowTriggers,\n forceTriggerWorkflows,\n skipConflictUpdates,\n isSilent,\n attributes,\n concurrency,\n }: UploadPreferencesCommandFlags,\n): Promise<void> {\n if (!!directory && !!file) {\n logger.error(\n colors.red(\n 'Cannot provide both a directory and a file. Please provide only one.',\n ),\n );\n this.process.exit(1);\n }\n\n if (!file && !directory) {\n logger.error(\n colors.red(\n 'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences',\n ),\n );\n this.process.exit(1);\n }\n\n doneInputValidation(this.process.exit);\n\n const files: string[] = [];\n\n if (directory) {\n try {\n const filesInDirectory = readdirSync(directory);\n const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv'));\n\n if (csvFiles.length === 0) {\n logger.error(\n colors.red(`No CSV files found in directory: ${directory}`),\n );\n this.process.exit(1);\n }\n\n // Add full paths for each CSV file\n files.push(...csvFiles.map((file) => join(directory, file)));\n } catch (err) {\n logger.error(colors.red(`Failed to read directory: ${directory}`));\n logger.error(colors.red((err as Error).message));\n this.process.exit(1);\n }\n } else {\n try {\n // Verify file exists and is a CSV\n if (!file.endsWith('.csv')) {\n logger.error(colors.red('File must be a CSV file'));\n this.process.exit(1);\n }\n files.push(file);\n } catch (err) {\n logger.error(colors.red(`Failed to access file: ${file}`));\n logger.error(colors.red((err as Error).message));\n this.process.exit(1);\n }\n }\n\n logger.info(\n colors.green(\n `Processing ${files.length} consent preferences files for partition: ${partition}`,\n ),\n );\n logger.debug(`Files to process: ${files.join(', ')}`);\n\n if (skipExistingRecordCheck) {\n logger.info(\n colors.bgYellow(\n `Skipping existing record check: ${skipExistingRecordCheck}`,\n ),\n );\n }\n\n await map(\n files,\n async (filePath) => {\n const fileName = basename(filePath).replace('.csv', '');\n await uploadPreferenceManagementPreferencesInteractive({\n receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`),\n auth,\n sombraAuth,\n file: filePath,\n partition,\n transcendUrl,\n skipConflictUpdates,\n skipWorkflowTriggers,\n skipExistingRecordCheck,\n isSilent,\n dryRun,\n attributes: splitCsvToList(attributes),\n forceTriggerWorkflows,\n });\n },\n { concurrency },\n );\n}\n","import {\n buildTranscendGraphQLClient,\n createSombraGotInstance,\n fetchAllPurposes,\n fetchAllPreferenceTopics,\n PreferenceTopic,\n Purpose,\n} from '../graphql';\nimport colors from 'colors';\nimport { map } from 'bluebird';\nimport { chunk } from 'lodash-es';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { parseAttributesFromString } from '../requests';\nimport { PersistedState } from '@transcend-io/persisted-state';\nimport { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv';\nimport { PreferenceState } from './codecs';\nimport { PreferenceUpdateItem } from '@transcend-io/privacy-types';\nimport { apply } from '@transcend-io/type-utils';\nimport { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\n\n/**\n * Upload a set of consent preferences\n *\n * @param options - Options\n */\nexport async function uploadPreferenceManagementPreferencesInteractive({\n auth,\n sombraAuth,\n receiptFilepath,\n file,\n partition,\n isSilent = true,\n dryRun = false,\n skipWorkflowTriggers = false,\n skipConflictUpdates = false,\n skipExistingRecordCheck = false,\n attributes = [],\n transcendUrl,\n forceTriggerWorkflows = false,\n}: {\n /** The Transcend API key */\n auth: string;\n /** Sombra API key authentication */\n sombraAuth?: string;\n /** Partition key */\n partition: string;\n /** File where to store receipt and continue from where left off */\n receiptFilepath: string;\n /** The file to process */\n file: string;\n /** API URL for Transcend backend */\n transcendUrl: string;\n /** Whether to do a dry run */\n dryRun?: boolean;\n /** Whether to upload as isSilent */\n isSilent?: boolean;\n /** Attributes string pre-parse. In format Key:Value */\n attributes?: string[];\n /** Skip workflow triggers */\n skipWorkflowTriggers?: boolean;\n /**\n * When true, only update preferences that do not conflict with existing\n * preferences. When false, update all preferences in CSV based on timestamp.\n */\n skipConflictUpdates?: boolean;\n /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n skipExistingRecordCheck?: boolean;\n /** Whether to force trigger workflows */\n forceTriggerWorkflows?: boolean;\n}): Promise<void> {\n // Parse out the extra attributes to apply to all requests uploaded\n const parsedAttributes = parseAttributesFromString(attributes);\n\n // Create a new state file to store the requests from this run\n const preferenceState = new PersistedState(receiptFilepath, PreferenceState, {\n fileMetadata: {},\n failingUpdates: {},\n pendingUpdates: {},\n });\n const failingRequests = preferenceState.getValue('failingUpdates');\n const pendingRequests = preferenceState.getValue('pendingUpdates');\n let fileMetadata = preferenceState.getValue('fileMetadata');\n\n logger.info(\n colors.magenta(\n 'Restored cache, there are: \\n' +\n `${\n Object.values(failingRequests).length\n } failing requests to be retried\\n` +\n `${\n Object.values(pendingRequests).length\n } pending requests to be processed\\n` +\n `The following files are stored in cache and will be used:\\n${Object.keys(\n fileMetadata,\n )\n .map((x) => x)\n .join('\\n')}\\n` +\n `The following file will be processed: ${file}\\n`,\n ),\n );\n\n // Create GraphQL client to connect to Transcend backend\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n const [sombra, purposes, preferenceTopics] = await Promise.all([\n // Create sombra instance to communicate with\n createSombraGotInstance(transcendUrl, auth, sombraAuth),\n // get all purposes and topics\n forceTriggerWorkflows\n ? Promise.resolve([] as Purpose[])\n : fetchAllPurposes(client),\n forceTriggerWorkflows\n ? Promise.resolve([] as PreferenceTopic[])\n : fetchAllPreferenceTopics(client),\n ]);\n\n // Process the file\n await parsePreferenceManagementCsvWithCache(\n {\n file,\n purposeSlugs: purposes.map((x) => x.trackingType),\n preferenceTopics,\n sombra,\n partitionKey: partition,\n skipExistingRecordCheck,\n forceTriggerWorkflows,\n },\n preferenceState,\n );\n\n // Construct the pending updates\n const pendingUpdates: Record<string, PreferenceUpdateItem> = {};\n fileMetadata = preferenceState.getValue('fileMetadata');\n const metadata = fileMetadata[file];\n\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.pendingSafeUpdates).length\n } safe updates in ${file}`,\n ),\n );\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.pendingConflictUpdates).length\n } conflict updates in ${file}`,\n ),\n );\n logger.info(\n colors.magenta(\n `Found ${\n Object.entries(metadata.skippedUpdates).length\n } skipped updates in ${file}`,\n ),\n );\n\n // Update either safe updates only or safe + conflict\n Object.entries({\n ...metadata.pendingSafeUpdates,\n ...(skipConflictUpdates\n ? {}\n : apply(metadata.pendingConflictUpdates, ({ row }) => row)),\n }).forEach(([userId, update]) => {\n // Determine timestamp\n const timestamp =\n metadata.timestampColum === NONE_PREFERENCE_MAP\n ? new Date()\n : new Date(update[metadata.timestampColum!]);\n\n // Determine updates\n const updates = getPreferenceUpdatesFromRow({\n row: update,\n columnToPurposeName: metadata.columnToPurposeName,\n preferenceTopics,\n purposeSlugs: purposes.map((x) => x.trackingType),\n });\n pendingUpdates[userId] = {\n userId,\n partition,\n timestamp: timestamp.toISOString(),\n purposes: Object.entries(updates).map(([purpose, value]) => ({\n ...value,\n purpose,\n workflowSettings: {\n attributes: parsedAttributes,\n isSilent,\n skipWorkflowTrigger: skipWorkflowTriggers,\n },\n })),\n };\n });\n await preferenceState.setValue(pendingUpdates, 'pendingUpdates');\n await preferenceState.setValue({}, 'failingUpdates');\n\n // Exist early if dry run\n if (dryRun) {\n logger.info(\n colors.green(\n `Dry run complete, exiting. ${\n Object.values(pendingUpdates).length\n } pending updates. Check file: ${receiptFilepath}`,\n ),\n );\n return;\n }\n\n logger.info(\n colors.magenta(\n `Uploading ${\n Object.values(pendingUpdates).length\n } preferences to partition: ${partition}`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n\n // Build a GraphQL client\n let total = 0;\n const updatesToRun = Object.entries(pendingUpdates);\n const chunkedUpdates = chunk(updatesToRun, skipWorkflowTriggers ? 100 : 10);\n progressBar.start(updatesToRun.length, 0);\n await map(\n chunkedUpdates,\n async (currentChunk) => {\n // Make the request\n try {\n await sombra\n .put('v1/preferences', {\n json: {\n records: currentChunk.map(([, update]) => update),\n skipWorkflowTriggers,\n forceTriggerWorkflows,\n },\n })\n .json();\n } catch (err) {\n try {\n const parsed = JSON.parse(err?.response?.body || '{}');\n if (parsed.error) {\n logger.error(colors.red(`Error: ${parsed.error}`));\n }\n } catch (e) {\n // continue\n }\n logger.error(\n colors.red(\n `Failed to upload ${\n currentChunk.length\n } user preferences to partition ${partition}: ${\n err?.response?.body || err?.message\n }`,\n ),\n );\n const failingUpdates = preferenceState.getValue('failingUpdates');\n currentChunk.forEach(([userId, update]) => {\n failingUpdates[userId] = {\n uploadedAt: new Date().toISOString(),\n update,\n error: err?.response?.body || err?.message || 'Unknown error',\n };\n });\n await preferenceState.setValue(failingUpdates, 'failingUpdates');\n }\n\n total += currentChunk.length;\n progressBar.update(total);\n },\n {\n concurrency: 40,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n logger.info(\n colors.green(\n `Successfully uploaded ${\n updatesToRun.length\n } user preferences to partition ${partition} in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n}\n","import { PersistedState } from '@transcend-io/persisted-state';\nimport type { Got } from 'got';\nimport { keyBy } from 'lodash-es';\nimport * as t from 'io-ts';\nimport colors from 'colors';\nimport { FileMetadataState, PreferenceState } from './codecs';\nimport { logger } from '../../logger';\nimport { readCsv } from '../requests';\nimport { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers';\nimport { PreferenceTopic } from '../graphql';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\nimport { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv';\nimport { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv';\nimport { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv';\nimport { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp';\nimport { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict';\n\n/**\n * Parse a file into the cache\n *\n *\n * @param options - Options\n * @param cache - The cache to store the parsed file in\n * @returns The cache with the parsed file\n */\nexport async function parsePreferenceManagementCsvWithCache(\n {\n file,\n sombra,\n purposeSlugs,\n preferenceTopics,\n partitionKey,\n skipExistingRecordCheck,\n forceTriggerWorkflows,\n }: {\n /** File to parse */\n file: string;\n /** The purpose slugs that are allowed to be updated */\n purposeSlugs: string[];\n /** The preference topics */\n preferenceTopics: PreferenceTopic[];\n /** Sombra got instance */\n sombra: Got;\n /** Partition key */\n partitionKey: string;\n /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n skipExistingRecordCheck: boolean;\n /** Wheather to force workflow triggers */\n forceTriggerWorkflows: boolean;\n },\n cache: PersistedState<typeof PreferenceState>,\n): Promise<void> {\n // Start the timer\n const t0 = new Date().getTime();\n\n // Get the current metadata\n const fileMetadata = cache.getValue('fileMetadata');\n\n // Read in the file\n logger.info(colors.magenta(`Reading in file: \"${file}\"`));\n let preferences = readCsv(file, t.record(t.string, t.string));\n\n // start building the cache, can use previous cache as well\n let currentState: FileMetadataState = {\n columnToPurposeName: {},\n pendingSafeUpdates: {},\n pendingConflictUpdates: {},\n skippedUpdates: {},\n // Load in the last fetched time\n ...((fileMetadata[file] || {}) as Partial<FileMetadataState>),\n lastFetchedAt: new Date().toISOString(),\n };\n\n // Validate that all timestamps are present in the file\n currentState = await parsePreferenceTimestampsFromCsv(\n preferences,\n currentState,\n );\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Validate that all identifiers are present and unique\n const result = await parsePreferenceIdentifiersFromCsv(\n preferences,\n currentState,\n );\n currentState = result.currentState;\n preferences = result.preferences;\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Ensure all other columns are mapped to purpose and preference\n // slug values\n currentState = await parsePreferenceAndPurposeValuesFromCsv(\n preferences,\n currentState,\n {\n preferenceTopics,\n purposeSlugs,\n forceTriggerWorkflows,\n },\n );\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n\n // Grab existing preference store records\n const identifiers = preferences.map(\n (pref) => pref[currentState.identifierColumn!],\n );\n const existingConsentRecords = skipExistingRecordCheck\n ? []\n : await getPreferencesForIdentifiers(sombra, {\n identifiers: identifiers.map((x) => ({ value: x })),\n partitionKey,\n });\n const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId');\n\n // Clear out previous updates\n currentState.pendingConflictUpdates = {};\n currentState.pendingSafeUpdates = {};\n currentState.skippedUpdates = {};\n\n // Process each row\n preferences.forEach((pref) => {\n // Grab unique Id for the user\n const userId = pref[currentState.identifierColumn!];\n\n // determine updates for user\n const pendingUpdates = getPreferenceUpdatesFromRow({\n row: pref,\n columnToPurposeName: currentState.columnToPurposeName,\n preferenceTopics,\n purposeSlugs,\n });\n\n // Grab current state of the update\n const currentConsentRecord = consentRecordByIdentifier[userId];\n if (forceTriggerWorkflows && !currentConsentRecord) {\n throw new Error(\n `No existing consent record found for user with id: ${userId}. \n When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`,\n );\n }\n // Check if the update can be skipped\n // this is the case if a record exists, and the purpose\n // and preference values are all in sync\n if (\n currentConsentRecord &&\n checkIfPendingPreferenceUpdatesAreNoOp({\n currentConsentRecord,\n pendingUpdates,\n preferenceTopics,\n }) &&\n !forceTriggerWorkflows\n ) {\n currentState.skippedUpdates[userId] = pref;\n return;\n }\n\n // Determine if there are any conflicts\n if (\n currentConsentRecord &&\n checkIfPendingPreferenceUpdatesCauseConflict({\n currentConsentRecord,\n pendingUpdates,\n preferenceTopics,\n })\n ) {\n currentState.pendingConflictUpdates[userId] = {\n row: pref,\n record: currentConsentRecord,\n };\n return;\n }\n\n // Add to pending updates\n currentState.pendingSafeUpdates[userId] = pref;\n });\n\n // Read in the file\n fileMetadata[file] = currentState;\n await cache.setValue(fileMetadata, 'fileMetadata');\n const t1 = new Date().getTime();\n logger.info(\n colors.green(\n `Successfully pre-processed file: \"${file}\" in ${(t1 - t0) / 1000}s`,\n ),\n );\n}\n","import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\nimport colors from 'colors';\nimport cliProgress from 'cli-progress';\nimport { chunk } from 'lodash-es';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport * as t from 'io-ts';\nimport { map } from 'bluebird';\nimport { logger } from '../../logger';\n\nconst PreferenceRecordsQueryResponse = t.intersection([\n t.type({\n nodes: t.array(PreferenceQueryResponseItem),\n }),\n t.partial({\n /** The base64 encoded(PreferenceStorePaginationKey) cursor for pagination */\n cursor: t.string,\n }),\n]);\n\nconst MSGS = [\n 'ENOTFOUND',\n 'ETIMEDOUT',\n '504 Gateway Time-out',\n 'Task timed out after',\n];\n\n/**\n * Grab the current consent preference values for a list of identifiers\n *\n * @param sombra - Backend to make API call to\n * @param options - Options\n * @returns Plaintext context information\n */\nexport async function getPreferencesForIdentifiers(\n sombra: Got,\n {\n identifiers,\n partitionKey,\n skipLogging = false,\n }: {\n /** The list of identifiers to look up */\n identifiers: {\n /** The value of the identifier */\n value: string;\n }[];\n /** The partition key to look up */\n partitionKey: string;\n /** Whether to skip logging */\n skipLogging?: boolean;\n },\n): Promise<PreferenceQueryResponseItem[]> {\n const results: PreferenceQueryResponseItem[] = [];\n const groupedIdentifiers = chunk(identifiers, 100);\n\n // create a new progress bar instance and use shades_classic theme\n const t0 = new Date().getTime();\n const progressBar = new cliProgress.SingleBar(\n {},\n cliProgress.Presets.shades_classic,\n );\n if (!skipLogging) {\n progressBar.start(identifiers.length, 0);\n }\n\n let total = 0;\n await map(\n groupedIdentifiers,\n async (group) => {\n // Make the request with retry logic\n let attempts = 0;\n const maxAttempts = 3;\n while (attempts < maxAttempts) {\n try {\n const rawResult = await sombra\n .post(`v1/preferences/${partitionKey}/query`, {\n json: {\n filter: {\n identifiers: group,\n },\n limit: group.length,\n },\n })\n .json();\n\n const result = decodeCodec(PreferenceRecordsQueryResponse, rawResult);\n results.push(...result.nodes);\n total += group.length;\n progressBar.update(total);\n break; // Exit loop if successful\n } catch (err) {\n attempts += 1;\n const msg = err?.response?.body || err?.message || '';\n if (\n attempts >= maxAttempts ||\n !MSGS.some((errorMessage) => msg.includes(errorMessage))\n ) {\n throw new Error(\n `Received an error from server after ${attempts} attempts: ${msg}`,\n );\n }\n\n logger.warn(\n colors.yellow(\n `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` +\n `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`,\n ),\n );\n }\n }\n },\n {\n concurrency: 40,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n if (!skipLogging) {\n // Log completion time\n logger.info(\n colors.green(`Completed download in \"${totalTime / 1000}\" seconds.`),\n );\n }\n\n return results;\n}\n","import { uniq, groupBy, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { inquirerConfirmBoolean } from '../helpers';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse identifiers from a CSV list of preferences\n *\n * Ensures that all rows have a valid identifier\n * and that all identifiers are unique.\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceIdentifiersFromCsv(\n preferences: Record<string, string>[],\n currentState: FileMetadataState,\n): Promise<{\n /** The updated state */\n currentState: FileMetadataState;\n /** The updated preferences */\n preferences: Record<string, string>[];\n}> {\n // Determine columns to map\n const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n // Determine the columns that could potentially be used for identifier\n const remainingColumnsForIdentifier = difference(columnNames, [\n ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n ...Object.keys(currentState.columnToPurposeName),\n ]);\n\n // Determine the identifier column to work off of\n if (!currentState.identifierColumn) {\n const { identifierName } = await inquirer.prompt<{\n /** Identifier name */\n identifierName: string;\n }>([\n {\n name: 'identifierName',\n message:\n 'Choose the column that will be used as the identifier to upload consent preferences by',\n type: 'list',\n default:\n remainingColumnsForIdentifier.find((col) =>\n col.toLowerCase().includes('email'),\n ) || remainingColumnsForIdentifier[0],\n choices: remainingColumnsForIdentifier,\n },\n ]);\n currentState.identifierColumn = identifierName;\n }\n logger.info(\n colors.magenta(\n `Using identifier column \"${currentState.identifierColumn}\"`,\n ),\n );\n\n // Validate that the identifier column is present for all rows and unique\n const identifierColumnsMissing = preferences\n .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind]))\n .filter((x): x is number[] => !!x)\n .flat();\n if (identifierColumnsMissing.length > 0) {\n const msg = `The identifier column \"${\n currentState.identifierColumn\n }\" is missing a value for the following rows: ${identifierColumnsMissing.join(\n ', ',\n )}`;\n logger.warn(colors.yellow(msg));\n\n // Ask user if they would like to skip rows missing an identifier\n const skip = await inquirerConfirmBoolean({\n message: 'Would you like to skip rows missing an identifier?',\n });\n if (!skip) {\n throw new Error(msg);\n }\n\n // Filter out rows missing an identifier\n const previous = preferences.length;\n preferences = preferences.filter(\n (pref) => pref[currentState.identifierColumn!],\n );\n logger.info(\n colors.yellow(\n `Skipped ${previous - preferences.length} rows missing an identifier`,\n ),\n );\n }\n logger.info(\n colors.magenta(\n `The identifier column \"${currentState.identifierColumn}\" is present for all rows`,\n ),\n );\n\n // Validate that all identifiers are unique\n const rowsByUserId = groupBy(preferences, currentState.identifierColumn);\n const duplicateIdentifiers = Object.entries(rowsByUserId).filter(\n ([, rows]) => rows.length > 1,\n );\n if (duplicateIdentifiers.length > 0) {\n const msg = `The identifier column \"${\n currentState.identifierColumn\n }\" has duplicate values for the following rows: ${duplicateIdentifiers\n .slice(0, 10)\n .map(([userId, rows]) => `${userId} (${rows.length})`)\n .join('\\n')}`;\n logger.warn(colors.yellow(msg));\n\n // Ask user if they would like to take the most recent update\n // for each duplicate identifier\n const skip = await inquirerConfirmBoolean({\n message: 'Would you like to automatically take the latest update?',\n });\n if (!skip) {\n throw new Error(msg);\n }\n preferences = Object.entries(rowsByUserId)\n .map(([, rows]) => {\n const sorted = rows.sort(\n (a, b) =>\n new Date(b[currentState.timestampColum!]).getTime() -\n new Date(a[currentState.timestampColum!]).getTime(),\n );\n return sorted[0];\n })\n .filter((x) => x);\n }\n\n return { currentState, preferences };\n}\n/* eslint-enable no-param-reassign */\n","import { uniq, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { mapSeries } from 'bluebird';\nimport { PreferenceTopic } from '../graphql';\nimport { PreferenceTopicType } from '@transcend-io/privacy-types';\nimport { splitCsvToList } from '../requests';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse out the purpose.enabled and preference values from a CSV file\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @param options - Options\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceAndPurposeValuesFromCsv(\n preferences: Record<string, string>[],\n currentState: FileMetadataState,\n {\n purposeSlugs,\n preferenceTopics,\n forceTriggerWorkflows,\n }: {\n /** The purpose slugs that are allowed to be updated */\n purposeSlugs: string[];\n /** The preference topics */\n preferenceTopics: PreferenceTopic[];\n /** Force workflow triggers */\n forceTriggerWorkflows: boolean;\n },\n): Promise<FileMetadataState> {\n // Determine columns to map\n const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n // Determine the columns that could potentially be used for identifier\n const otherColumns = difference(columnNames, [\n ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n ...(currentState.timestampColum ? [currentState.timestampColum] : []),\n ]);\n if (otherColumns.length === 0) {\n if (forceTriggerWorkflows) {\n return currentState;\n }\n throw new Error('No other columns to process');\n }\n\n // The purpose and preferences to map to\n const purposeNames = [\n ...purposeSlugs,\n ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`),\n ];\n\n // Ensure all columns are accounted for\n await mapSeries(otherColumns, async (col) => {\n // Determine the unique values to map in this column\n const uniqueValues = uniq(preferences.map((x) => x[col]));\n\n // Map the column to a purpose\n let purposeMapping = currentState.columnToPurposeName[col];\n if (purposeMapping) {\n logger.info(\n colors.magenta(\n `Column \"${col}\" is associated with purpose \"${purposeMapping.purpose}\"`,\n ),\n );\n } else {\n const { purposeName } = await inquirer.prompt<{\n /** purpose name */\n purposeName: string;\n }>([\n {\n name: 'purposeName',\n message: `Choose the purpose that column ${col} is associated with`,\n type: 'list',\n default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])),\n choices: purposeNames,\n },\n ]);\n const [purposeSlug, preferenceSlug] = purposeName.split('->');\n purposeMapping = {\n purpose: purposeSlug,\n preference: preferenceSlug || null,\n valueMapping: {},\n };\n }\n\n // map each value to the purpose value\n await mapSeries(uniqueValues, async (value) => {\n if (purposeMapping.valueMapping[value] !== undefined) {\n logger.info(\n colors.magenta(\n `Value \"${value}\" is associated with purpose value \"${purposeMapping.valueMapping[value]}\"`,\n ),\n );\n return;\n }\n // if preference is null, this column is just for the purpose\n if (purposeMapping.preference === null) {\n const { purposeValue } = await inquirer.prompt<{\n /** purpose value */\n purposeValue: boolean;\n }>([\n {\n name: 'purposeValue',\n message: `Choose the purpose value for value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'confirm',\n default: value !== 'false',\n },\n ]);\n purposeMapping.valueMapping[value] = purposeValue;\n }\n\n // if preference is not null, this column is for a specific preference\n if (purposeMapping.preference !== null) {\n const preferenceTopic = preferenceTopics.find(\n (x) => x.slug === purposeMapping.preference,\n );\n if (!preferenceTopic) {\n logger.error(\n colors.red(\n `Preference topic \"${purposeMapping.preference}\" not found`,\n ),\n );\n return;\n }\n const preferenceOptions = preferenceTopic.preferenceOptionValues.map(\n ({ slug }) => slug,\n );\n\n if (preferenceTopic.type === PreferenceTopicType.Boolean) {\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n message:\n // eslint-disable-next-line max-len\n `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'confirm',\n default: value !== 'false',\n },\n ]);\n purposeMapping.valueMapping[value] = preferenceValue;\n return;\n }\n\n if (preferenceTopic.type === PreferenceTopicType.Select) {\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n // eslint-disable-next-line max-len\n message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'list',\n choices: preferenceOptions,\n default: preferenceOptions.find((x) => x === value),\n },\n ]);\n purposeMapping.valueMapping[value] = preferenceValue;\n return;\n }\n\n if (preferenceTopic.type === PreferenceTopicType.MultiSelect) {\n const parsedValues = splitCsvToList(value);\n // need to do this serially\n await mapSeries(parsedValues, async (parsedValue) => {\n // if we already have a value, skip re-processing it again\n if (purposeMapping.valueMapping[parsedValue] !== undefined) {\n return;\n }\n const { preferenceValue } = await inquirer.prompt<{\n /** purpose value */\n preferenceValue: boolean;\n }>([\n {\n name: 'preferenceValue',\n // eslint-disable-next-line max-len\n message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${parsedValue}\" associated with purpose \"${purposeMapping.purpose}\"`,\n type: 'list',\n choices: preferenceOptions,\n default: preferenceOptions.find((x) => x === parsedValue),\n },\n ]);\n purposeMapping.valueMapping[parsedValue] = preferenceValue;\n });\n return;\n }\n\n throw new Error(\n `Unknown preference topic type: ${preferenceTopic.type}`,\n );\n }\n });\n\n currentState.columnToPurposeName[col] = purposeMapping;\n });\n\n return currentState;\n}\n/* eslint-enable no-param-reassign */\n","import {\n PreferenceQueryResponseItem,\n PreferenceUpdateItem,\n} from '@transcend-io/privacy-types';\nimport * as t from 'io-ts';\n\nexport const PurposeRowMapping = t.type({\n /**\n * The slug or trackingType of the purpose to map to\n *\n * e.g. `Marketing`\n */\n purpose: t.string,\n /**\n * If the column maps to a preference instead of a purpose\n * this is the slug of the purpose.\n *\n * null value indicates that this column maps to the true/false\n * value of the purpose\n */\n preference: t.union([t.string, t.null]),\n /**\n * The mapping between each row value and purpose/preference value.\n *\n * e.g. for a boolean preference or purpose\n * {\n * 'true': true,\n * 'false': false,\n * '': true,\n * }\n *\n * or for a single or multi select preference\n * {\n * '': true,\n * 'value1': 'Value1',\n * 'value2': 'Value2',\n * }\n */\n valueMapping: t.record(\n t.string,\n t.union([t.string, t.boolean, t.null, t.undefined]),\n ),\n});\n\n/** Override type */\nexport type PurposeRowMapping = t.TypeOf<typeof PurposeRowMapping>;\n\n/**\n * Mapping of column name to purpose row mapping.\n * This is used to map each column in the CSV to the relevant purpose and preference definitions in\n * transcend.\n */\nexport const ColumnPurposeMap = t.record(t.string, PurposeRowMapping);\n\n/** Override type */\nexport type ColumnPurposeMap = t.TypeOf<typeof ColumnPurposeMap>;\n\nexport const IdentifierMetadataForPreference = t.type({\n /** The identifier name */\n name: t.string,\n /** Is unique on preference store */\n isUniqueOnPreferenceStore: t.boolean,\n});\n\n/** Override type */\nexport type IdentifierMetadataForPreference = t.TypeOf<\n typeof IdentifierMetadataForPreference\n>;\n\n/**\n * Mapping of identifier name to the column name in the CSV file.\n * This is used to map each identifier name to the column in the CSV file.\n */\nexport const ColumnIdentifierMap = t.record(\n t.string,\n IdentifierMetadataForPreference,\n);\n\n/** Override type */\nexport type ColumnIdentifierMap = t.TypeOf<typeof ColumnIdentifierMap>;\n\nexport const FileMetadataState = t.intersection([\n t.type({\n /**\n * Definition of how to map each column in the CSV to\n * the relevant purpose and preference definitions in transcend\n */\n columnToPurposeName: t.record(t.string, PurposeRowMapping),\n /** Last time the file was last parsed at */\n lastFetchedAt: t.string,\n /**\n * Mapping of userId to the rows in the file that need to be uploaded\n * These uploads are overwriting non-existent preferences and are safe\n */\n pendingSafeUpdates: t.record(t.string, t.record(t.string, t.string)),\n /**\n * Mapping of userId to the rows in the file that need to be uploaded\n * these records have conflicts with existing consent preferences\n */\n pendingConflictUpdates: t.record(\n t.string,\n t.type({\n record: PreferenceQueryResponseItem,\n row: t.record(t.string, t.string),\n }),\n ),\n /**\n * Mapping of userId to the rows in the file that can be skipped because\n * their preferences are already in the store\n */\n skippedUpdates: t.record(t.string, t.record(t.string, t.string)),\n }),\n t.partial({\n /** Determine which column name in file maps to consent record identifier to upload on */\n identifierColumn: t.string,\n /** Determine which column name in file maps to the timestamp */\n timestampColum: t.string,\n }),\n]);\n\n/** Override type */\nexport type FileMetadataState = t.TypeOf<typeof FileMetadataState>;\n\n/**\n * This is the type of the receipts that are stored in the file\n * that is used to track the state of the upload process.\n * It is used to resume the upload process from where it left off.\n * It is used to persist the state of the upload process across multiple runs.\n */\nexport const PreferenceUpdateMap = t.record(\n t.string,\n // This can either be true to indicate the record is pending\n // or it can be an object showing the object\n // We only return a fixed number of results to avoid\n // making the JSON file too large\n t.union([t.boolean, PreferenceUpdateItem]),\n);\n\n/** Override type */\nexport type PreferenceUpdateMap = t.TypeOf<typeof PreferenceUpdateMap>;\n\n/**\n * This is the type of the pending updates that are safe to run without\n * conflicts with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is safe to upload.\n */\nexport const PendingSafePreferenceUpdates = t.record(\n t.string,\n // This can either be true to indicate the record is safe\n // or it can be an object showing the object\n // We only return a fixed number of results to avoid\n // making the JSON file too large\n t.union([t.boolean, t.record(t.string, t.string)]),\n);\n\n/** Override type */\nexport type PendingSafePreferenceUpdates = t.TypeOf<\n typeof PendingSafePreferenceUpdates\n>;\n\n/**\n * These are the updates that failed to be uploaded to the API.\n */\nexport const FailingPreferenceUpdates = t.record(\n t.string,\n t.type({\n /** Time upload ran at */\n uploadedAt: t.string,\n /** Attempts to upload that resulted in an error */\n error: t.string,\n /** The update body */\n update: PreferenceUpdateItem,\n }),\n);\n\n/** Override type */\nexport type FailingPreferenceUpdates = t.TypeOf<\n typeof FailingPreferenceUpdates\n>;\n\n/**\n * This is the type of the pending updates that are in conflict with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is pending upload.\n */\nexport const PendingWithConflictPreferenceUpdates = t.record(\n t.string,\n // We always return the conflicts for investigation\n t.type({\n /** Record to be inserted to transcend v1/preferences API */\n record: PreferenceQueryResponseItem,\n /** The row in the file that is pending upload */\n row: t.record(t.string, t.string),\n }),\n);\n\n/** Override type */\nexport type PendingWithConflictPreferenceUpdates = t.TypeOf<\n typeof PendingWithConflictPreferenceUpdates\n>;\n\n/**\n * The set of preference updates that are skipped\n * Key is primaryKey and value is the row in the CSV\n * that is skipped.\n *\n * This is usually because the preferences are already in the store\n * or there are duplicate rows in the CSV file that are identical.\n */\nexport const SkippedPreferenceUpdates = t.record(\n t.string,\n t.record(t.string, t.string),\n);\n\n/** Override type */\nexport type SkippedPreferenceUpdates = t.TypeOf<\n typeof SkippedPreferenceUpdates\n>;\n\n/** Persist this data between runs of the script */\nexport const PreferenceState = t.type({\n /**\n * Store a cache of previous files read in\n */\n fileMetadata: t.record(t.string, FileMetadataState),\n /**\n * The set of successful uploads to Transcend\n * Mapping from userId to the upload metadata\n */\n failingUpdates: t.record(\n t.string,\n t.type({\n /** Time upload ran at */\n uploadedAt: t.string,\n /** Attempts to upload that resulted in an error */\n error: t.string,\n /** The update body */\n update: PreferenceUpdateItem,\n }),\n ),\n /**\n * The set of pending uploads to Transcend\n * Mapping from userId to the upload metadata\n */\n pendingUpdates: t.record(t.string, PreferenceUpdateItem),\n});\n\n/** Override type */\nexport type PreferenceState = t.TypeOf<typeof PreferenceState>;\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function a({auth:o,transcendUrl:r,enricherIds:e}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.Lf.call(void 0, {transcendUrl:r,auth:o,enricherIds:e})}exports.skipPreflightJobs = a;
|
|
2
|
+
//# sourceMappingURL=impl-TXQ7EFQE.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-TXQ7EFQE.cjs","../src/commands/request/skip-preflight-jobs/impl.ts"],"names":["skipPreflightJobs","auth","transcendUrl","enricherIds","doneInputValidation"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCUtM,SAAsBA,CAAAA,CAEpB,CAAE,IAAA,CAAAC,CAAAA,CAAM,YAAA,CAAAC,CAAAA,CAAc,WAAA,CAAAC,CAAY,CAAA,CACnB,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMJ,kCAAAA,CACJ,YAAA,CAAAE,CAAAA,CACA,IAAA,CAAAD,CAAAA,CACA,WAAA,CAAAE,CACF,CAAC,CACH,CAAA,8BAAA","file":"/home/runner/work/cli/cli/dist/impl-TXQ7EFQE.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { skipPreflightJobs as skipPreflightJobsHelper } from '../../../lib/requests';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface SkipPreflightJobsCommandFlags {\n auth: string;\n enricherIds: string[];\n transcendUrl: string;\n}\n\nexport async function skipPreflightJobs(\n this: LocalContext,\n { auth, transcendUrl, enricherIds }: SkipPreflightJobsCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await skipPreflightJobsHelper({\n transcendUrl,\n auth,\n enricherIds,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkZUT7UIQUcjs = require('./chunk-ZUT7UIQU.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function g({base64EncryptionKey:t,base64SigningKey:s,partition:i,file:a,consentUrl:c,concurrency:p}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let m=_chunkKKAJPI6Ocjs.rc.call(void 0, a,_chunkZUT7UIQUcjs.b);await _chunkZUT7UIQUcjs.f.call(void 0, {base64EncryptionKey:t,base64SigningKey:s,preferences:m,partition:i,concurrency:p,transcendUrl:c})}exports.uploadConsentPreferences = g;
|
|
2
|
+
//# sourceMappingURL=impl-VKEICKZ7.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-VKEICKZ7.cjs","../src/commands/consent/upload-consent-preferences/impl.ts"],"names":["uploadConsentPreferences","base64EncryptionKey","base64SigningKey","partition","file","consentUrl","concurrency","doneInputValidation","preferences","readCsv","ConsentPreferenceUpload","uploadConsents"],"mappings":"AAAA,iIAA+C,wDAAyC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCgBtP,SAAsBA,CAAAA,CAEpB,CACE,mBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAGrC,IAAMC,CAAAA,CAAcC,kCAAAA,CAAQL,CAAMM,mBAAuB,CAAA,CAGzD,MAAMC,iCAAAA,CACJ,mBAAA,CAAAV,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,WAAA,CAAAM,CAAAA,CACA,SAAA,CAAAL,CAAAA,CACA,WAAA,CAAAG,CAAAA,CACA,YAAA,CAAcD,CAChB,CAAC,CACH,CAAA,qCAAA","file":"/home/runner/work/cli/cli/dist/impl-VKEICKZ7.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\n\nimport { uploadConsents } from '../../../lib/consent-manager/uploadConsents';\nimport { ConsentPreferenceUpload } from '../../../lib/consent-manager/types';\nimport { readCsv } from '../../../lib/requests';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadConsentPreferencesCommandFlags {\n base64EncryptionKey: string;\n base64SigningKey: string;\n partition: string;\n file: string;\n consentUrl: string;\n concurrency: number;\n}\n\nexport async function uploadConsentPreferences(\n this: LocalContext,\n {\n base64EncryptionKey,\n base64SigningKey,\n partition,\n file,\n consentUrl,\n concurrency,\n }: UploadConsentPreferencesCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n // Load in preferences from csv\n const preferences = readCsv(file, ConsentPreferenceUpload);\n\n // Upload cookies\n await uploadConsents({\n base64EncryptionKey,\n base64SigningKey,\n preferences,\n partition,\n concurrency,\n transcendUrl: consentUrl,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function h({auth:i,enricherId:s,actions:o,requestEnricherStatuses:a,requestIds:c,createdAtBefore:e,createdAtAfter:t,concurrency:u,transcendUrl:d}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.Mf.call(void 0, {auth:i,enricherId:s,requestActions:o,requestEnricherStatuses:a,requestIds:c,createdAtBefore:e?new Date(e):void 0,createdAtAfter:t?new Date(t):void 0,concurrency:u,transcendUrl:d})}exports.enricherRestart = h;
|
|
2
|
+
//# sourceMappingURL=impl-VMV24ZJH.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-VMV24ZJH.cjs","../src/commands/request/enricher-restart/impl.ts"],"names":["enricherRestart","auth","enricherId","actions","requestEnricherStatuses","requestIds","createdAtBefore","createdAtAfter","concurrency","transcendUrl","doneInputValidation","bulkRetryEnrichers"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCoBtM,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,uBAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMC,kCAAAA,CACJ,IAAA,CAAAV,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,cAAA,CAAgBC,CAAAA,CAChB,uBAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,eAAA,CAAiBC,CAAAA,CAAkB,IAAI,IAAA,CAAKA,CAAe,CAAA,CAAI,KAAA,CAAA,CAC/D,cAAA,CAAgBC,CAAAA,CAAiB,IAAI,IAAA,CAAKA,CAAc,CAAA,CAAI,KAAA,CAAA,CAC5D,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CACF,CAAC,CACH,CAAA,4BAAA","file":"/home/runner/work/cli/cli/dist/impl-VMV24ZJH.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { bulkRetryEnrichers } from '../../../lib/requests';\nimport type {\n RequestAction,\n RequestEnricherStatus,\n} from '@transcend-io/privacy-types';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface EnricherRestartCommandFlags {\n auth: string;\n enricherId: string;\n actions?: RequestAction[];\n requestEnricherStatuses?: RequestEnricherStatus[];\n transcendUrl: string;\n concurrency: number;\n requestIds?: string[];\n createdAtBefore?: Date;\n createdAtAfter?: Date;\n}\n\nexport async function enricherRestart(\n this: LocalContext,\n {\n auth,\n enricherId,\n actions,\n requestEnricherStatuses,\n requestIds,\n createdAtBefore,\n createdAtAfter,\n concurrency,\n transcendUrl,\n }: EnricherRestartCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await bulkRetryEnrichers({\n auth,\n enricherId,\n requestActions: actions,\n requestEnricherStatuses,\n requestIds,\n createdAtBefore: createdAtBefore ? new Date(createdAtBefore) : undefined,\n createdAtAfter: createdAtAfter ? new Date(createdAtAfter) : undefined,\n concurrency,\n transcendUrl,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function l({auth:i,transcendUrl:o,createdAtBefore:n,createdAtAfter:r,actions:a,daysLeft:s,days:m,requestIds:c,emailTemplate:d,concurrency:f}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.Df.call(void 0, {transcendUrl:o,requestActions:a,auth:i,emailTemplate:d,days:m,daysLeft:s,requestIds:c,concurrency:f,createdAtBefore:n,createdAtAfter:r})}exports.notifyAdditionalTime = l;
|
|
2
|
+
//# sourceMappingURL=impl-VXHUL55C.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-VXHUL55C.cjs","../src/commands/request/notify-additional-time/impl.ts"],"names":["notifyAdditionalTime","auth","transcendUrl","createdAtBefore","createdAtAfter","actions","daysLeft","days","requestIds","emailTemplate","concurrency","doneInputValidation","notifyPrivacyRequestsAdditionalTime"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCkBtM,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,QAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,aAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMC,kCAAAA,CACJ,YAAA,CAAAV,CAAAA,CACA,cAAA,CAAgBG,CAAAA,CAChB,IAAA,CAAAJ,CAAAA,CACA,aAAA,CAAAQ,CAAAA,CACA,IAAA,CAAAF,CAAAA,CACA,QAAA,CAAAD,CAAAA,CACA,UAAA,CAAAE,CAAAA,CACA,WAAA,CAAAE,CAAAA,CACA,eAAA,CAAAP,CAAAA,CACA,cAAA,CAAAC,CACF,CAAC,CACH,CAAA,iCAAA","file":"/home/runner/work/cli/cli/dist/impl-VXHUL55C.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { notifyPrivacyRequestsAdditionalTime } from '../../../lib/requests';\nimport type { RequestAction } from '@transcend-io/privacy-types';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface NotifyAdditionalTimeCommandFlags {\n auth: string;\n createdAtBefore: Date;\n createdAtAfter?: Date;\n actions?: RequestAction[];\n daysLeft: number;\n days: number;\n requestIds?: string[];\n emailTemplate: string;\n transcendUrl: string;\n concurrency: number;\n}\n\nexport async function notifyAdditionalTime(\n this: LocalContext,\n {\n auth,\n transcendUrl,\n createdAtBefore,\n createdAtAfter,\n actions,\n daysLeft,\n days,\n requestIds,\n emailTemplate,\n concurrency,\n }: NotifyAdditionalTimeCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await notifyPrivacyRequestsAdditionalTime({\n transcendUrl,\n requestActions: actions,\n auth,\n emailTemplate,\n days,\n daysLeft,\n requestIds,\n concurrency,\n createdAtBefore,\n createdAtAfter,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkTRQBFWZPcjs = require('./chunk-TRQBFWZP.cjs');require('./chunk-LR3CPNDM.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function u({auth:r,transcendUrl:i,file:e,enricherId:o,concurrency:s,markSilent:a,sombraAuth:m}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkTRQBFWZPcjs.d.call(void 0, {file:e,transcendUrl:i,enricherId:o,concurrency:s,markSilent:a,auth:r,sombraAuth:m})}exports.pushIdentifiers = u;
|
|
2
|
+
//# sourceMappingURL=impl-WIGDO7DS.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-WIGDO7DS.cjs","../src/commands/request/preflight/push-identifiers/impl.ts"],"names":["pushIdentifiers","auth","transcendUrl","file","enricherId","concurrency","markSilent","sombraAuth","doneInputValidation","pushManualEnrichmentIdentifiersFromCsv"],"mappings":"AAAA,iIAAwC,gCAA6B,wDAAyC,gCAA6B,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCc/P,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMC,iCAAAA,CACJ,IAAA,CAAAN,CAAAA,CACA,YAAA,CAAAD,CAAAA,CACA,UAAA,CAAAE,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,IAAA,CAAAL,CAAAA,CACA,UAAA,CAAAM,CACF,CAAC,CACH,CAAA,4BAAA","file":"/home/runner/work/cli/cli/dist/impl-WIGDO7DS.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../../context';\nimport { pushManualEnrichmentIdentifiersFromCsv } from '../../../../lib/manual-enrichment';\nimport { doneInputValidation } from '../../../../lib/cli/done-input-validation';\n\nexport interface PushIdentifiersCommandFlags {\n auth: string;\n enricherId: string;\n sombraAuth?: string;\n transcendUrl: string;\n file: string;\n markSilent: boolean;\n concurrency: number;\n}\n\nexport async function pushIdentifiers(\n this: LocalContext,\n {\n auth,\n transcendUrl,\n file,\n enricherId,\n concurrency,\n markSilent,\n sombraAuth,\n }: PushIdentifiersCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await pushManualEnrichmentIdentifiersFromCsv({\n file,\n transcendUrl,\n enricherId,\n concurrency,\n markSilent,\n auth,\n sombraAuth,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkWAXPOCXWcjs = require('./chunk-WAXPOCXW.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);async function q(){let e=Number(process.env.WORKER_ID||"0");_chunkZUNVPK23cjs.a.info(`[w${e}] ready pid=${process.pid}`),_optionalChain([process, 'access', _ => _.send, 'optionalCall', _2 => _2({type:"ready"})]),process.on("message",async o=>{if(!o||typeof o!="object"||(o.type==="shutdown"&&process.exit(0),o.type!=="task"))return;let{filePath:t,options:p}=o.payload,{outputDir:u,clearOutputDir:i}=p;try{_chunkZUNVPK23cjs.a.info(`[w${e}] processing ${t}`),await _chunkKKAJPI6Ocjs.cg.call(void 0, {filePath:t,outputDir:u,clearOutputDir:i,onProgress:(s,n)=>_optionalChain([process, 'access', _3 => _3.send, 'optionalCall', _4 => _4({type:"progress",payload:{filePath:t,processed:s,total:n}})])}),_optionalChain([process, 'access', _5 => _5.send, 'optionalCall', _6 => _6({type:"result",payload:{ok:!0,filePath:t}})])}catch(s){let n=_chunkKKAJPI6Ocjs.Wf.call(void 0, s);_chunkZUNVPK23cjs.a.error(`[w${e}] ERROR ${t}: ${s.stack||n}`),_optionalChain([process, 'access', _7 => _7.send, 'optionalCall', _8 => _8({type:"result",payload:{ok:!1,filePath:t,error:n}})])}}),await new Promise(()=>{})}function R(e){return _chunkWAXPOCXWcjs.d.call(void 0, e)}function $(e){return _chunkWAXPOCXWcjs.e.call(void 0, e)}var v={renderHeader:R,renderWorkers:$};function O(){return typeof __filename<"u"?__filename:process.argv[1]}async function N(e){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let{directory:o,outputDir:t,clearOutputDir:p,concurrency:u,viewerMode:i}=e,s=_chunkKKAJPI6Ocjs.bg.call(void 0, o,this),{poolSize:n,cpuCount:d}=_chunkWAXPOCXWcjs.a.call(void 0, u,s.length);_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Converting ${s.length} Parquet file(s) \u2192 CSV with pool size ${n} (CPU=${d})`));let w=s.map(r=>({filePath:r,options:{outputDir:t,clearOutputDir:p}})),b={nextTask:()=>w.shift(),taskLabel:r=>r.filePath,initTotals:()=>({}),initSlotProgress:()=>{},onProgress:r=>r,onResult:(r,l)=>({totals:r,ok:!!l.ok}),postProcess:async()=>{}};await _chunkWAXPOCXWcjs.f.call(void 0, {title:`Parquet \u2192 CSV - ${o}`,baseDir:o||t||process.cwd(),childFlag:_chunkWAXPOCXWcjs.b,childModulePath:O(),poolSize:n,cpuCount:d,filesTotal:s.length,hooks:b,viewerMode:i,render:r=>_chunkWAXPOCXWcjs.c.call(void 0, r,v,i),extraKeyHandler:({logsBySlot:r,repaint:l,setPaused:D})=>_chunkWAXPOCXWcjs.g.call(void 0, {logsBySlot:r,repaint:l,setPaused:D})})}process.argv.includes(_chunkWAXPOCXWcjs.b)&&q().catch(e=>{_chunkZUNVPK23cjs.a.error(e),process.exit(1)});exports.parquetToCsv = N;
|
|
2
|
+
//# sourceMappingURL=impl-WKBZDQYY.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-WKBZDQYY.cjs","../src/commands/admin/parquet-to-csv/impl.ts","../src/commands/admin/parquet-to-csv/worker.ts"],"names":["runChild","workerId","logger"],"mappings":"AAAA,quBAAkF,wDAAyC,wDAA0D,gCAA6B,wDAAoC,gCAA6B,gCAA6B,gFCC7R,MCgCnB,SAAsBA,CAAAA,CAAAA,CAA0B,CAC9C,IAAMC,CAAAA,CAAW,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,SAAA,EAAa,GAAG,CAAA,CACpDC,mBAAAA,CAAO,IAAA,CAAK,CAAA,EAAA,EAAKD,CAAQ,CAAA,YAAA,EAAe,OAAA,CAAQ,GAAG,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/impl-WKBZDQYY.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport colors from 'colors';\nimport { logger } from '../../../logger';\nimport { collectParquetFilesOrExit } from '../../../lib/helpers';\nimport {\n computePoolSize,\n createExtraKeyHandler,\n CHILD_FLAG,\n type PoolHooks,\n runPool,\n dashboardPlugin,\n} from '../../../lib/pooling';\nimport {\n runChild,\n type ParquetProgress,\n type ParquetResult,\n type ParquetTask,\n} from './worker';\nimport { parquetToCsvPlugin } from './ui';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\n/**\n * Returns the current module's path so the worker pool knows what file to re-exec.\n * In Node ESM, __filename is undefined, so we fall back to argv[1].\n *\n * @returns The current module's path.\n */\nfunction getCurrentModulePath(): string {\n if (typeof __filename !== 'undefined') {\n return __filename as unknown as string;\n }\n return process.argv[1];\n}\n\n/** No custom totals for the header; the runner’s built-ins suffice. */\ntype Totals = Record<string, never>;\n\nexport type ParquetToCsvCommandFlags = {\n directory: string;\n outputDir?: string;\n clearOutputDir: boolean;\n concurrency?: number;\n viewerMode: boolean;\n};\n\n/**\n * Convert all Parquet files in a directory to CSV, in parallel.\n *\n * @param flags - The command flags.\n */\nexport async function parquetToCsv(\n this: LocalContext,\n flags: ParquetToCsvCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n const { directory, outputDir, clearOutputDir, concurrency, viewerMode } =\n flags;\n\n /* 1) Discover .parquet inputs */\n const files = collectParquetFilesOrExit(directory, this);\n\n /* 2) Size the pool */\n const { poolSize, cpuCount } = computePoolSize(concurrency, files.length);\n\n logger.info(\n colors.green(\n `Converting ${files.length} Parquet file(s) → CSV with pool size ${poolSize} (CPU=${cpuCount})`,\n ),\n );\n\n /* 3) Build FIFO queue of tasks (one per file) */\n const queue = files.map<ParquetTask>((filePath) => ({\n filePath,\n options: { outputDir, clearOutputDir },\n }));\n\n /* 4) Pool hooks */\n const hooks: PoolHooks<ParquetTask, ParquetProgress, ParquetResult, Totals> =\n {\n nextTask: () => queue.shift(),\n taskLabel: (t) => t.filePath,\n initTotals: () => ({} as Totals),\n initSlotProgress: () => undefined,\n onProgress: (totals) => totals,\n onResult: (totals, res) => ({ totals, ok: !!res.ok }),\n postProcess: async () => {\n // nothing special post-run\n },\n };\n\n /* 5) Launch the pool runner with custom dashboard plugin */\n await runPool({\n title: `Parquet → CSV - ${directory}`,\n baseDir: directory || outputDir || process.cwd(),\n childFlag: CHILD_FLAG,\n childModulePath: getCurrentModulePath(),\n poolSize,\n cpuCount,\n filesTotal: files.length,\n hooks,\n viewerMode,\n render: (input) => dashboardPlugin(input, parquetToCsvPlugin, viewerMode),\n extraKeyHandler: ({ logsBySlot, repaint, setPaused }) =>\n createExtraKeyHandler({ logsBySlot, repaint, setPaused }),\n });\n}\n\n/* -------------------------------------------------------------------------------------------------\n * If invoked directly as a child process, enter worker loop\n * ------------------------------------------------------------------------------------------------- */\nif (process.argv.includes(CHILD_FLAG)) {\n runChild().catch((err) => {\n logger.error(err);\n process.exit(1);\n });\n}\n","import { parquetToCsvOneFile, extractErrorMessage } from '../../../lib/helpers';\nimport type { ToWorker } from '../../../lib/pooling';\nimport { logger } from '../../../logger';\n\nexport type ParquetTask = {\n /** Absolute path of the Parquet file to convert. */\n filePath: string;\n options: {\n /** Optional directory where CSV output files should be written. */\n outputDir?: string;\n /** Whether to clear any pre-existing output before writing new ones. */\n clearOutputDir: boolean;\n };\n};\n\nexport type ParquetProgress = {\n /** File being processed by the worker. */\n filePath: string;\n /** Rows processed so far. */\n processed: number;\n /** Optional known total rows (not always available). */\n total?: number;\n};\n\nexport type ParquetResult = {\n ok: boolean;\n filePath: string;\n error?: string;\n};\n\n/**\n * Worker loop: convert a single Parquet file to one or more CSV files.\n */\nexport async function runChild(): Promise<void> {\n const workerId = Number(process.env.WORKER_ID || '0');\n logger.info(`[w${workerId}] ready pid=${process.pid}`);\n process.send?.({ type: 'ready' });\n\n process.on('message', async (msg: ToWorker<ParquetTask>) => {\n if (!msg || typeof msg !== 'object') return;\n\n if (msg.type === 'shutdown') {\n process.exit(0);\n }\n if (msg.type !== 'task') return;\n\n const { filePath, options } = msg.payload;\n const { outputDir, clearOutputDir } = options;\n\n try {\n logger.info(`[w${workerId}] processing ${filePath}`);\n await parquetToCsvOneFile({\n filePath,\n outputDir,\n clearOutputDir,\n onProgress: (processed, total) =>\n process.send?.({\n type: 'progress',\n payload: { filePath, processed, total },\n }),\n });\n\n process.send?.({\n type: 'result',\n payload: { ok: true, filePath },\n });\n } catch (err) {\n const message = extractErrorMessage(err);\n logger.error(`[w${workerId}] ERROR ${filePath}: ${err.stack || message}`);\n process.send?.({\n type: 'result',\n payload: { ok: false, filePath, error: message },\n });\n }\n });\n\n // keep alive until shutdown\n await new Promise<never>(() => {\n // Do nothing\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function y({auth:s,requestReceiptFolder:r,sombraAuth:o,actions:a,statuses:i,requestIds:n,createdAt:c,emailIsVerified:u,silentModeBefore:d,sendEmailReceipt:m,copyIdentifiers:p,skipWaitingPeriod:l,createdAtBefore:f,createdAtAfter:R,concurrency:q,transcendUrl:g}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.Kf.call(void 0, {requestReceiptFolder:r,auth:s,sombraAuth:o,requestActions:a,requestStatuses:i,requestIds:n,createdAt:c,emailIsVerified:u,silentModeBefore:d,sendEmailReceipt:m,copyIdentifiers:p,skipWaitingPeriod:l,createdAtBefore:f,createdAtAfter:R,concurrency:q,transcendUrl:g})}exports.restart = y;
|
|
2
|
+
//# sourceMappingURL=impl-WLYOQTIL.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-WLYOQTIL.cjs","../src/commands/request/restart/impl.ts"],"names":["restart","auth","requestReceiptFolder","sombraAuth","actions","statuses","requestIds","createdAt","emailIsVerified","silentModeBefore","sendEmailReceipt","copyIdentifiers","skipWaitingPeriod","createdAtBefore","createdAtAfter","concurrency","transcendUrl","doneInputValidation","bulkRestartRequests"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCwBtM,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,oBAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,QAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMC,kCAAAA,CACJ,oBAAA,CAAAhB,CAAAA,CACA,IAAA,CAAAD,CAAAA,CACA,UAAA,CAAAE,CAAAA,CACA,cAAA,CAAgBC,CAAAA,CAChB,eAAA,CAAiBC,CAAAA,CACjB,UAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,iBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CACF,CAAC,CACH,CAAA,oBAAA","file":"/home/runner/work/cli/cli/dist/impl-WLYOQTIL.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { bulkRestartRequests } from '../../../lib/requests';\nimport type { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface RestartCommandFlags {\n auth: string;\n actions: RequestAction[];\n statuses: RequestStatus[];\n transcendUrl: string;\n requestReceiptFolder: string;\n sombraAuth?: string;\n concurrency: number;\n requestIds?: string[];\n emailIsVerified: boolean;\n createdAt?: Date;\n silentModeBefore?: Date;\n createdAtBefore?: Date;\n createdAtAfter?: Date;\n sendEmailReceipt: boolean;\n copyIdentifiers: boolean;\n skipWaitingPeriod: boolean;\n}\n\nexport async function restart(\n this: LocalContext,\n {\n auth,\n requestReceiptFolder,\n sombraAuth,\n actions,\n statuses,\n requestIds,\n createdAt,\n emailIsVerified,\n silentModeBefore,\n sendEmailReceipt,\n copyIdentifiers,\n skipWaitingPeriod,\n createdAtBefore,\n createdAtAfter,\n concurrency,\n transcendUrl,\n }: RestartCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await bulkRestartRequests({\n requestReceiptFolder,\n auth,\n sombraAuth,\n requestActions: actions,\n requestStatuses: statuses,\n requestIds,\n createdAt,\n emailIsVerified,\n silentModeBefore,\n sendEmailReceipt,\n copyIdentifiers,\n skipWaitingPeriod,\n createdAtBefore,\n createdAtAfter,\n concurrency,\n transcendUrl,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var _chunkNA6JTKNJcjs = require('./chunk-NA6JTKNJ.cjs');var _chunkLR3CPNDMcjs = require('./chunk-LR3CPNDM.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkVY55L6P5cjs = require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);async function A({file:u,transcendUrl:c,auth:g,sombraAuth:v,dataSiloId:b,actions:C,pageLimit:r,skipRequestCount:n,chunkSize:e}){n&&_chunkZUNVPK23cjs.a.info(_colors2.default.yellow("Skipping request count as requested. This may help speed up the call.")),(Number.isNaN(e)||e<=0||e%r!==0)&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Invalid chunk size: "${e}". Must be a positive integer that is a multiple of ${r}.`)),this.process.exit(1)),_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let{baseName:h,extension:I}=_chunkLR3CPNDMcjs.d.call(void 0, u),a=0;await _chunkNA6JTKNJcjs.g.call(void 0, {transcendUrl:c,apiPageSize:r,savePageSize:e,onSave:i=>{let s=`${h}-${a}${I}`;_chunkZUNVPK23cjs.a.info(_colors2.default.blue(`Saving ${i.length} identifiers to file "${s}"`));let $=_chunkVY55L6P5cjs.j.call(void 0, i.map(x=>Object.keys(x)).flat());return _chunkLR3CPNDMcjs.c.call(void 0, s,i,$),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${i.length} identifiers to file "${s}"`)),a+=1,Promise.resolve()},actions:C,auth:g,sombraAuth:v,dataSiloId:b,skipRequestCount:n})}exports.pullIdentifiers = A;
|
|
2
|
+
//# sourceMappingURL=impl-YEXHKEWQ.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-YEXHKEWQ.cjs","../src/commands/request/cron/pull-identifiers/impl.ts"],"names":["pullIdentifiers","file","transcendUrl","auth","sombraAuth","dataSiloId","actions","pageLimit","skipRequestCount","chunkSize","logger","colors","doneInputValidation","baseName","extension","parseFilePath","fileCount","pullChunkedCustomSiloOutstandingIdentifiers","chunk","numberedFileName"],"mappings":"AAAA,iOAAwC,wDAAgD,wDAAyC,gCAA6B,gCAA6B,wDAAyC,wDAAyC,gCAA6B,gFCCvR,MAyBnB,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CACF,CAAA,CACe,CACXD,CAAAA,EACFE,mBAAAA,CAAO,IAAA,CACLC,gBAAAA,CAAO,MAAA,CACL,uEACF,CACF,CAAA,CAAA,CAIA,MAAA,CAAO,KAAA,CAAMF,CAAS,CAAA,EACtBA,CAAAA,EAAa,CAAA,EACbA,CAAAA,CAAYF,CAAAA,GAAc,CAAA,CAAA,EAAA,CAE1BG,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,qBAAA,EAAwBF,CAAS,CAAA,oDAAA,EAAuDF,CAAS,CAAA,CAAA,CACnG,CACF,CAAA,CACA,IAAA,CAAK,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,CAAA,CAGrBK,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,GAAM,CAAE,QAAA,CAAAC,CAAAA,CAAU,SAAA,CAAAC,CAAU,CAAA,CAAIC,iCAAAA,CAAkB,CAAA,CAC9CC,CAAAA,CAAY,CAAA,CAsBhB,MAAMC,iCAAAA,CACJ,YAAA,CAAAf,CAAAA,CACA,WAAA,CAAaK,CAAAA,CACb,YAAA,CAAcE,CAAAA,CACd,MAAA,CAxBcS,CAAAA,EAAmD,CACjE,IAAMC,CAAAA,CAAmB,CAAA,EAAA","file":"/home/runner/work/cli/cli/dist/impl-YEXHKEWQ.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../../context';\nimport colors from 'colors';\n\nimport { logger } from '../../../../logger';\nimport { uniq } from 'lodash-es';\nimport {\n CsvFormattedIdentifier,\n parseFilePath,\n pullChunkedCustomSiloOutstandingIdentifiers,\n writeCsv,\n} from '../../../../lib/cron';\nimport { RequestAction } from '@transcend-io/privacy-types';\nimport { doneInputValidation } from '../../../../lib/cli/done-input-validation';\n\nexport interface PullIdentifiersCommandFlags {\n file: string;\n transcendUrl: string;\n auth: string;\n sombraAuth?: string;\n dataSiloId: string;\n actions: RequestAction[];\n pageLimit: number;\n skipRequestCount: boolean;\n chunkSize: number;\n}\n\nexport async function pullIdentifiers(\n this: LocalContext,\n {\n file,\n transcendUrl,\n auth,\n sombraAuth,\n dataSiloId,\n actions,\n pageLimit,\n skipRequestCount,\n chunkSize,\n }: PullIdentifiersCommandFlags,\n): Promise<void> {\n if (skipRequestCount) {\n logger.info(\n colors.yellow(\n 'Skipping request count as requested. This may help speed up the call.',\n ),\n );\n }\n\n if (\n Number.isNaN(chunkSize) ||\n chunkSize <= 0 ||\n chunkSize % pageLimit !== 0\n ) {\n logger.error(\n colors.red(\n `Invalid chunk size: \"${chunkSize}\". Must be a positive integer that is a multiple of ${pageLimit}.`,\n ),\n );\n this.process.exit(1);\n }\n\n doneInputValidation(this.process.exit);\n\n const { baseName, extension } = parseFilePath(file);\n let fileCount = 0;\n\n const onSave = (chunk: CsvFormattedIdentifier[]): Promise<void> => {\n const numberedFileName = `${baseName}-${fileCount}${extension}`;\n logger.info(\n colors.blue(\n `Saving ${chunk.length} identifiers to file \"${numberedFileName}\"`,\n ),\n );\n\n const headers = uniq(chunk.map((d) => Object.keys(d)).flat());\n writeCsv(numberedFileName, chunk, headers);\n logger.info(\n colors.green(\n `Successfully wrote ${chunk.length} identifiers to file \"${numberedFileName}\"`,\n ),\n );\n fileCount += 1;\n return Promise.resolve();\n };\n\n // Pull down outstanding identifiers\n await pullChunkedCustomSiloOutstandingIdentifiers({\n transcendUrl,\n apiPageSize: pageLimit,\n savePageSize: chunkSize,\n onSave,\n actions,\n auth,\n sombraAuth,\n dataSiloId,\n skipRequestCount,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var _chunkYFF6A56Ncjs = require('./chunk-YFF6A56N.cjs');require('./chunk-O3EACWMC.cjs');require('./chunk-5UBGZNDC.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkVY55L6P5cjs = require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _child_process = require('child_process');var d='A repository name must be provided. You can specify using --repositoryName=$REPO_NAME or by ensuring the command "git config --get remote.origin.url" returns the name of the repository';async function A({auth:f,scanPath:r,ignoreDirs:u,repositoryName:h,transcendUrl:y}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let e=h;if(!e)try{let o=_child_process.execSync.call(void 0, `cd ${r} && git config --get remote.origin.url`).toString("utf-8").trim();[e]=o.includes("https:")?o.split("/").slice(3).join("/").split("."):(o.split(":").pop()||"").split("."),e||(_chunkZUNVPK23cjs.a.error(_colors2.default.red(d)),this.process.exit(1))}catch(a){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`${d} - Got error: ${a.message}`)),this.process.exit(1)}let x=_chunkKKAJPI6Ocjs.wc.call(void 0, y,f),n=await _chunkYFF6A56Ncjs.b.call(void 0, {scanPath:r,ignoreDirs:u,repositoryName:e});await _chunkKKAJPI6Ocjs.Me.call(void 0, x,n);let s=new URL(_chunkVY55L6P5cjs.o);s.pathname="/code-scanning/code-packages",_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Scan found ${n.length} packages at ${r}! View results at '${s.href}'`))}exports.scanPackages = A;
|
|
2
|
+
//# sourceMappingURL=impl-YHIIZRG6.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-YHIIZRG6.cjs","../src/commands/inventory/scan-packages/impl.ts"],"names":["REPO_ERROR","scanPackages","auth","scanPath","ignoreDirs","repositoryName","transcendUrl","doneInputValidation","gitRepositoryName","url","execSync","logger","colors","err"],"mappings":"AAAA,iOAAwC,gCAA6B,gCAA6B,wDAAyC,wDAAkD,gCAA6B,wDAAyC,wDAAyC,gCAA6B,gFCEtT,8CAOM,IAGnBA,CAAAA,CACJ,0LAAA,CAYF,MAAA,SAAsBC,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,QAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAGrC,IAAIC,CAAAA,CAAoBH,CAAAA,CACxB,EAAA,CAAI,CAACG,CAAAA,CACH,GAAI,CAKF,IAAMC,CAAAA,CAJOC,qCAAAA,CACX,GAAA,EAAMP,CAAQ,CAAA,sCAAA,CAChB,CAAA,CAEiB,QAAA,CAAS,OAAO,CAAA,CAAE,IAAA,CAAK,CAAA,CACxC,CAACK,CAAiB,CAAA,CAAKC,CAAAA,CAAI,QAAA,CAAS,QAAQ,CAAA,CAExCA,CAAAA,CAAI,KAAA,CAAM,GAAG,CAAA,CAAE,KAAA,CAAM,CAAC,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA,CAAE,KAAA,CAAM,GAAG,CAAA,CAAA,CAD1CA,CAAAA,CAAI,KAAA,CAAM,GAAG,CAAA,CAAE,GAAA,CAAI,CAAA,EAAK,EAAA,CAAA,CAAI,KAAA,CAAM,GAAG,CAAA,CAErCD,CAAAA,EAAAA,CACHG,mBAAAA,CAAO,KAAA,CAAMC,gBAAAA,CAAO,GAAA,CAAIZ,CAAU,CAAC,CAAA,CACnC,IAAA,CAAK,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,CAEvB,CAAA,KAAA,CAASa,CAAAA,CAAK,CACZF,mBAAAA,CAAO,KAAA,CAAMC,gBAAAA,CAAO,GAAA,CAAI,CAAA,EAAA","file":"/home/runner/work/cli/cli/dist/impl-YHIIZRG6.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { logger } from '../../../logger';\nimport colors from 'colors';\nimport { ADMIN_DASH } from '../../../constants';\nimport { findCodePackagesInFolder } from '../../../lib/code-scanning';\nimport {\n buildTranscendGraphQLClient,\n syncCodePackages,\n} from '../../../lib/graphql';\nimport { execSync } from 'child_process';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nconst REPO_ERROR =\n 'A repository name must be provided. ' +\n 'You can specify using --repositoryName=$REPO_NAME or by ensuring the ' +\n 'command \"git config --get remote.origin.url\" returns the name of the repository';\n\nexport interface ScanPackagesCommandFlags {\n auth: string;\n scanPath: string;\n ignoreDirs?: string[];\n repositoryName?: string;\n transcendUrl: string;\n}\n\nexport async function scanPackages(\n this: LocalContext,\n {\n auth,\n scanPath,\n ignoreDirs,\n repositoryName,\n transcendUrl,\n }: ScanPackagesCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n // Ensure repository name is specified\n let gitRepositoryName = repositoryName;\n if (!gitRepositoryName) {\n try {\n const name = execSync(\n `cd ${scanPath} && git config --get remote.origin.url`,\n );\n // Trim and parse the URL\n const url = name.toString('utf-8').trim();\n [gitRepositoryName] = !url.includes('https:')\n ? (url.split(':').pop() || '').split('.')\n : url.split('/').slice(3).join('/').split('.');\n if (!gitRepositoryName) {\n logger.error(colors.red(REPO_ERROR));\n this.process.exit(1);\n }\n } catch (err) {\n logger.error(colors.red(`${REPO_ERROR} - Got error: ${err.message}`));\n this.process.exit(1);\n }\n }\n\n // Create a GraphQL client\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n // Scan the codebase to discovery packages\n const results = await findCodePackagesInFolder({\n scanPath,\n ignoreDirs,\n repositoryName: gitRepositoryName,\n });\n\n // Report scan to Transcend\n await syncCodePackages(client, results);\n\n const newUrl = new URL(ADMIN_DASH);\n newUrl.pathname = '/code-scanning/code-packages';\n\n // Indicate success\n logger.info(\n colors.green(\n `Scan found ${results.length} packages at ${scanPath}! ` +\n `View results at '${newUrl.href}'`,\n ),\n );\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function F({auth:i,file:n,transcendUrl:r,cacheFilepath:a,requestReceiptFolder:s,sombraAuth:l,concurrency:p,attributes:d,isTest:c,isSilent:u,skipSendingReceipt:m,emailIsVerified:b,skipFilterStep:g,dryRun:f,debug:C,defaultPhoneCountryCode:h}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.If.call(void 0, {cacheFilepath:a,requestReceiptFolder:s,file:n,auth:i,sombraAuth:l,concurrency:p,transcendUrl:r,defaultPhoneCountryCode:h,attributes:_chunkKKAJPI6Ocjs.oc.call(void 0, d),debug:C,skipFilterStep:g,isSilent:u,skipSendingReceipt:m,emailIsVerified:b,isTest:c,dryRun:f})}exports.upload = F;
|
|
2
|
+
//# sourceMappingURL=impl-YPKYWIYP.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-
|
|
1
|
+
{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-YPKYWIYP.cjs","../src/commands/request/upload/impl.ts"],"names":["upload","auth","file","transcendUrl","cacheFilepath","requestReceiptFolder","sombraAuth","concurrency","attributes","isTest","isSilent","skipSendingReceipt","emailIsVerified","skipFilterStep","dryRun","debug","defaultPhoneCountryCode","doneInputValidation","uploadPrivacyRequestsFromCsv","splitCsvToList"],"mappings":"AAAA,iIAAwC,wDAAkD,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MC0B9M,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,aAAA,CAAAC,CAAAA,CACA,oBAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,MAAA,CAAAC,CAAAA,CACA,QAAA,CAAAC,CAAAA,CACA,kBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,MAAA,CAAAC,CAAAA,CACA,KAAA,CAAAC,CAAAA,CACA,uBAAA,CAAAC,CACF,CAAA,CACe,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMC,kCAAAA,CACJ,aAAA,CAAAd,CAAAA,CACA,oBAAA,CAAAC,CAAAA,CACA,IAAA,CAAAH,CAAAA,CACA,IAAA,CAAAD,CAAAA,CACA,UAAA,CAAAK,CAAAA,CACA,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAJ,CAAAA,CACA,uBAAA,CAAAa,CAAAA,CACA,UAAA,CAAYG,kCAAAA,CAAyB,CAAA,CACrC,KAAA,CAAAJ,CAAAA,CACA,cAAA,CAAAF,CAAAA,CACA,QAAA,CAAAH,CAAAA,CACA,kBAAA,CAAAC,CAAAA,CACA,eAAA,CAAAC,CAAAA,CACA,MAAA,CAAAH,CAAAA,CACA,MAAA,CAAAK,CACF,CAAC,CACH,CAAA,mBAAA","file":"/home/runner/work/cli/cli/dist/impl-YPKYWIYP.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport {\n splitCsvToList,\n uploadPrivacyRequestsFromCsv,\n} from '../../../lib/requests';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadCommandFlags {\n auth: string;\n file: string;\n transcendUrl: string;\n cacheFilepath: string;\n requestReceiptFolder: string;\n sombraAuth?: string;\n concurrency: number;\n attributes: string;\n isTest: boolean;\n isSilent: boolean;\n skipSendingReceipt: boolean;\n emailIsVerified: boolean;\n skipFilterStep: boolean;\n dryRun: boolean;\n debug: boolean;\n defaultPhoneCountryCode: string;\n}\n\nexport async function upload(\n this: LocalContext,\n {\n auth,\n file,\n transcendUrl,\n cacheFilepath,\n requestReceiptFolder,\n sombraAuth,\n concurrency,\n attributes,\n isTest,\n isSilent,\n skipSendingReceipt,\n emailIsVerified,\n skipFilterStep,\n dryRun,\n debug,\n defaultPhoneCountryCode,\n }: UploadCommandFlags,\n): Promise<void> {\n doneInputValidation(this.process.exit);\n\n await uploadPrivacyRequestsFromCsv({\n cacheFilepath,\n requestReceiptFolder,\n file,\n auth,\n sombraAuth,\n concurrency,\n transcendUrl,\n defaultPhoneCountryCode,\n attributes: splitCsvToList(attributes),\n debug,\n skipFilterStep,\n isSilent,\n skipSendingReceipt,\n emailIsVerified,\n isTest,\n dryRun,\n });\n}\n"]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var
|
|
2
|
-
//# sourceMappingURL=impl-
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }var _chunkAOO3U7W3cjs = require('./chunk-AOO3U7W3.cjs');require('./chunk-ZUT7UIQU.cjs');var _chunkO3EACWMCcjs = require('./chunk-O3EACWMC.cjs');require('./chunk-5UBGZNDC.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _fs = require('fs');async function L({auth:d,xdiLocation:c,file:o,removeIpAddresses:l,domainBlockList:i,xdiAllowedCommands:a,transcendUrl:m}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let p=await _chunkO3EACWMCcjs.b.call(void 0, d),{syncGroups:f,html:u}=await _chunkAOO3U7W3cjs.g.call(void 0, p,{xdiLocation:c,transcendUrl:m,removeIpAddresses:l,domainBlockList:i.length>0?i:void 0,xdiAllowedCommands:a});_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully constructed sync endpoint for sync groups: ${JSON.stringify(f,null,2)}`)),_fs.writeFileSync.call(void 0, o,u),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Wrote configuration to file "${o}"!`))}exports.buildXdiSyncEndpoint = L;
|
|
2
|
+
//# sourceMappingURL=impl-ZVMYU6PT.cjs.map
|