@transcend-io/cli 8.23.1 → 8.24.1

package/dist/{impl-UBV237P2.cjs.map → impl-RHAYM5BX.cjs.map}

@@ -1 +1 @@
-{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-UBV237P2.cjs","../src/commands/migration/sync-ot/impl.ts","../src/lib/oneTrust/createOneTrustGotInstance.ts","../src/lib/oneTrust/helpers/oneTrustAssessmentToJson.ts","../src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts","../src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts","../src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts"],"names":["createOneTrustGotInstance","hostname","auth","got"],"mappings":"AAAA,y0BAAwC,wDAA0D,gCAA6B,wDAAoC,wDAAgD,wDAAyC,gFCEzO,oECFM,IAQZA,CAAAA,CAA4B,CAAC,CACxC,QAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CACF,CAAA,CAAA,EAMEC,aAAAA,CAAI,MAAA,CAAO,CACT,SAAA,CAAW,CAAA,QAAA,EAAWF,CAAQ,CAAA,CAAA;ACiBwB;AAIzC;AClCf,CAAA;ACsE2F,mBAAA;ACC3B","file":"/home/runner/work/cli/cli/dist/impl-UBV237P2.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { logger } from '../../../logger';\nimport colors from 'colors';\nimport { createOneTrustGotInstance } from '../../../lib/oneTrust';\nimport {\n  OneTrustFileFormat,\n  OneTrustPullResource,\n  OneTrustPullSource,\n} from '../../../enums';\nimport { buildTranscendGraphQLClient } from '../../../lib/graphql';\nimport {\n  syncOneTrustAssessmentsFromFile,\n  syncOneTrustAssessmentsFromOneTrust,\n} from '../../../lib/oneTrust/helpers';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\n// Command flag interface\nexport interface SyncOtCommandFlags {\n  hostname?: string;\n  oneTrustAuth?: string;\n  source: OneTrustPullSource;\n  transcendAuth?: string;\n  transcendUrl: string;\n  file?: string;\n  resource: OneTrustPullResource;\n  dryRun: boolean;\n  debug: boolean;\n}\n\n// Command implementation\nexport async function syncOt(\n  this: LocalContext,\n  {\n    hostname,\n    oneTrustAuth,\n    source,\n    transcendAuth,\n    transcendUrl,\n    resource,\n    file,\n    dryRun,\n    debug,\n  }: SyncOtCommandFlags,\n): Promise<void> {\n  // Must be able to authenticate to transcend to sync resources to it\n  if (!dryRun && !transcendAuth) {\n    throw new Error(\n      // eslint-disable-next-line no-template-curly-in-string\n      'Must specify a \"transcendAuth\" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}',\n    );\n  }\n\n  // If trying to sync to disk, must specify a file path\n  if (dryRun && !file) {\n    throw new Error(\n      'Must set a \"file\" parameter when \"dryRun\" is \"true\". e.g. --file=./oneTrustAssessments.json',\n    );\n  }\n\n  if (file) {\n    const splitFile = file.split('.');\n    if (splitFile.length < 2) {\n      throw new Error(\n        'The \"file\" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.',\n      );\n    }\n    if (splitFile.at(-1) !== OneTrustFileFormat.Json) {\n      throw new Error(\n        `Expected the format of the \"file\" parameters '${file}' to be '${\n          OneTrustFileFormat.Json\n        }', but got '${splitFile.at(-1)}'.`,\n      );\n    }\n  }\n\n  // if reading assessments from a OneTrust\n  if (source === OneTrustPullSource.OneTrust) {\n    // must specify the OneTrust hostname\n    if (!hostname) {\n      throw new Error(\n        'Missing required parameter \"hostname\". e.g. --hostname=customer.my.onetrust.com',\n      );\n    }\n    // must specify the OneTrust auth\n    if (!oneTrustAuth) {\n      throw new Error(\n        'Missing required parameter \"oneTrustAuth\". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN',\n      );\n    }\n  } else {\n    // if reading the assessments from a file, must specify a file to read from\n    if (!file) {\n      throw new Error(\n        'Must specify a \"file\" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json',\n      );\n    }\n\n    // Cannot try reading from file and save assessments to a file simultaneously\n    if (dryRun) {\n      throw new Error(\n        'Cannot read and write to a file simultaneously.' +\n          ` Emit the \"source\" parameter or set it to ${OneTrustPullSource.OneTrust} if \"dryRun\" is enabled.`,\n      );\n    }\n  }\n\n  doneInputValidation(this.process.exit);\n\n  // instantiate a client to talk to OneTrust\n  const oneTrust =\n    hostname && oneTrustAuth\n      ? createOneTrustGotInstance({\n          hostname,\n          auth: oneTrustAuth,\n        })\n      : undefined;\n\n  // instantiate a client to talk to Transcend\n  const transcend =\n    transcendUrl && transcendAuth\n      ? buildTranscendGraphQLClient(transcendUrl, transcendAuth)\n      : undefined;\n\n  try {\n    if (resource === OneTrustPullResource.Assessments) {\n      if (source === OneTrustPullSource.OneTrust && oneTrust) {\n        await syncOneTrustAssessmentsFromOneTrust({\n          oneTrust,\n          file,\n          dryRun,\n          ...(transcend && { transcend }),\n        });\n      } else if (source === OneTrustPullSource.File && file && transcend) {\n        await syncOneTrustAssessmentsFromFile({ file, transcend });\n      }\n    }\n  } catch (err) {\n    throw new Error(\n      `An error occurred syncing the resource ${resource} from OneTrust: ${\n        debug ? err.stack : err.message\n      }`,\n    );\n  }\n\n  // Indicate success\n  logger.info(\n    colors.green(\n      `Successfully synced OneTrust ${resource} to ${\n        dryRun ? `disk at \"${file}\"` : 'Transcend'\n      }!`,\n    ),\n  );\n}\n","import got, { Got } from 'got';\n\n/**\n * Instantiate an instance of got that is capable of making requests to OneTrust\n *\n * @param param - information about the OneTrust URL\n * @returns The instance of got that is capable of making requests to the customer ingress\n */\nexport const createOneTrustGotInstance = ({\n  hostname,\n  auth,\n}: {\n  /** Hostname of the OneTrust API */\n  hostname: string;\n  /** The OAuth access token */\n  auth: string;\n}): Got =>\n  got.extend({\n    prefixUrl: `https://${hostname}`,\n    headers: {\n      accept: 'application/json',\n      'content-type': 'application/json',\n      authorization: `Bearer ${auth}`,\n    },\n  });\n","import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types';\n\n/**\n * Converts the assessment into a json entry.\n *\n * @param param - information about the assessment and amount of entries\n * @returns a stringified json entry ready to be appended to a file\n */\nexport const oneTrustAssessmentToJson = ({\n  assessment,\n  index,\n  total,\n  wrap = true,\n}: {\n  /** The assessment to convert */\n  assessment: OneTrustEnrichedAssessment;\n  /** The position of the assessment in the final Json object */\n  index: number;\n  /** The total amount of the assessments in the final Json object */\n  total?: number;\n  /** Whether to wrap every entry in brackets */\n  wrap?: boolean;\n}): string => {\n  let jsonEntry = '';\n  // start with an opening bracket\n  if (index === 0 || wrap) {\n    jsonEntry = '[\\n';\n  }\n\n  const stringifiedAssessment = JSON.stringify(assessment);\n\n  // Add comma for all items except the last one\n  const comma = total && index < total - 1 && !wrap ? ',' : '';\n\n  // write to file\n  jsonEntry = `${jsonEntry + stringifiedAssessment + comma}\\n`;\n\n  // end with closing bracket\n  if ((total && index === total - 1) || wrap) {\n    jsonEntry += '\\n]';\n  }\n\n  return jsonEntry;\n};\n","import { Got } from 'got';\nimport { logger } from '../../../logger';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport {\n  OneTrustAssessment,\n  OneTrustGetListOfAssessmentsResponse,\n} from '@transcend-io/privacy-types';\n\n/**\n * Fetch a list of all assessments from the OneTrust client.\n * ref: https://developer.onetrust.com/onetrust/reference/getallassessmentbasicdetailsusingget\n *\n * @param param - the information about the OneTrust client\n * @returns a list of OneTrustAssessment\n */\nexport const getListOfOneTrustAssessments = async ({\n  oneTrust,\n}: {\n  /** The OneTrust client instance */\n  oneTrust: Got;\n}): Promise<OneTrustAssessment[]> => {\n  let currentPage = 0;\n  let totalPages = 1;\n  let totalElements = 0;\n\n  const allAssessments: OneTrustAssessment[] = [];\n\n  while (currentPage < totalPages) {\n    const { body } = await oneTrust.get(\n      `api/assessment/v2/assessments?page=${currentPage}&size=2000`,\n    );\n\n    const { page, content } = decodeCodec(\n      OneTrustGetListOfAssessmentsResponse,\n      body,\n    );\n    allAssessments.push(...(content ?? []));\n    if (currentPage === 0) {\n      totalPages = page?.totalPages ?? 0;\n      totalElements = page?.totalElements ?? 0;\n    }\n    currentPage += 1;\n\n    // log progress\n    logger.info(\n      `Fetched ${allAssessments.length} of ${totalElements} assessments.`,\n    );\n  }\n\n  return allAssessments;\n};\n","import { logger } from '../../../logger';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport {\n  IMPORT_ONE_TRUST_ASSESSMENT_FORMS,\n  makeGraphQLRequest,\n} from '../../graphql';\nimport { ImportOnetrustAssessmentsInput } from '../../../codecs';\nimport { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types';\nimport { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson';\n\nexport interface AssessmentForm {\n  /** ID of Assessment Form */\n  id: string;\n  /** Title of Assessment Form */\n  name: string;\n}\n\n/**\n * Write the assessment to a Transcend instance.\n *\n *\n * @param param - information about the assessment and Transcend instance to write to\n */\nexport const syncOneTrustAssessmentToTranscend = async ({\n  transcend,\n  assessment,\n  total,\n  index,\n}: {\n  /** the Transcend client instance */\n  transcend: GraphQLClient;\n  /** the assessment to sync to Transcend */\n  assessment: OneTrustEnrichedAssessment;\n  /** The index of the assessment being written to the file */\n  index: number;\n  /** The total amount of assessments that we will write */\n  total?: number;\n}): Promise<void> => {\n  logger.info(\n    colors.magenta(\n      `Writing enriched assessment ${index + 1} ${\n        total ? `of ${total} ` : ' '\n      }to Transcend...`,\n    ),\n  );\n\n  // convert the OneTrust assessment object into a json record\n  const json = oneTrustAssessmentToJson({\n    assessment,\n    index,\n    total,\n  });\n\n  // transform the json record into a valid input to the mutation\n  const input: ImportOnetrustAssessmentsInput = {\n    json,\n  };\n\n  try {\n    await makeGraphQLRequest<{\n      /** the importOneTrustAssessmentForms mutation */\n      importOneTrustAssessmentForms: {\n        /** Created Assessment Forms */\n        assessmentForms: AssessmentForm[];\n      };\n    }>(transcend, IMPORT_ONE_TRUST_ASSESSMENT_FORMS, {\n      input,\n    });\n  } catch (e) {\n    logger.error(\n      colors.red(\n        `Failed to sync assessment ${index + 1} ${\n          total ? `of ${total} ` : ' '\n        }to Transcend.\\n` +\n          `\\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\\n`,\n      ),\n    );\n  }\n};\n","import type { Got } from 'got';\nimport colors from 'colors';\nimport {\n  getListOfOneTrustAssessments,\n  getOneTrustAssessment,\n  getOneTrustRisk,\n  getOneTrustUser,\n} from '../endpoints';\nimport { mapSeries, map } from 'bluebird';\nimport { logger } from '../../../logger';\nimport {\n  OneTrustAssessmentQuestion,\n  OneTrustAssessmentSection,\n  OneTrustEnrichedAssessment,\n  OneTrustGetRiskResponse,\n  OneTrustGetUserResponse,\n} from '@transcend-io/privacy-types';\nimport { uniq } from 'lodash-es';\nimport { enrichOneTrustAssessment } from './enrichOneTrustAssessment';\nimport { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk';\nimport { GraphQLClient } from 'graphql-request';\nimport { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend';\n\nexport interface AssessmentForm {\n  /** ID of Assessment Form */\n  id: string;\n  /** Title of Assessment Form */\n  name: string;\n}\n\n/**\n * Reads all the assessments from a OneTrust instance and syncs them to Transcend or to Disk.\n *\n * @param param - the information about the assessment, its OneTrust source, and destination (disk or Transcend)\n */\nexport const syncOneTrustAssessmentsFromOneTrust = async ({\n  oneTrust,\n  file,\n  dryRun,\n  transcend,\n}: {\n  /** the OneTrust client instance */\n  oneTrust: Got;\n  /** the Transcend client instance */\n  transcend?: GraphQLClient;\n  /** Whether to write to file instead of syncing to Transcend */\n  dryRun: boolean;\n  /** the path to the file in case dryRun is true */\n  file?: string;\n}): Promise<void> => {\n  // fetch the list of all assessments in the OneTrust organization\n  logger.info('Getting list of all assessments from OneTrust...');\n  const assessments = await getListOfOneTrustAssessments({ oneTrust });\n\n  // a cache of OneTrust users so we avoid requesting already fetched users\n  const oneTrustCachedUsers: Record<string, OneTrustGetUserResponse> = {};\n\n  // split all assessments in batches, so we can process some of steps in parallel\n  const BATCH_SIZE = 5;\n  const assessmentBatches = Array.from(\n    {\n      length: Math.ceil(assessments.length / BATCH_SIZE),\n    },\n    (_, i) => assessments.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE),\n  );\n\n  // process each batch and sync the batch right away so it's garbage collected and we don't run out of memory\n  await mapSeries(assessmentBatches, async (assessmentBatch, batch) => {\n    const batchEnrichedAssessments: OneTrustEnrichedAssessment[] = [];\n\n    // fetch assessment details from OneTrust in parallel\n    await map(\n      assessmentBatch,\n      async (assessment, index) => {\n        const assessmentNumber = BATCH_SIZE * batch + index + 1;\n        logger.info(\n          `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...`,\n        );\n        const { templateName, assessmentId } = assessment;\n        const assessmentDetails = await getOneTrustAssessment({\n          oneTrust,\n          assessmentId,\n        });\n        // fetch assessment's creator information\n        const creatorId = assessmentDetails.createdBy.id;\n        let creator = oneTrustCachedUsers[creatorId];\n        if (!creator) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...`,\n          );\n          try {\n            creator = await getOneTrustUser({\n              oneTrust,\n              userId: creatorId,\n            });\n            oneTrustCachedUsers[creatorId] = creator;\n          } catch (e) {\n            logger.warn(\n              colors.yellow(\n                `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch form creator.` +\n                  `\\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n              ),\n            );\n          }\n        }\n\n        // fetch assessment approvers information\n        const { approvers } = assessmentDetails;\n        let approversDetails: OneTrustGetUserResponse[][] = [];\n        if (approvers.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...`,\n          );\n          approversDetails = await map(\n            approvers.map(({ id }) => id),\n            async (userId) => {\n              try {\n                let approver = oneTrustCachedUsers[userId];\n                if (!approver) {\n                  approver = await getOneTrustUser({ oneTrust, userId });\n                  oneTrustCachedUsers[userId] = approver;\n                }\n                return [approver];\n              } catch (e) {\n                logger.warn(\n                  colors.yellow(\n                    `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a form approver.` +\n                      `\\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n                  ),\n                );\n                return [];\n              }\n            },\n            { concurrency: 5 },\n          );\n        }\n\n        // fetch assessment internal respondents information\n        const { respondents } = assessmentDetails;\n        // if a user is an internal respondents, their 'name' field can't be an email.\n        const internalRespondents = respondents.filter(\n          (r) => !r.name.includes('@'),\n        );\n        let respondentsDetails: OneTrustGetUserResponse[][] = [];\n        if (internalRespondents.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...`,\n          );\n          respondentsDetails = await map(\n            internalRespondents.map(({ id }) => id),\n            async (userId) => {\n              try {\n                let respondent = oneTrustCachedUsers[userId];\n                if (!respondent) {\n                  respondent = await getOneTrustUser({ oneTrust, userId });\n                  oneTrustCachedUsers[userId] = respondent;\n                }\n                return [respondent];\n              } catch (e) {\n                logger.warn(\n                  colors.yellow(\n                    `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a respondent.` +\n                      `\\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n                  ),\n                );\n                return [];\n              }\n            },\n            { concurrency: 5 },\n          );\n        }\n\n        // fetch assessment risk information\n        let riskDetails: OneTrustGetRiskResponse[] = [];\n        const riskIds = uniq(\n          assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) =>\n            s.questions.flatMap((q: OneTrustAssessmentQuestion) =>\n              (q.risks ?? []).flatMap((r) => r.riskId),\n            ),\n          ),\n        );\n        if (riskIds.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...`,\n          );\n          riskDetails = await map(\n            riskIds,\n            (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }),\n            {\n              concurrency: 5,\n            },\n          );\n        }\n\n        // enrich the assessments with user and risk details\n        const enrichedAssessment = enrichOneTrustAssessment({\n          assessment,\n          assessmentDetails,\n          riskDetails,\n          creatorDetails: creator,\n          approversDetails: approversDetails.flat(),\n          respondentsDetails: respondentsDetails.flat(),\n        });\n\n        batchEnrichedAssessments.push(enrichedAssessment);\n      },\n      { concurrency: BATCH_SIZE },\n    );\n\n    // sync assessments in series to avoid concurrency bugs\n    await mapSeries(\n      batchEnrichedAssessments,\n      async (enrichedAssessment, index) => {\n        // the assessment's global index takes its batch into consideration\n        const globalIndex = batch * BATCH_SIZE + index;\n\n        if (dryRun && file) {\n          // sync to file\n          syncOneTrustAssessmentToDisk({\n            assessment: enrichedAssessment,\n            index: globalIndex,\n            total: assessments.length,\n            file,\n          });\n        } else if (transcend) {\n          // sync to transcend\n          await syncOneTrustAssessmentToTranscend({\n            assessment: enrichedAssessment,\n            transcend,\n            total: assessments.length,\n            index: globalIndex,\n          });\n        }\n      },\n    );\n  });\n};\n"]}
+{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-RHAYM5BX.cjs","../src/commands/migration/sync-ot/impl.ts","../src/lib/oneTrust/createOneTrustGotInstance.ts","../src/lib/oneTrust/helpers/oneTrustAssessmentToJson.ts","../src/lib/oneTrust/endpoints/getListOfOneTrustAssessments.ts","../src/lib/oneTrust/helpers/syncOneTrustAssessmentToTranscend.ts","../src/lib/oneTrust/helpers/syncOneTrustAssessmentsFromOneTrust.ts"],"names":["createOneTrustGotInstance","hostname","auth","got"],"mappings":"AAAA,y0BAAwC,wDAA0D,gCAA6B,wDAAoC,wDAAgD,wDAAyC,gFCEzO,oECFM,IAQZA,CAAAA,CAA4B,CAAC,CACxC,QAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CACF,CAAA,CAAA,EAMEC,aAAAA,CAAI,MAAA,CAAO,CACT,SAAA,CAAW,CAAA,QAAA,EAAWF,CAAQ,CAAA,CAAA;ACiBwB;AAIzC;AClCf,CAAA;ACsE2F,mBAAA;ACC3B","file":"/home/runner/work/cli/cli/dist/impl-RHAYM5BX.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { logger } from '../../../logger';\nimport colors from 'colors';\nimport { createOneTrustGotInstance } from '../../../lib/oneTrust';\nimport {\n  OneTrustFileFormat,\n  OneTrustPullResource,\n  OneTrustPullSource,\n} from '../../../enums';\nimport { buildTranscendGraphQLClient } from '../../../lib/graphql';\nimport {\n  syncOneTrustAssessmentsFromFile,\n  syncOneTrustAssessmentsFromOneTrust,\n} from '../../../lib/oneTrust/helpers';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\n// Command flag interface\nexport interface SyncOtCommandFlags {\n  hostname?: string;\n  oneTrustAuth?: string;\n  source: OneTrustPullSource;\n  transcendAuth?: string;\n  transcendUrl: string;\n  file?: string;\n  resource: OneTrustPullResource;\n  dryRun: boolean;\n  debug: boolean;\n}\n\n// Command implementation\nexport async function syncOt(\n  this: LocalContext,\n  {\n    hostname,\n    oneTrustAuth,\n    source,\n    transcendAuth,\n    transcendUrl,\n    resource,\n    file,\n    dryRun,\n    debug,\n  }: SyncOtCommandFlags,\n): Promise<void> {\n  // Must be able to authenticate to transcend to sync resources to it\n  if (!dryRun && !transcendAuth) {\n    throw new Error(\n      // eslint-disable-next-line no-template-curly-in-string\n      'Must specify a \"transcendAuth\" parameter to sync resources to Transcend. e.g. --transcendAuth=${TRANSCEND_API_KEY}',\n    );\n  }\n\n  // If trying to sync to disk, must specify a file path\n  if (dryRun && !file) {\n    throw new Error(\n      'Must set a \"file\" parameter when \"dryRun\" is \"true\". e.g. --file=./oneTrustAssessments.json',\n    );\n  }\n\n  if (file) {\n    const splitFile = file.split('.');\n    if (splitFile.length < 2) {\n      throw new Error(\n        'The \"file\" parameter has an invalid format. Expected a path with extensions. e.g. --file=./pathToFile.json.',\n      );\n    }\n    if (splitFile.at(-1) !== OneTrustFileFormat.Json) {\n      throw new Error(\n        `Expected the format of the \"file\" parameters '${file}' to be '${\n          OneTrustFileFormat.Json\n        }', but got '${splitFile.at(-1)}'.`,\n      );\n    }\n  }\n\n  // if reading assessments from a OneTrust\n  if (source === OneTrustPullSource.OneTrust) {\n    // must specify the OneTrust hostname\n    if (!hostname) {\n      throw new Error(\n        'Missing required parameter \"hostname\". e.g. --hostname=customer.my.onetrust.com',\n      );\n    }\n    // must specify the OneTrust auth\n    if (!oneTrustAuth) {\n      throw new Error(\n        'Missing required parameter \"oneTrustAuth\". e.g. --oneTrustAuth=$ONE_TRUST_AUTH_TOKEN',\n      );\n    }\n  } else {\n    // if reading the assessments from a file, must specify a file to read from\n    if (!file) {\n      throw new Error(\n        'Must specify a \"file\" parameter to read the OneTrust assessments from. e.g. --source=./oneTrustAssessments.json',\n      );\n    }\n\n    // Cannot try reading from file and save assessments to a file simultaneously\n    if (dryRun) {\n      throw new Error(\n        'Cannot read and write to a file simultaneously.' +\n          ` Emit the \"source\" parameter or set it to ${OneTrustPullSource.OneTrust} if \"dryRun\" is enabled.`,\n      );\n    }\n  }\n\n  doneInputValidation(this.process.exit);\n\n  // instantiate a client to talk to OneTrust\n  const oneTrust =\n    hostname && oneTrustAuth\n      ? createOneTrustGotInstance({\n          hostname,\n          auth: oneTrustAuth,\n        })\n      : undefined;\n\n  // instantiate a client to talk to Transcend\n  const transcend =\n    transcendUrl && transcendAuth\n      ? buildTranscendGraphQLClient(transcendUrl, transcendAuth)\n      : undefined;\n\n  try {\n    if (resource === OneTrustPullResource.Assessments) {\n      if (source === OneTrustPullSource.OneTrust && oneTrust) {\n        await syncOneTrustAssessmentsFromOneTrust({\n          oneTrust,\n          file,\n          dryRun,\n          ...(transcend && { transcend }),\n        });\n      } else if (source === OneTrustPullSource.File && file && transcend) {\n        await syncOneTrustAssessmentsFromFile({ file, transcend });\n      }\n    }\n  } catch (err) {\n    throw new Error(\n      `An error occurred syncing the resource ${resource} from OneTrust: ${\n        debug ? err.stack : err.message\n      }`,\n    );\n  }\n\n  // Indicate success\n  logger.info(\n    colors.green(\n      `Successfully synced OneTrust ${resource} to ${\n        dryRun ? `disk at \"${file}\"` : 'Transcend'\n      }!`,\n    ),\n  );\n}\n","import got, { Got } from 'got';\n\n/**\n * Instantiate an instance of got that is capable of making requests to OneTrust\n *\n * @param param - information about the OneTrust URL\n * @returns The instance of got that is capable of making requests to the customer ingress\n */\nexport const createOneTrustGotInstance = ({\n  hostname,\n  auth,\n}: {\n  /** Hostname of the OneTrust API */\n  hostname: string;\n  /** The OAuth access token */\n  auth: string;\n}): Got =>\n  got.extend({\n    prefixUrl: `https://${hostname}`,\n    headers: {\n      accept: 'application/json',\n      'content-type': 'application/json',\n      authorization: `Bearer ${auth}`,\n    },\n  });\n","import { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types';\n\n/**\n * Converts the assessment into a json entry.\n *\n * @param param - information about the assessment and amount of entries\n * @returns a stringified json entry ready to be appended to a file\n */\nexport const oneTrustAssessmentToJson = ({\n  assessment,\n  index,\n  total,\n  wrap = true,\n}: {\n  /** The assessment to convert */\n  assessment: OneTrustEnrichedAssessment;\n  /** The position of the assessment in the final Json object */\n  index: number;\n  /** The total amount of the assessments in the final Json object */\n  total?: number;\n  /** Whether to wrap every entry in brackets */\n  wrap?: boolean;\n}): string => {\n  let jsonEntry = '';\n  // start with an opening bracket\n  if (index === 0 || wrap) {\n    jsonEntry = '[\\n';\n  }\n\n  const stringifiedAssessment = JSON.stringify(assessment);\n\n  // Add comma for all items except the last one\n  const comma = total && index < total - 1 && !wrap ? ',' : '';\n\n  // write to file\n  jsonEntry = `${jsonEntry + stringifiedAssessment + comma}\\n`;\n\n  // end with closing bracket\n  if ((total && index === total - 1) || wrap) {\n    jsonEntry += '\\n]';\n  }\n\n  return jsonEntry;\n};\n","import { Got } from 'got';\nimport { logger } from '../../../logger';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport {\n  OneTrustAssessment,\n  OneTrustGetListOfAssessmentsResponse,\n} from '@transcend-io/privacy-types';\n\n/**\n * Fetch a list of all assessments from the OneTrust client.\n * ref: https://developer.onetrust.com/onetrust/reference/getallassessmentbasicdetailsusingget\n *\n * @param param - the information about the OneTrust client\n * @returns a list of OneTrustAssessment\n */\nexport const getListOfOneTrustAssessments = async ({\n  oneTrust,\n}: {\n  /** The OneTrust client instance */\n  oneTrust: Got;\n}): Promise<OneTrustAssessment[]> => {\n  let currentPage = 0;\n  let totalPages = 1;\n  let totalElements = 0;\n\n  const allAssessments: OneTrustAssessment[] = [];\n\n  while (currentPage < totalPages) {\n    const { body } = await oneTrust.get(\n      `api/assessment/v2/assessments?page=${currentPage}&size=2000`,\n    );\n\n    const { page, content } = decodeCodec(\n      OneTrustGetListOfAssessmentsResponse,\n      body,\n    );\n    allAssessments.push(...(content ?? []));\n    if (currentPage === 0) {\n      totalPages = page?.totalPages ?? 0;\n      totalElements = page?.totalElements ?? 0;\n    }\n    currentPage += 1;\n\n    // log progress\n    logger.info(\n      `Fetched ${allAssessments.length} of ${totalElements} assessments.`,\n    );\n  }\n\n  return allAssessments;\n};\n","import { logger } from '../../../logger';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport {\n  IMPORT_ONE_TRUST_ASSESSMENT_FORMS,\n  makeGraphQLRequest,\n} from '../../graphql';\nimport { ImportOnetrustAssessmentsInput } from '../../../codecs';\nimport { OneTrustEnrichedAssessment } from '@transcend-io/privacy-types';\nimport { oneTrustAssessmentToJson } from './oneTrustAssessmentToJson';\n\nexport interface AssessmentForm {\n  /** ID of Assessment Form */\n  id: string;\n  /** Title of Assessment Form */\n  name: string;\n}\n\n/**\n * Write the assessment to a Transcend instance.\n *\n *\n * @param param - information about the assessment and Transcend instance to write to\n */\nexport const syncOneTrustAssessmentToTranscend = async ({\n  transcend,\n  assessment,\n  total,\n  index,\n}: {\n  /** the Transcend client instance */\n  transcend: GraphQLClient;\n  /** the assessment to sync to Transcend */\n  assessment: OneTrustEnrichedAssessment;\n  /** The index of the assessment being written to the file */\n  index: number;\n  /** The total amount of assessments that we will write */\n  total?: number;\n}): Promise<void> => {\n  logger.info(\n    colors.magenta(\n      `Writing enriched assessment ${index + 1} ${\n        total ? `of ${total} ` : ' '\n      }to Transcend...`,\n    ),\n  );\n\n  // convert the OneTrust assessment object into a json record\n  const json = oneTrustAssessmentToJson({\n    assessment,\n    index,\n    total,\n  });\n\n  // transform the json record into a valid input to the mutation\n  const input: ImportOnetrustAssessmentsInput = {\n    json,\n  };\n\n  try {\n    await makeGraphQLRequest<{\n      /** the importOneTrustAssessmentForms mutation */\n      importOneTrustAssessmentForms: {\n        /** Created Assessment Forms */\n        assessmentForms: AssessmentForm[];\n      };\n    }>(transcend, IMPORT_ONE_TRUST_ASSESSMENT_FORMS, {\n      input,\n    });\n  } catch (e) {\n    logger.error(\n      colors.red(\n        `Failed to sync assessment ${index + 1} ${\n          total ? `of ${total} ` : ' '\n        }to Transcend.\\n` +\n          `\\tAssessment Title: ${assessment.name}. Template Title: ${assessment.template.name}\\n`,\n      ),\n    );\n  }\n};\n","import type { Got } from 'got';\nimport colors from 'colors';\nimport {\n  getListOfOneTrustAssessments,\n  getOneTrustAssessment,\n  getOneTrustRisk,\n  getOneTrustUser,\n} from '../endpoints';\nimport { mapSeries, map } from 'bluebird';\nimport { logger } from '../../../logger';\nimport {\n  OneTrustAssessmentQuestion,\n  OneTrustAssessmentSection,\n  OneTrustEnrichedAssessment,\n  OneTrustGetRiskResponse,\n  OneTrustGetUserResponse,\n} from '@transcend-io/privacy-types';\nimport { uniq } from 'lodash-es';\nimport { enrichOneTrustAssessment } from './enrichOneTrustAssessment';\nimport { syncOneTrustAssessmentToDisk } from './syncOneTrustAssessmentToDisk';\nimport { GraphQLClient } from 'graphql-request';\nimport { syncOneTrustAssessmentToTranscend } from './syncOneTrustAssessmentToTranscend';\n\nexport interface AssessmentForm {\n  /** ID of Assessment Form */\n  id: string;\n  /** Title of Assessment Form */\n  name: string;\n}\n\n/**\n * Reads all the assessments from a OneTrust instance and syncs them to Transcend or to Disk.\n *\n * @param param - the information about the assessment, its OneTrust source, and destination (disk or Transcend)\n */\nexport const syncOneTrustAssessmentsFromOneTrust = async ({\n  oneTrust,\n  file,\n  dryRun,\n  transcend,\n}: {\n  /** the OneTrust client instance */\n  oneTrust: Got;\n  /** the Transcend client instance */\n  transcend?: GraphQLClient;\n  /** Whether to write to file instead of syncing to Transcend */\n  dryRun: boolean;\n  /** the path to the file in case dryRun is true */\n  file?: string;\n}): Promise<void> => {\n  // fetch the list of all assessments in the OneTrust organization\n  logger.info('Getting list of all assessments from OneTrust...');\n  const assessments = await getListOfOneTrustAssessments({ oneTrust });\n\n  // a cache of OneTrust users so we avoid requesting already fetched users\n  const oneTrustCachedUsers: Record<string, OneTrustGetUserResponse> = {};\n\n  // split all assessments in batches, so we can process some of steps in parallel\n  const BATCH_SIZE = 5;\n  const assessmentBatches = Array.from(\n    {\n      length: Math.ceil(assessments.length / BATCH_SIZE),\n    },\n    (_, i) => assessments.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE),\n  );\n\n  // process each batch and sync the batch right away so it's garbage collected and we don't run out of memory\n  await mapSeries(assessmentBatches, async (assessmentBatch, batch) => {\n    const batchEnrichedAssessments: OneTrustEnrichedAssessment[] = [];\n\n    // fetch assessment details from OneTrust in parallel\n    await map(\n      assessmentBatch,\n      async (assessment, index) => {\n        const assessmentNumber = BATCH_SIZE * batch + index + 1;\n        logger.info(\n          `[assessment ${assessmentNumber} of ${assessments.length}]: fetching details...`,\n        );\n        const { templateName, assessmentId } = assessment;\n        const assessmentDetails = await getOneTrustAssessment({\n          oneTrust,\n          assessmentId,\n        });\n        // fetch assessment's creator information\n        const creatorId = assessmentDetails.createdBy.id;\n        let creator = oneTrustCachedUsers[creatorId];\n        if (!creator) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching creator...`,\n          );\n          try {\n            creator = await getOneTrustUser({\n              oneTrust,\n              userId: creatorId,\n            });\n            oneTrustCachedUsers[creatorId] = creator;\n          } catch (e) {\n            logger.warn(\n              colors.yellow(\n                `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch form creator.` +\n                  `\\tcreatorId: ${creatorId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n              ),\n            );\n          }\n        }\n\n        // fetch assessment approvers information\n        const { approvers } = assessmentDetails;\n        let approversDetails: OneTrustGetUserResponse[][] = [];\n        if (approvers.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching approvers...`,\n          );\n          approversDetails = await map(\n            approvers.map(({ id }) => id),\n            async (userId) => {\n              try {\n                let approver = oneTrustCachedUsers[userId];\n                if (!approver) {\n                  approver = await getOneTrustUser({ oneTrust, userId });\n                  oneTrustCachedUsers[userId] = approver;\n                }\n                return [approver];\n              } catch (e) {\n                logger.warn(\n                  colors.yellow(\n                    `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a form approver.` +\n                      `\\tapproverId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n                  ),\n                );\n                return [];\n              }\n            },\n            { concurrency: 5 },\n          );\n        }\n\n        // fetch assessment internal respondents information\n        const { respondents } = assessmentDetails;\n        // if a user is an internal respondents, their 'name' field can't be an email.\n        const internalRespondents = respondents.filter(\n          (r) => !r.name.includes('@'),\n        );\n        let respondentsDetails: OneTrustGetUserResponse[][] = [];\n        if (internalRespondents.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching respondents...`,\n          );\n          respondentsDetails = await map(\n            internalRespondents.map(({ id }) => id),\n            async (userId) => {\n              try {\n                let respondent = oneTrustCachedUsers[userId];\n                if (!respondent) {\n                  respondent = await getOneTrustUser({ oneTrust, userId });\n                  oneTrustCachedUsers[userId] = respondent;\n                }\n                return [respondent];\n              } catch (e) {\n                logger.warn(\n                  colors.yellow(\n                    `[assessment ${assessmentNumber} of ${assessments.length}]: failed to fetch a respondent.` +\n                      `\\trespondentId: ${userId}. Assessment Title: ${assessment.name}. Template Title: ${templateName}`,\n                  ),\n                );\n                return [];\n              }\n            },\n            { concurrency: 5 },\n          );\n        }\n\n        // fetch assessment risk information\n        let riskDetails: OneTrustGetRiskResponse[] = [];\n        const riskIds = uniq(\n          assessmentDetails.sections.flatMap((s: OneTrustAssessmentSection) =>\n            s.questions.flatMap((q: OneTrustAssessmentQuestion) =>\n              (q.risks ?? []).flatMap((r) => r.riskId),\n            ),\n          ),\n        );\n        if (riskIds.length > 0) {\n          logger.info(\n            `[assessment ${assessmentNumber} of ${assessments.length}]: fetching risks...`,\n          );\n          riskDetails = await map(\n            riskIds,\n            (riskId) => getOneTrustRisk({ oneTrust, riskId: riskId as string }),\n            {\n              concurrency: 5,\n            },\n          );\n        }\n\n        // enrich the assessments with user and risk details\n        const enrichedAssessment = enrichOneTrustAssessment({\n          assessment,\n          assessmentDetails,\n          riskDetails,\n          creatorDetails: creator,\n          approversDetails: approversDetails.flat(),\n          respondentsDetails: respondentsDetails.flat(),\n        });\n\n        batchEnrichedAssessments.push(enrichedAssessment);\n      },\n      { concurrency: BATCH_SIZE },\n    );\n\n    // sync assessments in series to avoid concurrency bugs\n    await mapSeries(\n      batchEnrichedAssessments,\n      async (enrichedAssessment, index) => {\n        // the assessment's global index takes its batch into consideration\n        const globalIndex = batch * BATCH_SIZE + index;\n\n        if (dryRun && file) {\n          // sync to file\n          syncOneTrustAssessmentToDisk({\n            assessment: enrichedAssessment,\n            index: globalIndex,\n            total: assessments.length,\n            file,\n          });\n        } else if (transcend) {\n          // sync to transcend\n          await syncOneTrustAssessmentToTranscend({\n            assessment: enrichedAssessment,\n            transcend,\n            total: assessments.length,\n            index: globalIndex,\n          });\n        }\n      },\n    );\n  });\n};\n"]}

package/dist/{impl-PHDAL2KZ.cjs → impl-SMK2SUTD.cjs}

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkHVH5FPKYcjs = require('./chunk-HVH5FPKY.cjs');var _chunkLR3CPNDMcjs = require('./chunk-LR3CPNDM.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkCUUCCUX2cjs = require('./chunk-CUUCCUX2.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkP7WCYR3Acjs = require('./chunk-P7WCYR3A.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function Z({file:l,fileTarget:f,transcendUrl:p,auth:g,sombraAuth:x,cronDataSiloId:v,targetDataSiloId:N,actions:R,skipRequestCount:h,pageLimit:s,chunkSize:r}){h&&_chunkZUNVPK23cjs.a.info(_colors2.default.yellow("Skipping request count as requested. This may help speed up the call.")),(Number.isNaN(r)||r<=0||r%s!==0)&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Invalid chunk size: "${r}". Must be a positive integer that is a multiple of ${s}.`)),this.process.exit(1)),_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let S=_chunkCUUCCUX2cjs.wc.call(void 0, p,g),{baseName:j,extension:O}=_chunkLR3CPNDMcjs.d.call(void 0, l),{baseName:A,extension:D}=_chunkLR3CPNDMcjs.d.call(void 0, f),$=0,b=0,d=0;await _chunkHVH5FPKYcjs.g.call(void 0, {dataSiloId:v,auth:g,sombraAuth:x,actions:R,apiPageSize:s,savePageSize:r,onSave:async n=>{$+=n.length;let k=n.map(t=>t.requestId),E=_chunkP7WCYR3Acjs.j.call(void 0, k),G=_chunkP7WCYR3Acjs.b.call(void 0, E,s),y=await _bluebird.map.call(void 0, G,async t=>(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Fetching target identifiers for ${t.length} requests`)),(await _chunkCUUCCUX2cjs.Id.call(void 0, S,s*2,{requestIds:t,dataSiloIds:[N]})).map(({fileName:I,remoteId:C})=>{if(!C)throw new Error(`Failed to find remoteId for ${I}`);return{RecordId:C,Object:_optionalChain([I, 'access', _ => _.replace, 'call', _2 => _2(".json",""), 'access', _3 => _3.split, 'call', _4 => _4("/"), 'access', _5 => _5.pop, 'call', _6 => _6(), 'optionalAccess', _7 => _7.replace, 'call', _8 => _8(" Information","")]),Comment:"Customer data deletion request submitted via transcend.io"}})),{concurrency:1});b+=y.flat().length;let M=_chunkP7WCYR3Acjs.j.call(void 0, n.map(t=>Object.keys(t)).flat()),Q=`${j}-${d}${O}`,T=`${A}-${d}${D}`;_chunkLR3CPNDMcjs.c.call(void 0, Q,n,M),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${n.length} identifiers to file "${l}"`));let o=y.flat(),V=_chunkP7WCYR3Acjs.j.call(void 0, o.map(t=>Object.keys(t)).flat());_chunkLR3CPNDMcjs.c.call(void 0, T,o,V),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${o.length} identifiers to file "${f}"`)),_chunkZUNVPK23cjs.a.info(_colors2.default.blue(`Processed chunk of ${_chunkP7WCYR3Acjs.b.length} identifiers, found ${o.length} target identifiers`)),d+=1},transcendUrl:p,skipRequestCount:h}),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${$} identifiers to file "${l}"`)),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${b} identifiers to file "${f}"`))}exports.pullProfiles = Z;
-//# sourceMappingURL=impl-PHDAL2KZ.cjs.map
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkNA6JTKNJcjs = require('./chunk-NA6JTKNJ.cjs');var _chunkLR3CPNDMcjs = require('./chunk-LR3CPNDM.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkVY55L6P5cjs = require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');async function Z({file:l,fileTarget:f,transcendUrl:p,auth:g,sombraAuth:x,cronDataSiloId:v,targetDataSiloId:N,actions:R,skipRequestCount:h,pageLimit:s,chunkSize:r}){h&&_chunkZUNVPK23cjs.a.info(_colors2.default.yellow("Skipping request count as requested. This may help speed up the call.")),(Number.isNaN(r)||r<=0||r%s!==0)&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Invalid chunk size: "${r}". Must be a positive integer that is a multiple of ${s}.`)),this.process.exit(1)),_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let S=_chunkKKAJPI6Ocjs.wc.call(void 0, p,g),{baseName:j,extension:O}=_chunkLR3CPNDMcjs.d.call(void 0, l),{baseName:A,extension:D}=_chunkLR3CPNDMcjs.d.call(void 0, f),$=0,b=0,d=0;await _chunkNA6JTKNJcjs.g.call(void 0, {dataSiloId:v,auth:g,sombraAuth:x,actions:R,apiPageSize:s,savePageSize:r,onSave:async n=>{$+=n.length;let k=n.map(t=>t.requestId),E=_chunkVY55L6P5cjs.j.call(void 0, k),G=_chunkVY55L6P5cjs.b.call(void 0, E,s),y=await _bluebird.map.call(void 0, G,async t=>(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Fetching target identifiers for ${t.length} requests`)),(await _chunkKKAJPI6Ocjs.Id.call(void 0, S,s*2,{requestIds:t,dataSiloIds:[N]})).map(({fileName:I,remoteId:C})=>{if(!C)throw new Error(`Failed to find remoteId for ${I}`);return{RecordId:C,Object:_optionalChain([I, 'access', _ => _.replace, 'call', _2 => _2(".json",""), 'access', _3 => _3.split, 'call', _4 => _4("/"), 'access', _5 => _5.pop, 'call', _6 => _6(), 'optionalAccess', _7 => _7.replace, 'call', _8 => _8(" Information","")]),Comment:"Customer data deletion request submitted via transcend.io"}})),{concurrency:1});b+=y.flat().length;let M=_chunkVY55L6P5cjs.j.call(void 0, n.map(t=>Object.keys(t)).flat()),Q=`${j}-${d}${O}`,T=`${A}-${d}${D}`;_chunkLR3CPNDMcjs.c.call(void 0, Q,n,M),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${n.length} identifiers to file "${l}"`));let o=y.flat(),V=_chunkVY55L6P5cjs.j.call(void 0, o.map(t=>Object.keys(t)).flat());_chunkLR3CPNDMcjs.c.call(void 0, T,o,V),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${o.length} identifiers to file "${f}"`)),_chunkZUNVPK23cjs.a.info(_colors2.default.blue(`Processed chunk of ${_chunkVY55L6P5cjs.b.length} identifiers, found ${o.length} target identifiers`)),d+=1},transcendUrl:p,skipRequestCount:h}),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${$} identifiers to file "${l}"`)),_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully wrote ${b} identifiers to file "${f}"`))}exports.pullProfiles = Z;
+//# sourceMappingURL=impl-SMK2SUTD.cjs.map

package/dist/{impl-PHDAL2KZ.cjs.map → impl-SMK2SUTD.cjs.map}

@@ -1 +1 @@
-{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-PHDAL2KZ.cjs","../src/commands/request/cron/pull-profiles/impl.ts"],"names":["pullProfiles","file","fileTarget","transcendUrl","auth","sombraAuth","cronDataSiloId","targetDataSiloId","actions","skipRequestCount","pageLimit","chunkSize","logger","colors","doneInputValidation","client","buildTranscendGraphQLClient","baseName","extension","parseFilePath","baseNameTarget","extensionTarget","allIdentifiersCount","allTargetIdentifiersCount","fileCount","pullChunkedCustomSiloOutstandingIdentifiers","chunkToSave","requestIds","d","uniqueRequestIds","uniq_default","chunkedRequestIds","chunk_default","results","map","fetchRequestFilesForRequest","fileName","remoteId"],"mappings":"AAAA,quBAAwC,wDAAgD,wDAAyC,wDAAkD,gCAA6B,wDAAyC,wDAAgD,gCAA6B,gFCEnT,oCAEC,MA4BpB,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CACF,CAAA,CACe,CACXF,CAAAA,EACFG,mBAAAA,CAAO,IAAA,CACLC,gBAAAA,CAAO,MAAA,CACL,uEACF,CACF,CAAA,CAAA,CAIA,MAAA,CAAO,KAAA,CAAMF,CAAS,CAAA,EACtBA,CAAAA,EAAa,CAAA,EACbA,CAAAA,CAAYD,CAAAA,GAAc,CAAA,CAAA,EAAA,CAE1BE,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,qBAAA,EAAwBF,CAAS,CAAA,oDAAA,EAAuDD,CAAS,CAAA,CAAA,CACnG,CACF,CAAA,CACA,IAAA,CAAK,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,CAAA,CAGrBI,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAGrC,IAAMC,CAAAA,CAASC,kCAAAA,CAA4Bb,CAAcC,CAAI,CAAA,CACvD,CAAE,QAAA,CAAAa,CAAAA,CAAU,SAAA,CAAAC,CAAU,CAAA,CAAIC,iCAAAA,CAAkB,CAAA,CAC5C,CAAE,QAAA,CAAUC,CAAAA,CAAgB,SAAA,CAAWC,CAAgB,CAAA,CAC3DF,iCAAAA,CAAwB,CAAA,CAEtBG,CAAAA,CAAsB,CAAA,CACtBC,CAAAA,CAA4B,CAAA,CAC5BC,CAAAA,CAAY,CAAA,CAmFhB,MAAMC,iCAAAA,CACJ,UAAA,CAAYnB,CAAAA,CACZ,IAAA,CAAAF,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,OAAA,CAAAG,CAAAA,CACA,WAAA,CAAaE,CAAAA,CACb,YAAA,CAAcC,CAAAA,CACd,MAAA,CAxFa,MACbe,CAAAA,EACkB,CAElBJ,CAAAA,EAAuBI,CAAAA,CAAY,MAAA,CAGnC,IAAMC,CAAAA,CAAaD,CAAAA,CAAY,GAAA,CAAKE,CAAAA,EAAMA,CAAAA,CAAE,SAAmB,CAAA,CACzDC,CAAAA,CAAmBC,iCAAAA,CAAe,CAAA,CAGlCC,CAAAA,CAAoBC,iCAAAA,CAAMH,CAAkBnB,CAAS,CAAA,CACrDuB,CAAAA,CAAU,MAAMC,2BAAAA,CACpBH,CACA,MAAOJ,CAAAA,EAAAA,CACLf,mBAAAA,CAAO,IAAA,CACLC,gBAAAA,CAAO,OAAA,CACL,CAAA,gCAAA,EAAmCc,CAAAA,CAAW,MAAM,CAAA,SAAA,CACtD,CACF,CAAA,CAAA,CACgB,MAAMQ,kCAAAA,CACpBpB,CACAL,CAAAA,CAAY,CAAA,CACZ,CACE,UAAA,CAAAiB,CAAAA,CACA,WAAA,CAAa,CAACpB,CAAgB,CAChC,CACF,CAAA,CAAA,CACe,GAAA,CAAI,CAAC,CAAE,QAAA,CAAA6B,CAAAA,CAAU,QAAA,CAAAC,CAAS,CAAA,CAAA,EAAM,CAC7C,EAAA,CAAI,CAACA,CAAAA,CACH,MAAM,IAAI,KAAA,CAAM,CAAA,4BAAA,EAA+BD,CAAQ,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/impl-PHDAL2KZ.cjs","sourcesContent":[null,"import type { RequestAction } from '@transcend-io/privacy-types';\nimport { logger } from '../../../../logger';\nimport colors from 'colors';\nimport { uniq, chunk } from 'lodash-es';\nimport { map } from 'bluebird';\nimport {\n  buildTranscendGraphQLClient,\n  fetchRequestFilesForRequest,\n} from '../../../../lib/graphql';\nimport type { LocalContext } from '../../../../context';\nimport {\n  parseFilePath,\n  pullChunkedCustomSiloOutstandingIdentifiers,\n  writeCsv,\n  type CsvFormattedIdentifier,\n} from '../../../../lib/cron';\nimport { doneInputValidation } from '../../../../lib/cli/done-input-validation';\n\nexport interface PullProfilesCommandFlags {\n  file: string;\n  fileTarget: string;\n  transcendUrl: string;\n  auth: string;\n  sombraAuth?: string;\n  cronDataSiloId: string;\n  targetDataSiloId: string;\n  actions: RequestAction[];\n  skipRequestCount: boolean;\n  pageLimit: number;\n  chunkSize: number;\n}\n\nexport async function pullProfiles(\n  this: LocalContext,\n  {\n    file,\n    fileTarget,\n    transcendUrl,\n    auth,\n    sombraAuth,\n    cronDataSiloId,\n    targetDataSiloId,\n    actions,\n    skipRequestCount,\n    pageLimit,\n    chunkSize,\n  }: PullProfilesCommandFlags,\n): Promise<void> {\n  if (skipRequestCount) {\n    logger.info(\n      colors.yellow(\n        'Skipping request count as requested. This may help speed up the call.',\n      ),\n    );\n  }\n\n  if (\n    Number.isNaN(chunkSize) ||\n    chunkSize <= 0 ||\n    chunkSize % pageLimit !== 0\n  ) {\n    logger.error(\n      colors.red(\n        `Invalid chunk size: \"${chunkSize}\". Must be a positive integer that is a multiple of ${pageLimit}.`,\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  doneInputValidation(this.process.exit);\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const { baseName, extension } = parseFilePath(file);\n  const { baseName: baseNameTarget, extension: extensionTarget } =\n    parseFilePath(fileTarget);\n\n  let allIdentifiersCount = 0;\n  let allTargetIdentifiersCount = 0;\n  let fileCount = 0;\n  // Create onSave callback to handle chunked processing\n  const onSave = async (\n    chunkToSave: CsvFormattedIdentifier[],\n  ): Promise<void> => {\n    // Add to all identifiers\n    allIdentifiersCount += chunkToSave.length;\n\n    // Get unique request IDs from this chunk\n    const requestIds = chunkToSave.map((d) => d.requestId as string);\n    const uniqueRequestIds = uniq(requestIds);\n\n    // Pull down target identifiers for this chunk\n    const chunkedRequestIds = chunk(uniqueRequestIds, pageLimit);\n    const results = await map(\n      chunkedRequestIds,\n      async (requestIds) => {\n        logger.info(\n          colors.magenta(\n            `Fetching target identifiers for ${requestIds.length} requests`,\n          ),\n        );\n        const results = await fetchRequestFilesForRequest(\n          client,\n          pageLimit * 2,\n          {\n            requestIds,\n            dataSiloIds: [targetDataSiloId],\n          },\n        );\n        return results.map(({ fileName, remoteId }) => {\n          if (!remoteId) {\n            throw new Error(`Failed to find remoteId for ${fileName}`);\n          }\n          return {\n            RecordId: remoteId,\n            Object: fileName\n              .replace('.json', '')\n              .split('/')\n              .pop()\n              ?.replace(' Information', ''),\n            Comment:\n              'Customer data deletion request submitted via transcend.io',\n          };\n        });\n      },\n      // We are grabbing all the request files for the 'pageLimit' # of requests at a time\n      {\n        concurrency: 1,\n      },\n    );\n\n    allTargetIdentifiersCount += results.flat().length;\n\n    // Write the identifiers and target identifiers to CSV\n    const headers = uniq(chunkToSave.map((d) => Object.keys(d)).flat());\n    const numberedFileName = `${baseName}-${fileCount}${extension}`;\n    const numberedFileNameTarget = `${baseNameTarget}-${fileCount}${extensionTarget}`;\n    writeCsv(numberedFileName, chunkToSave, headers);\n    logger.info(\n      colors.green(\n        `Successfully wrote ${chunkToSave.length} identifiers to file \"${file}\"`,\n      ),\n    );\n\n    const targetIdentifiers = results.flat();\n    const headers2 = uniq(targetIdentifiers.map((d) => Object.keys(d)).flat());\n    writeCsv(numberedFileNameTarget, targetIdentifiers, headers2);\n    logger.info(\n      colors.green(\n        `Successfully wrote ${targetIdentifiers.length} identifiers to file \"${fileTarget}\"`,\n      ),\n    );\n\n    logger.info(\n      colors.blue(\n        `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`,\n      ),\n    );\n    fileCount += 1;\n  };\n\n  // Pull down outstanding identifiers using the new chunked function\n  await pullChunkedCustomSiloOutstandingIdentifiers({\n    dataSiloId: cronDataSiloId,\n    auth,\n    sombraAuth,\n    actions,\n    apiPageSize: pageLimit,\n    savePageSize: chunkSize,\n    onSave,\n    transcendUrl,\n    skipRequestCount,\n  });\n\n  logger.info(\n    colors.green(\n      `Successfully wrote ${allIdentifiersCount} identifiers to file \"${file}\"`,\n    ),\n  );\n  logger.info(\n    colors.green(\n      `Successfully wrote ${allTargetIdentifiersCount} identifiers to file \"${fileTarget}\"`,\n    ),\n  );\n}\n"]}
+{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-SMK2SUTD.cjs","../src/commands/request/cron/pull-profiles/impl.ts"],"names":["pullProfiles","file","fileTarget","transcendUrl","auth","sombraAuth","cronDataSiloId","targetDataSiloId","actions","skipRequestCount","pageLimit","chunkSize","logger","colors","doneInputValidation","client","buildTranscendGraphQLClient","baseName","extension","parseFilePath","baseNameTarget","extensionTarget","allIdentifiersCount","allTargetIdentifiersCount","fileCount","pullChunkedCustomSiloOutstandingIdentifiers","chunkToSave","requestIds","d","uniqueRequestIds","uniq_default","chunkedRequestIds","chunk_default","results","map","fetchRequestFilesForRequest","fileName","remoteId"],"mappings":"AAAA,quBAAwC,wDAAgD,wDAAyC,wDAAkD,gCAA6B,wDAAyC,wDAAgD,gCAA6B,gFCEnT,oCAEC,MA4BpB,SAAsBA,CAAAA,CAEpB,CACE,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,IAAA,CAAAC,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,cAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,OAAA,CAAAC,CAAAA,CACA,gBAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CAAAA,CACA,SAAA,CAAAC,CACF,CAAA,CACe,CACXF,CAAAA,EACFG,mBAAAA,CAAO,IAAA,CACLC,gBAAAA,CAAO,MAAA,CACL,uEACF,CACF,CAAA,CAAA,CAIA,MAAA,CAAO,KAAA,CAAMF,CAAS,CAAA,EACtBA,CAAAA,EAAa,CAAA,EACbA,CAAAA,CAAYD,CAAAA,GAAc,CAAA,CAAA,EAAA,CAE1BE,mBAAAA,CAAO,KAAA,CACLC,gBAAAA,CAAO,GAAA,CACL,CAAA,qBAAA,EAAwBF,CAAS,CAAA,oDAAA,EAAuDD,CAAS,CAAA,CAAA,CACnG,CACF,CAAA,CACA,IAAA,CAAK,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,CAAA,CAGrBI,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAGrC,IAAMC,CAAAA,CAASC,kCAAAA,CAA4Bb,CAAcC,CAAI,CAAA,CACvD,CAAE,QAAA,CAAAa,CAAAA,CAAU,SAAA,CAAAC,CAAU,CAAA,CAAIC,iCAAAA,CAAkB,CAAA,CAC5C,CAAE,QAAA,CAAUC,CAAAA,CAAgB,SAAA,CAAWC,CAAgB,CAAA,CAC3DF,iCAAAA,CAAwB,CAAA,CAEtBG,CAAAA,CAAsB,CAAA,CACtBC,CAAAA,CAA4B,CAAA,CAC5BC,CAAAA,CAAY,CAAA,CAmFhB,MAAMC,iCAAAA,CACJ,UAAA,CAAYnB,CAAAA,CACZ,IAAA,CAAAF,CAAAA,CACA,UAAA,CAAAC,CAAAA,CACA,OAAA,CAAAG,CAAAA,CACA,WAAA,CAAaE,CAAAA,CACb,YAAA,CAAcC,CAAAA,CACd,MAAA,CAxFa,MACbe,CAAAA,EACkB,CAElBJ,CAAAA,EAAuBI,CAAAA,CAAY,MAAA,CAGnC,IAAMC,CAAAA,CAAaD,CAAAA,CAAY,GAAA,CAAKE,CAAAA,EAAMA,CAAAA,CAAE,SAAmB,CAAA,CACzDC,CAAAA,CAAmBC,iCAAAA,CAAe,CAAA,CAGlCC,CAAAA,CAAoBC,iCAAAA,CAAMH,CAAkBnB,CAAS,CAAA,CACrDuB,CAAAA,CAAU,MAAMC,2BAAAA,CACpBH,CACA,MAAOJ,CAAAA,EAAAA,CACLf,mBAAAA,CAAO,IAAA,CACLC,gBAAAA,CAAO,OAAA,CACL,CAAA,gCAAA,EAAmCc,CAAAA,CAAW,MAAM,CAAA,SAAA,CACtD,CACF,CAAA,CAAA,CACgB,MAAMQ,kCAAAA,CACpBpB,CACAL,CAAAA,CAAY,CAAA,CACZ,CACE,UAAA,CAAAiB,CAAAA,CACA,WAAA,CAAa,CAACpB,CAAgB,CAChC,CACF,CAAA,CAAA,CACe,GAAA,CAAI,CAAC,CAAE,QAAA,CAAA6B,CAAAA,CAAU,QAAA,CAAAC,CAAS,CAAA,CAAA,EAAM,CAC7C,EAAA,CAAI,CAACA,CAAAA,CACH,MAAM,IAAI,KAAA,CAAM,CAAA,4BAAA,EAA+BD,CAAQ,CAAA,CAAA","file":"/home/runner/work/cli/cli/dist/impl-SMK2SUTD.cjs","sourcesContent":[null,"import type { RequestAction } from '@transcend-io/privacy-types';\nimport { logger } from '../../../../logger';\nimport colors from 'colors';\nimport { uniq, chunk } from 'lodash-es';\nimport { map } from 'bluebird';\nimport {\n  buildTranscendGraphQLClient,\n  fetchRequestFilesForRequest,\n} from '../../../../lib/graphql';\nimport type { LocalContext } from '../../../../context';\nimport {\n  parseFilePath,\n  pullChunkedCustomSiloOutstandingIdentifiers,\n  writeCsv,\n  type CsvFormattedIdentifier,\n} from '../../../../lib/cron';\nimport { doneInputValidation } from '../../../../lib/cli/done-input-validation';\n\nexport interface PullProfilesCommandFlags {\n  file: string;\n  fileTarget: string;\n  transcendUrl: string;\n  auth: string;\n  sombraAuth?: string;\n  cronDataSiloId: string;\n  targetDataSiloId: string;\n  actions: RequestAction[];\n  skipRequestCount: boolean;\n  pageLimit: number;\n  chunkSize: number;\n}\n\nexport async function pullProfiles(\n  this: LocalContext,\n  {\n    file,\n    fileTarget,\n    transcendUrl,\n    auth,\n    sombraAuth,\n    cronDataSiloId,\n    targetDataSiloId,\n    actions,\n    skipRequestCount,\n    pageLimit,\n    chunkSize,\n  }: PullProfilesCommandFlags,\n): Promise<void> {\n  if (skipRequestCount) {\n    logger.info(\n      colors.yellow(\n        'Skipping request count as requested. This may help speed up the call.',\n      ),\n    );\n  }\n\n  if (\n    Number.isNaN(chunkSize) ||\n    chunkSize <= 0 ||\n    chunkSize % pageLimit !== 0\n  ) {\n    logger.error(\n      colors.red(\n        `Invalid chunk size: \"${chunkSize}\". Must be a positive integer that is a multiple of ${pageLimit}.`,\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  doneInputValidation(this.process.exit);\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const { baseName, extension } = parseFilePath(file);\n  const { baseName: baseNameTarget, extension: extensionTarget } =\n    parseFilePath(fileTarget);\n\n  let allIdentifiersCount = 0;\n  let allTargetIdentifiersCount = 0;\n  let fileCount = 0;\n  // Create onSave callback to handle chunked processing\n  const onSave = async (\n    chunkToSave: CsvFormattedIdentifier[],\n  ): Promise<void> => {\n    // Add to all identifiers\n    allIdentifiersCount += chunkToSave.length;\n\n    // Get unique request IDs from this chunk\n    const requestIds = chunkToSave.map((d) => d.requestId as string);\n    const uniqueRequestIds = uniq(requestIds);\n\n    // Pull down target identifiers for this chunk\n    const chunkedRequestIds = chunk(uniqueRequestIds, pageLimit);\n    const results = await map(\n      chunkedRequestIds,\n      async (requestIds) => {\n        logger.info(\n          colors.magenta(\n            `Fetching target identifiers for ${requestIds.length} requests`,\n          ),\n        );\n        const results = await fetchRequestFilesForRequest(\n          client,\n          pageLimit * 2,\n          {\n            requestIds,\n            dataSiloIds: [targetDataSiloId],\n          },\n        );\n        return results.map(({ fileName, remoteId }) => {\n          if (!remoteId) {\n            throw new Error(`Failed to find remoteId for ${fileName}`);\n          }\n          return {\n            RecordId: remoteId,\n            Object: fileName\n              .replace('.json', '')\n              .split('/')\n              .pop()\n              ?.replace(' Information', ''),\n            Comment:\n              'Customer data deletion request submitted via transcend.io',\n          };\n        });\n      },\n      // We are grabbing all the request files for the 'pageLimit' # of requests at a time\n      {\n        concurrency: 1,\n      },\n    );\n\n    allTargetIdentifiersCount += results.flat().length;\n\n    // Write the identifiers and target identifiers to CSV\n    const headers = uniq(chunkToSave.map((d) => Object.keys(d)).flat());\n    const numberedFileName = `${baseName}-${fileCount}${extension}`;\n    const numberedFileNameTarget = `${baseNameTarget}-${fileCount}${extensionTarget}`;\n    writeCsv(numberedFileName, chunkToSave, headers);\n    logger.info(\n      colors.green(\n        `Successfully wrote ${chunkToSave.length} identifiers to file \"${file}\"`,\n      ),\n    );\n\n    const targetIdentifiers = results.flat();\n    const headers2 = uniq(targetIdentifiers.map((d) => Object.keys(d)).flat());\n    writeCsv(numberedFileNameTarget, targetIdentifiers, headers2);\n    logger.info(\n      colors.green(\n        `Successfully wrote ${targetIdentifiers.length} identifiers to file \"${fileTarget}\"`,\n      ),\n    );\n\n    logger.info(\n      colors.blue(\n        `Processed chunk of ${chunk.length} identifiers, found ${targetIdentifiers.length} target identifiers`,\n      ),\n    );\n    fileCount += 1;\n  };\n\n  // Pull down outstanding identifiers using the new chunked function\n  await pullChunkedCustomSiloOutstandingIdentifiers({\n    dataSiloId: cronDataSiloId,\n    auth,\n    sombraAuth,\n    actions,\n    apiPageSize: pageLimit,\n    savePageSize: chunkSize,\n    onSave,\n    transcendUrl,\n    skipRequestCount,\n  });\n\n  logger.info(\n    colors.green(\n      `Successfully wrote ${allIdentifiersCount} identifiers to file \"${file}\"`,\n    ),\n  );\n  logger.info(\n    colors.green(\n      `Successfully wrote ${allTargetIdentifiersCount} identifiers to file \"${fileTarget}\"`,\n    ),\n  );\n}\n"]}

package/dist/{impl-K4253LZW.cjs → impl-TJIMDEBN.cjs}

@@ -1,12 +1,12 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkCUUCCUX2cjs = require('./chunk-CUUCCUX2.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkP7WCYR3Acjs = require('./chunk-P7WCYR3A.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _persistedstate = require('@transcend-io/persisted-state');var _iots = require('io-ts'); var j = _interopRequireWildcard(_iots); var T = _interopRequireWildcard(_iots); var e = _interopRequireWildcard(_iots);var _privacytypes = require('@transcend-io/privacy-types');var _typeutils = require('@transcend-io/type-utils');var Ee=T.intersection([T.type({nodes:T.array(_privacytypes.PreferenceQueryResponseItem)}),T.partial({cursor:T.string})]),Oe=["ENOTFOUND","ETIMEDOUT","504 Gateway Time-out","Task timed out after"];async function ce(c,{identifiers:i,partitionKey:h,skipLogging:d=!1}){let a=[],u=_chunkP7WCYR3Acjs.b.call(void 0, i,100),l=new Date().getTime(),o=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);d||o.start(i.length,0);let f=0;await _bluebird.map.call(void 0, u,async t=>{let n=0,g=3;for(;n<g;)try{let m=await c.post(`v1/preferences/${h}/query`,{json:{filter:{identifiers:t},limit:t.length}}).json(),w=_typeutils.decodeCodec.call(void 0, Ee,m);a.push(...w.nodes),f+=t.length,o.update(f);break}catch(m){n+=1;let w=_optionalChain([m, 'optionalAccess', _2 => _2.response, 'optionalAccess', _3 => _3.body])||_optionalChain([m, 'optionalAccess', _4 => _4.message])||"";if(n>=g||!Oe.some(P=>w.includes(P)))throw new Error(`Received an error from server after ${n} attempts: ${w}`);_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(`[RETRYING FAILED REQUEST - Attempt ${n}] Failed to fetch ${t.length} user preferences from partition ${h}: ${w}`))}},{concurrency:40}),o.stop();let r=new Date().getTime()-l;return d||_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Completed download in "${r/1e3}" seconds.`)),a}function q({row:c,columnToPurposeName:i,purposeSlugs:h,preferenceTopics:d}){let a={};return Object.entries(i).forEach(([u,{purpose:l,preference:o,valueMapping:f}])=>{if(!h.includes(l))throw new Error(`Invalid purpose slug: ${l}, expected: ${h.join(", ")}`);let s=c[u];if(o){let r=d.find(t=>t.slug===o&&t.purpose.trackingType===l);if(!r){let t=d.filter(n=>n.purpose.trackingType===l).map(n=>n.slug);throw new Error(`Invalid preference slug: ${o} for purpose: ${l}. Allowed preference slugs for purpose are: ${t.join(",")}`)}switch(a[l]||(a[l]={preferences:[]}),a[l].preferences||(a[l].preferences=[]),r.type){case _privacytypes.PreferenceTopicType.Boolean:{let t=f[s];if(t===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(t==null)return;if(typeof t!="boolean")throw new Error(`Invalid value for boolean preference: ${o}, expected boolean, got: ${s}`);a[l].preferences.push({topic:o,choice:{booleanValue:t}});break}case _privacytypes.PreferenceTopicType.Select:{let t=f[s];if(t===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(t==null)return;if(typeof t!="string")throw new Error(`Invalid value for select preference: ${o}, expected string, got: ${s}`);let n=t.trim()||null;if(n&&!r.preferenceOptionValues.map(({slug:g})=>g).includes(n))throw new Error(`Invalid value for select preference: ${o}, expected one of: ${r.preferenceOptionValues.map(({slug:g})=>g).join(", ")}, got: ${s}`);a[l].preferences.push({topic:o,choice:{selectValue:n}});break}case _privacytypes.PreferenceTopicType.MultiSelect:{if(typeof s!="string")throw new Error(`Invalid value for multi select preference: ${o}, expected string, got: ${s}`);let t=_chunkCUUCCUX2cjs.oc.call(void 0, s).map(n=>{let g=f[n];if(g===void 0&&s!=="")throw new Error(`No preference mapping found for multi select token "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(g==null)return null;if(typeof g!="string")throw new Error(`Invalid value for multi select preference: ${o}, expected one of: ${r.preferenceOptionValues.map(({slug:m})=>m).join(", ")}, got: ${n}`);return g}).filter(n=>n!==null).sort((n,g)=>n.localeCompare(g));t.length>0&&a[l].preferences.push({topic:o,choice:{selectValues:t}});break}default:throw new Error(`Unknown preference type: ${r.type}`)}}else{let r=f[s];if(r===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=\u2205)`);if(r===null)return;a[l]?a[l].enabled=r===!0:a[l]={enabled:r===!0}}}),_typeutils.apply.call(void 0, a,(u,l)=>{if(typeof u.enabled!="boolean")throw new Error(`No mapping provided for purpose.enabled=true/false value: ${l}`);return{...u,enabled:u.enabled}})}var _inquirer = require('inquirer'); var _inquirer2 = _interopRequireDefault(_inquirer);var B="[NONE]";async function de(c,i){let h=_chunkP7WCYR3Acjs.j.call(void 0, c.map(a=>Object.keys(a)).flat()),d=_chunkP7WCYR3Acjs.c.call(void 0, h,[...i.identifierColumn?[i.identifierColumn]:[],...Object.keys(i.columnToPurposeName)]);if(!i.timestampColum){let{timestampName:a}=await _inquirer2.default.prompt([{name:"timestampName",message:"Choose the column that will be used as the timestamp of last preference update",type:"list",default:d.find(u=>u.toLowerCase().includes("date"))||d.find(u=>u.toLowerCase().includes("time"))||d[0],choices:[...d,B]}]);i.timestampColum=a}if(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Using timestamp column "${i.timestampColum}"`)),i.timestampColum!==B){let a=c.map((u,l)=>u[i.timestampColum]?null:[l]).filter(u=>!!u).flat();if(a.length>0)throw new Error(`The timestamp column "${i.timestampColum}" is missing a value for the following rows: ${a.join(`
-`)}`);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`The timestamp column "${i.timestampColum}" is present for all row`))}return i}async function ue(c,i){let h=_chunkP7WCYR3Acjs.j.call(void 0, c.map(o=>Object.keys(o)).flat()),d=_chunkP7WCYR3Acjs.c.call(void 0, h,[...i.identifierColumn?[i.identifierColumn]:[],...Object.keys(i.columnToPurposeName)]);if(!i.identifierColumn){let{identifierName:o}=await _inquirer2.default.prompt([{name:"identifierName",message:"Choose the column that will be used as the identifier to upload consent preferences by",type:"list",default:d.find(f=>f.toLowerCase().includes("email"))||d[0],choices:d}]);i.identifierColumn=o}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Using identifier column "${i.identifierColumn}"`));let a=c.map((o,f)=>o[i.identifierColumn]?null:[f]).filter(o=>!!o).flat();if(a.length>0){let o=`The identifier column "${i.identifierColumn}" is missing a value for the following rows: ${a.join(", ")}`;if(_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(o)),!await _chunkCUUCCUX2cjs.Rf.call(void 0, {message:"Would you like to skip rows missing an identifier?"}))throw new Error(o);let s=c.length;c=c.filter(r=>r[i.identifierColumn]),_chunkZUNVPK23cjs.a.info(_colors2.default.yellow(`Skipped ${s-c.length} rows missing an identifier`))}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`The identifier column "${i.identifierColumn}" is present for all rows`));let u=_chunkP7WCYR3Acjs.d.call(void 0, c,i.identifierColumn),l=Object.entries(u).filter(([,o])=>o.length>1);if(l.length>0){let o=`The identifier column "${i.identifierColumn}" has duplicate values for the following rows: ${l.slice(0,10).map(([s,r])=>`${s} (${r.length})`).join(`
-`)}`;if(_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(o)),!await _chunkCUUCCUX2cjs.Rf.call(void 0, {message:"Would you like to automatically take the latest update?"}))throw new Error(o);c=Object.entries(u).map(([,s])=>s.sort((t,n)=>new Date(n[i.timestampColum]).getTime()-new Date(t[i.timestampColum]).getTime())[0]).filter(s=>s)}return{currentState:i,preferences:c}}async function ge(c,i,{purposeSlugs:h,preferenceTopics:d,forceTriggerWorkflows:a}){let u=_chunkP7WCYR3Acjs.j.call(void 0, c.map(f=>Object.keys(f)).flat()),l=_chunkP7WCYR3Acjs.c.call(void 0, u,[...i.identifierColumn?[i.identifierColumn]:[],...i.timestampColum?[i.timestampColum]:[]]);if(l.length===0){if(a)return i;throw new Error("No other columns to process")}let o=[...h,...d.map(f=>`${f.purpose.trackingType}->${f.slug}`)];return await _bluebird.mapSeries.call(void 0, l,async f=>{let s=_chunkP7WCYR3Acjs.j.call(void 0, c.map(t=>t[f])),r=i.columnToPurposeName[f];if(r)_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Column "${f}" is associated with purpose "${r.purpose}"`));else{let{purposeName:t}=await _inquirer2.default.prompt([{name:"purposeName",message:`Choose the purpose that column ${f} is associated with`,type:"list",default:o.find(m=>m.startsWith(h[0])),choices:o}]),[n,g]=t.split("->");r={purpose:n,preference:g||null,valueMapping:{}}}await _bluebird.mapSeries.call(void 0, s,async t=>{if(r.valueMapping[t]!==void 0){_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Value "${t}" is associated with purpose value "${r.valueMapping[t]}"`));return}if(r.preference===null){let{purposeValue:n}=await _inquirer2.default.prompt([{name:"purposeValue",message:`Choose the purpose value for value "${t}" associated with purpose "${r.purpose}"`,type:"confirm",default:t!=="false"}]);r.valueMapping[t]=n}if(r.preference!==null){let n=d.find(m=>m.slug===r.preference);if(!n){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Preference topic "${r.preference}" not found`));return}let g=n.preferenceOptionValues.map(({slug:m})=>m);if(n.type===_privacytypes.PreferenceTopicType.Boolean){let{preferenceValue:m}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${t}" associated with purpose "${r.purpose}"`,type:"confirm",default:t!=="false"}]);r.valueMapping[t]=m;return}if(n.type===_privacytypes.PreferenceTopicType.Select){let{preferenceValue:m}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${t}" associated with purpose "${r.purpose}"`,type:"list",choices:g,default:g.find(w=>w===t)}]);r.valueMapping[t]=m;return}if(n.type===_privacytypes.PreferenceTopicType.MultiSelect){let m=_chunkCUUCCUX2cjs.oc.call(void 0, t);await _bluebird.mapSeries.call(void 0, m,async w=>{if(r.valueMapping[w]!==void 0)return;let{preferenceValue:P}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${w}" associated with purpose "${r.purpose}"`,type:"list",choices:g,default:g.find(y=>y===w)}]);r.valueMapping[w]=P});return}throw new Error(`Unknown preference topic type: ${n.type}`)}}),i.columnToPurposeName[f]=r}),i}function we({currentConsentRecord:c,pendingUpdates:i,preferenceTopics:h}){return Object.entries(i).every(([d,{preferences:a=[],enabled:u}])=>{let l=c.purposes.find(f=>f.purpose===d);return!!l&&l.enabled===u?a.every(({topic:f,choice:s})=>l.preferences&&l.preferences.find(r=>{if(r.topic!==f)return!1;let t=h.find(n=>n.slug===f&&n.purpose.trackingType===d);if(!t)throw new Error(`Could not find preference topic for ${f}`);switch(t.type){case _privacytypes.PreferenceTopicType.Boolean:return r.choice.booleanValue===s.booleanValue;case _privacytypes.PreferenceTopicType.Select:return r.choice.selectValue===s.selectValue;case _privacytypes.PreferenceTopicType.MultiSelect:let n=(r.choice.selectValues||[]).sort(),g=(s.selectValues||[]).sort();return n.length===g.length&&n.every((m,w)=>m===g[w]);default:throw new Error(`Unknown preference topic type: ${t.type}`)}})):!1})}function he({currentConsentRecord:c,pendingUpdates:i,preferenceTopics:h,log:d}){return!!Object.entries(i).find(([a,{preferences:u=[],enabled:l}])=>{let o=c.purposes.find(f=>f.purpose===a);return o?o.enabled!==l?(d&&_chunkZUNVPK23cjs.a.warn(`Purpose ${a} enabled value conflict for user ${c.userId}. Pending Value: ${l}, Current Value: ${o.enabled}`),!0):!!u.find(({topic:f,choice:s})=>{let r=(o.preferences||[]).find(m=>m.topic===f);if(!r)return d&&_chunkZUNVPK23cjs.a.warn(`No existing preference found for topic ${f} in purpose ${a} for user ${c.userId}.`),!1;let t=h.find(m=>m.slug===f&&m.purpose.trackingType===a);if(!t)throw new Error(`Could not find preference topic for ${f}`);let n,g;switch(t.type){case _privacytypes.PreferenceTopicType.Boolean:return n=r.choice.booleanValue!==s.booleanValue,d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} boolean value conflict for user ${c.userId}. Expected: ${s.booleanValue}, Found: ${r.choice.booleanValue}`),n;case _privacytypes.PreferenceTopicType.Select:return g=r.choice.selectValue!==s.selectValue,d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} select value conflict for user ${c.userId}. Expected: ${s.selectValue}, Found: ${r.choice.selectValue}`),g;case _privacytypes.PreferenceTopicType.MultiSelect:let m=(r.choice.selectValues||[]).sort(),w=(s.selectValues||[]).sort();return g=m.length!==w.length||!m.every((P,y)=>P===w[y]),d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} multi-select value conflict for user ${c.userId}. Expected: ${w.join(", ")}, Found: ${m.join(", ")}`),g;default:throw new Error(`Unknown preference topic type: ${t.type}`)}}):(d&&_chunkZUNVPK23cjs.a.warn(`No existing purpose found for ${a} in consent record for ${c.userId}.`),!1)})}async function Pe({file:c,sombra:i,purposeSlugs:h,preferenceTopics:d,partitionKey:a,skipExistingRecordCheck:u,forceTriggerWorkflows:l},o){let f=new Date().getTime(),s=o.getValue("fileMetadata");_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Reading in file: "${c}"`));let r=_chunkCUUCCUX2cjs.rc.call(void 0, c,j.record(j.string,j.string)),t={columnToPurposeName:{},pendingSafeUpdates:{},pendingConflictUpdates:{},skippedUpdates:{},...s[c]||{},lastFetchedAt:new Date().toISOString()};t=await de(r,t),s[c]=t,await o.setValue(s,"fileMetadata");let n=await ue(r,t);t=n.currentState,r=n.preferences,s[c]=t,await o.setValue(s,"fileMetadata"),t=await ge(r,t,{preferenceTopics:d,purposeSlugs:h,forceTriggerWorkflows:l}),s[c]=t,await o.setValue(s,"fileMetadata");let g=r.map(y=>y[t.identifierColumn]),m=u?[]:await ce(i,{identifiers:g.map(y=>({value:y})),partitionKey:a}),w=_chunkP7WCYR3Acjs.e.call(void 0, m,"userId");t.pendingConflictUpdates={},t.pendingSafeUpdates={},t.skippedUpdates={},r.forEach(y=>{let b=y[t.identifierColumn],S=q({row:y,columnToPurposeName:t.columnToPurposeName,preferenceTopics:d,purposeSlugs:h}),v=w[b];if(l&&!v)throw new Error(`No existing consent record found for user with id: ${b}. 
-        When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`);if(v&&we({currentConsentRecord:v,pendingUpdates:S,preferenceTopics:d})&&!l){t.skippedUpdates[b]=y;return}if(v&&he({currentConsentRecord:v,pendingUpdates:S,preferenceTopics:d})){t.pendingConflictUpdates[b]={row:y,record:v};return}t.pendingSafeUpdates[b]=y}),s[c]=t,await o.setValue(s,"fileMetadata");let P=new Date().getTime();_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pre-processed file: "${c}" in ${(P-f)/1e3}s`))}var be=e.type({purpose:e.string,preference:e.union([e.string,e.null]),valueMapping:e.record(e.string,e.union([e.string,e.boolean,e.null,e.undefined]))}),Kt=e.record(e.string,be),Ae=e.type({name:e.string,isUniqueOnPreferenceStore:e.boolean}),Yt=e.record(e.string,Ae),De=e.intersection([e.type({columnToPurposeName:e.record(e.string,be),lastFetchedAt:e.string,pendingSafeUpdates:e.record(e.string,e.record(e.string,e.string)),pendingConflictUpdates:e.record(e.string,e.type({record:_privacytypes.PreferenceQueryResponseItem,row:e.record(e.string,e.string)})),skippedUpdates:e.record(e.string,e.record(e.string,e.string))}),e.partial({identifierColumn:e.string,timestampColum:e.string})]),Jt=e.record(e.string,e.union([e.boolean,_privacytypes.PreferenceUpdateItem])),zt=e.record(e.string,e.union([e.boolean,e.record(e.string,e.string)])),Ht=e.record(e.string,e.type({uploadedAt:e.string,error:e.string,update:_privacytypes.PreferenceUpdateItem})),Xt=e.record(e.string,e.type({record:_privacytypes.PreferenceQueryResponseItem,row:e.record(e.string,e.string)})),Zt=e.record(e.string,e.record(e.string,e.string)),Ce=e.type({fileMetadata:e.record(e.string,De),failingUpdates:e.record(e.string,e.type({uploadedAt:e.string,error:e.string,update:_privacytypes.PreferenceUpdateItem})),pendingUpdates:e.record(e.string,_privacytypes.PreferenceUpdateItem)});async function Ue({auth:c,sombraAuth:i,receiptFilepath:h,file:d,partition:a,isSilent:u=!0,dryRun:l=!1,skipWorkflowTriggers:o=!1,skipConflictUpdates:f=!1,skipExistingRecordCheck:s=!1,attributes:r=[],transcendUrl:t,forceTriggerWorkflows:n=!1}){let g=_chunkCUUCCUX2cjs.qc.call(void 0, r),m=new (0, _persistedstate.PersistedState)(h,Ce,{fileMetadata:{},failingUpdates:{},pendingUpdates:{}}),w=m.getValue("failingUpdates"),P=m.getValue("pendingUpdates"),y=m.getValue("fileMetadata");_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Restored cache, there are: 
-${Object.values(w).length} failing requests to be retried
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');var _chunkZUNVPK23cjs = require('./chunk-ZUNVPK23.cjs');var _chunkVY55L6P5cjs = require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');var _colors = require('colors'); var _colors2 = _interopRequireDefault(_colors);var _bluebird = require('bluebird');var _cliprogress = require('cli-progress'); var _cliprogress2 = _interopRequireDefault(_cliprogress);var _persistedstate = require('@transcend-io/persisted-state');var _iots = require('io-ts'); var j = _interopRequireWildcard(_iots); var T = _interopRequireWildcard(_iots); var e = _interopRequireWildcard(_iots);var _privacytypes = require('@transcend-io/privacy-types');var _typeutils = require('@transcend-io/type-utils');var Oe=T.intersection([T.type({nodes:T.array(_privacytypes.PreferenceQueryResponseItem)}),T.partial({cursor:T.string})]),Ee=["ENOTFOUND","ETIMEDOUT","504 Gateway Time-out","Task timed out after"];async function ce(c,{identifiers:i,partitionKey:w,skipLogging:d=!1}){let a=[],u=_chunkVY55L6P5cjs.b.call(void 0, i,100),l=new Date().getTime(),o=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic);d||o.start(i.length,0);let f=0;await _bluebird.map.call(void 0, u,async t=>{let n=0,g=3;for(;n<g;)try{let m=await c.post(`v1/preferences/${w}/query`,{json:{filter:{identifiers:t},limit:t.length}}).json(),y=_typeutils.decodeCodec.call(void 0, Oe,m);a.push(...y.nodes),f+=t.length,o.update(f);break}catch(m){n+=1;let y=_optionalChain([m, 'optionalAccess', _2 => _2.response, 'optionalAccess', _3 => _3.body])||_optionalChain([m, 'optionalAccess', _4 => _4.message])||"";if(n>=g||!Ee.some(P=>y.includes(P)))throw new Error(`Received an error from server after ${n} attempts: ${y}`);_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(`[RETRYING FAILED REQUEST - Attempt ${n}] Failed to fetch ${t.length} user preferences from partition ${w}: ${y}`))}},{concurrency:40}),o.stop();let r=new Date().getTime()-l;return d||_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Completed download in "${r/1e3}" seconds.`)),a}function B({row:c,columnToPurposeName:i,purposeSlugs:w,preferenceTopics:d}){let a={};return Object.entries(i).forEach(([u,{purpose:l,preference:o,valueMapping:f}])=>{if(!w.includes(l))throw new Error(`Invalid purpose slug: ${l}, expected: ${w.join(", ")}`);let s=c[u];if(o){let r=d.find(t=>t.slug===o&&t.purpose.trackingType===l);if(!r){let t=d.filter(n=>n.purpose.trackingType===l).map(n=>n.slug);throw new Error(`Invalid preference slug: ${o} for purpose: ${l}. Allowed preference slugs for purpose are: ${t.join(",")}`)}switch(a[l]||(a[l]={preferences:[]}),a[l].preferences||(a[l].preferences=[]),r.type){case _privacytypes.PreferenceTopicType.Boolean:{let t=f[s];if(t===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(t==null)return;if(typeof t!="boolean")throw new Error(`Invalid value for boolean preference: ${o}, expected boolean, got: ${s}`);a[l].preferences.push({topic:o,choice:{booleanValue:t}});break}case _privacytypes.PreferenceTopicType.Select:{let t=f[s];if(t===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(t==null)return;if(typeof t!="string")throw new Error(`Invalid value for select preference: ${o}, expected string, got: ${s}`);let n=t.trim()||null;if(n&&!r.preferenceOptionValues.map(({slug:g})=>g).includes(n))throw new Error(`Invalid value for select preference: ${o}, expected one of: ${r.preferenceOptionValues.map(({slug:g})=>g).join(", ")}, got: ${s}`);a[l].preferences.push({topic:o,choice:{selectValue:n}});break}case _privacytypes.PreferenceTopicType.MultiSelect:{if(typeof s!="string")throw new Error(`Invalid value for multi select preference: ${o}, expected string, got: ${s}`);let t=_chunkKKAJPI6Ocjs.oc.call(void 0, s).map(n=>{let g=f[n];if(g===void 0&&s!=="")throw new Error(`No preference mapping found for multi select token "${s}" in column "${u}" (purpose=${l}, preference=${o})`);if(g==null)return null;if(typeof g!="string")throw new Error(`Invalid value for multi select preference: ${o}, expected one of: ${r.preferenceOptionValues.map(({slug:m})=>m).join(", ")}, got: ${n}`);return g}).filter(n=>n!==null).sort((n,g)=>n.localeCompare(g));t.length>0&&a[l].preferences.push({topic:o,choice:{selectValues:t}});break}default:throw new Error(`Unknown preference type: ${r.type}`)}}else{let r=f[s];if(r===void 0&&s!=="")throw new Error(`No preference mapping found for value "${s}" in column "${u}" (purpose=${l}, preference=\u2205)`);if(r===null)return;a[l]?a[l].enabled=r===!0:a[l]={enabled:r===!0}}}),_typeutils.apply.call(void 0, a,(u,l)=>{if(typeof u.enabled!="boolean")throw new Error(`No mapping provided for purpose.enabled=true/false value: ${l}`);return{...u,enabled:u.enabled}})}var _inquirer = require('inquirer'); var _inquirer2 = _interopRequireDefault(_inquirer);var q="[NONE]";async function de(c,i){let w=_chunkVY55L6P5cjs.j.call(void 0, c.map(a=>Object.keys(a)).flat()),d=_chunkVY55L6P5cjs.c.call(void 0, w,[...i.identifierColumn?[i.identifierColumn]:[],...Object.keys(i.columnToPurposeName)]);if(!i.timestampColum){let{timestampName:a}=await _inquirer2.default.prompt([{name:"timestampName",message:"Choose the column that will be used as the timestamp of last preference update",type:"list",default:d.find(u=>u.toLowerCase().includes("date"))||d.find(u=>u.toLowerCase().includes("time"))||d[0],choices:[...d,q]}]);i.timestampColum=a}if(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Using timestamp column "${i.timestampColum}"`)),i.timestampColum!==q){let a=c.map((u,l)=>u[i.timestampColum]?null:[l]).filter(u=>!!u).flat();if(a.length>0)throw new Error(`The timestamp column "${i.timestampColum}" is missing a value for the following rows: ${a.join(`
+`)}`);_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`The timestamp column "${i.timestampColum}" is present for all row`))}return i}async function ue(c,i){let w=_chunkVY55L6P5cjs.j.call(void 0, c.map(o=>Object.keys(o)).flat()),d=_chunkVY55L6P5cjs.c.call(void 0, w,[...i.identifierColumn?[i.identifierColumn]:[],...Object.keys(i.columnToPurposeName)]);if(!i.identifierColumn){let{identifierName:o}=await _inquirer2.default.prompt([{name:"identifierName",message:"Choose the column that will be used as the identifier to upload consent preferences by",type:"list",default:d.find(f=>f.toLowerCase().includes("email"))||d[0],choices:d}]);i.identifierColumn=o}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Using identifier column "${i.identifierColumn}"`));let a=c.map((o,f)=>o[i.identifierColumn]?null:[f]).filter(o=>!!o).flat();if(a.length>0){let o=`The identifier column "${i.identifierColumn}" is missing a value for the following rows: ${a.join(", ")}`;if(_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(o)),!await _chunkKKAJPI6Ocjs.Rf.call(void 0, {message:"Would you like to skip rows missing an identifier?"}))throw new Error(o);let s=c.length;c=c.filter(r=>r[i.identifierColumn]),_chunkZUNVPK23cjs.a.info(_colors2.default.yellow(`Skipped ${s-c.length} rows missing an identifier`))}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`The identifier column "${i.identifierColumn}" is present for all rows`));let u=_chunkVY55L6P5cjs.d.call(void 0, c,i.identifierColumn),l=Object.entries(u).filter(([,o])=>o.length>1);if(l.length>0){let o=`The identifier column "${i.identifierColumn}" has duplicate values for the following rows: ${l.slice(0,10).map(([s,r])=>`${s} (${r.length})`).join(`
+`)}`;if(_chunkZUNVPK23cjs.a.warn(_colors2.default.yellow(o)),!await _chunkKKAJPI6Ocjs.Rf.call(void 0, {message:"Would you like to automatically take the latest update?"}))throw new Error(o);c=Object.entries(u).map(([,s])=>s.sort((t,n)=>new Date(n[i.timestampColum]).getTime()-new Date(t[i.timestampColum]).getTime())[0]).filter(s=>s)}return{currentState:i,preferences:c}}async function ge(c,i,{purposeSlugs:w,preferenceTopics:d,forceTriggerWorkflows:a}){let u=_chunkVY55L6P5cjs.j.call(void 0, c.map(f=>Object.keys(f)).flat()),l=_chunkVY55L6P5cjs.c.call(void 0, u,[...i.identifierColumn?[i.identifierColumn]:[],...i.timestampColum?[i.timestampColum]:[]]);if(l.length===0){if(a)return i;throw new Error("No other columns to process")}let o=[...w,...d.map(f=>`${f.purpose.trackingType}->${f.slug}`)];return await _bluebird.mapSeries.call(void 0, l,async f=>{let s=_chunkVY55L6P5cjs.j.call(void 0, c.map(t=>t[f])),r=i.columnToPurposeName[f];if(r)_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Column "${f}" is associated with purpose "${r.purpose}"`));else{let{purposeName:t}=await _inquirer2.default.prompt([{name:"purposeName",message:`Choose the purpose that column ${f} is associated with`,type:"list",default:o.find(m=>m.startsWith(w[0])),choices:o}]),[n,g]=t.split("->");r={purpose:n,preference:g||null,valueMapping:{}}}await _bluebird.mapSeries.call(void 0, s,async t=>{if(r.valueMapping[t]!==void 0){_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Value "${t}" is associated with purpose value "${r.valueMapping[t]}"`));return}if(r.preference===null){let{purposeValue:n}=await _inquirer2.default.prompt([{name:"purposeValue",message:`Choose the purpose value for value "${t}" associated with purpose "${r.purpose}"`,type:"confirm",default:t!=="false"}]);r.valueMapping[t]=n}if(r.preference!==null){let n=d.find(m=>m.slug===r.preference);if(!n){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Preference topic "${r.preference}" not found`));return}let g=n.preferenceOptionValues.map(({slug:m})=>m);if(n.type===_privacytypes.PreferenceTopicType.Boolean){let{preferenceValue:m}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${t}" associated with purpose "${r.purpose}"`,type:"confirm",default:t!=="false"}]);r.valueMapping[t]=m;return}if(n.type===_privacytypes.PreferenceTopicType.Select){let{preferenceValue:m}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${t}" associated with purpose "${r.purpose}"`,type:"list",choices:g,default:g.find(y=>y===t)}]);r.valueMapping[t]=m;return}if(n.type===_privacytypes.PreferenceTopicType.MultiSelect){let m=_chunkKKAJPI6Ocjs.oc.call(void 0, t);await _bluebird.mapSeries.call(void 0, m,async y=>{if(r.valueMapping[y]!==void 0)return;let{preferenceValue:P}=await _inquirer2.default.prompt([{name:"preferenceValue",message:`Choose the preference value for "${n.slug}" value "${y}" associated with purpose "${r.purpose}"`,type:"list",choices:g,default:g.find(h=>h===y)}]);r.valueMapping[y]=P});return}throw new Error(`Unknown preference topic type: ${n.type}`)}}),i.columnToPurposeName[f]=r}),i}function ye({currentConsentRecord:c,pendingUpdates:i,preferenceTopics:w}){return Object.entries(i).every(([d,{preferences:a=[],enabled:u}])=>{let l=c.purposes.find(f=>f.purpose===d);return!!l&&l.enabled===u?a.every(({topic:f,choice:s})=>l.preferences&&l.preferences.find(r=>{if(r.topic!==f)return!1;let t=w.find(n=>n.slug===f&&n.purpose.trackingType===d);if(!t)throw new Error(`Could not find preference topic for ${f}`);switch(t.type){case _privacytypes.PreferenceTopicType.Boolean:return r.choice.booleanValue===s.booleanValue;case _privacytypes.PreferenceTopicType.Select:return r.choice.selectValue===s.selectValue;case _privacytypes.PreferenceTopicType.MultiSelect:let n=(r.choice.selectValues||[]).sort(),g=(s.selectValues||[]).sort();return n.length===g.length&&n.every((m,y)=>m===g[y]);default:throw new Error(`Unknown preference topic type: ${t.type}`)}})):!1})}function we({currentConsentRecord:c,pendingUpdates:i,preferenceTopics:w,log:d}){return!!Object.entries(i).find(([a,{preferences:u=[],enabled:l}])=>{let o=c.purposes.find(f=>f.purpose===a);return o?o.enabled!==l?(d&&_chunkZUNVPK23cjs.a.warn(`Purpose ${a} enabled value conflict for user ${c.userId}. Pending Value: ${l}, Current Value: ${o.enabled}`),!0):!!u.find(({topic:f,choice:s})=>{let r=(o.preferences||[]).find(m=>m.topic===f);if(!r)return d&&_chunkZUNVPK23cjs.a.warn(`No existing preference found for topic ${f} in purpose ${a} for user ${c.userId}.`),!1;let t=w.find(m=>m.slug===f&&m.purpose.trackingType===a);if(!t)throw new Error(`Could not find preference topic for ${f}`);let n,g;switch(t.type){case _privacytypes.PreferenceTopicType.Boolean:return n=r.choice.booleanValue!==s.booleanValue,d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} boolean value conflict for user ${c.userId}. Expected: ${s.booleanValue}, Found: ${r.choice.booleanValue}`),n;case _privacytypes.PreferenceTopicType.Select:return g=r.choice.selectValue!==s.selectValue,d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} select value conflict for user ${c.userId}. Expected: ${s.selectValue}, Found: ${r.choice.selectValue}`),g;case _privacytypes.PreferenceTopicType.MultiSelect:let m=(r.choice.selectValues||[]).sort(),y=(s.selectValues||[]).sort();return g=m.length!==y.length||!m.every((P,h)=>P===y[h]),d&&_chunkZUNVPK23cjs.a.warn(`Preference topic ${f} multi-select value conflict for user ${c.userId}. Expected: ${y.join(", ")}, Found: ${m.join(", ")}`),g;default:throw new Error(`Unknown preference topic type: ${t.type}`)}}):(d&&_chunkZUNVPK23cjs.a.warn(`No existing purpose found for ${a} in consent record for ${c.userId}.`),!1)})}async function Pe({file:c,sombra:i,purposeSlugs:w,preferenceTopics:d,partitionKey:a,skipExistingRecordCheck:u,forceTriggerWorkflows:l},o){let f=new Date().getTime(),s=o.getValue("fileMetadata");_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Reading in file: "${c}"`));let r=_chunkKKAJPI6Ocjs.rc.call(void 0, c,j.record(j.string,j.string)),t={columnToPurposeName:{},pendingSafeUpdates:{},pendingConflictUpdates:{},skippedUpdates:{},...s[c]||{},lastFetchedAt:new Date().toISOString()};t=await de(r,t),s[c]=t,await o.setValue(s,"fileMetadata");let n=await ue(r,t);t=n.currentState,r=n.preferences,s[c]=t,await o.setValue(s,"fileMetadata"),t=await ge(r,t,{preferenceTopics:d,purposeSlugs:w,forceTriggerWorkflows:l}),s[c]=t,await o.setValue(s,"fileMetadata");let g=r.map(h=>h[t.identifierColumn]),m=u?[]:await ce(i,{identifiers:g.map(h=>({value:h})),partitionKey:a}),y=_chunkVY55L6P5cjs.e.call(void 0, m,"userId");t.pendingConflictUpdates={},t.pendingSafeUpdates={},t.skippedUpdates={},r.forEach(h=>{let b=h[t.identifierColumn],F=B({row:h,columnToPurposeName:t.columnToPurposeName,preferenceTopics:d,purposeSlugs:w}),U=y[b];if(l&&!U)throw new Error(`No existing consent record found for user with id: ${b}. 
+        When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`);if(U&&ye({currentConsentRecord:U,pendingUpdates:F,preferenceTopics:d})&&!l){t.skippedUpdates[b]=h;return}if(U&&we({currentConsentRecord:U,pendingUpdates:F,preferenceTopics:d})){t.pendingConflictUpdates[b]={row:h,record:U};return}t.pendingSafeUpdates[b]=h}),s[c]=t,await o.setValue(s,"fileMetadata");let P=new Date().getTime();_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully pre-processed file: "${c}" in ${(P-f)/1e3}s`))}var be=e.type({purpose:e.string,preference:e.union([e.string,e.null]),valueMapping:e.record(e.string,e.union([e.string,e.boolean,e.null,e.undefined]))}),Kt=e.record(e.string,be),Ae=e.type({name:e.string,isUniqueOnPreferenceStore:e.boolean}),Yt=e.record(e.string,Ae),De=e.intersection([e.type({columnToPurposeName:e.record(e.string,be),lastFetchedAt:e.string,pendingSafeUpdates:e.record(e.string,e.record(e.string,e.string)),pendingConflictUpdates:e.record(e.string,e.type({record:_privacytypes.PreferenceQueryResponseItem,row:e.record(e.string,e.string)})),skippedUpdates:e.record(e.string,e.record(e.string,e.string))}),e.partial({identifierColumn:e.string,timestampColum:e.string})]),Jt=e.record(e.string,e.union([e.boolean,_privacytypes.PreferenceUpdateItem])),zt=e.record(e.string,e.union([e.boolean,e.record(e.string,e.string)])),Ht=e.record(e.string,e.type({uploadedAt:e.string,error:e.string,update:_privacytypes.PreferenceUpdateItem})),Xt=e.record(e.string,e.type({record:_privacytypes.PreferenceQueryResponseItem,row:e.record(e.string,e.string)})),Zt=e.record(e.string,e.record(e.string,e.string)),Ce=e.type({fileMetadata:e.record(e.string,De),failingUpdates:e.record(e.string,e.type({uploadedAt:e.string,error:e.string,update:_privacytypes.PreferenceUpdateItem})),pendingUpdates:e.record(e.string,_privacytypes.PreferenceUpdateItem)});async function ve({auth:c,sombraAuth:i,receiptFilepath:w,file:d,partition:a,isSilent:u=!0,dryRun:l=!1,skipWorkflowTriggers:o=!1,skipConflictUpdates:f=!1,skipExistingRecordCheck:s=!1,attributes:r=[],transcendUrl:t,forceTriggerWorkflows:n=!1}){let g=_chunkKKAJPI6Ocjs.qc.call(void 0, r),m=new (0, _persistedstate.PersistedState)(w,Ce,{fileMetadata:{},failingUpdates:{},pendingUpdates:{}}),y=m.getValue("failingUpdates"),P=m.getValue("pendingUpdates"),h=m.getValue("fileMetadata");_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Restored cache, there are: 
+${Object.values(y).length} failing requests to be retried
 ${Object.values(P).length} pending requests to be processed
 The following files are stored in cache and will be used:
-${Object.keys(y).map($=>$).join(`
+${Object.keys(h).map($=>$).join(`
 `)}
 The following file will be processed: ${d}
-`));let b=_chunkCUUCCUX2cjs.wc.call(void 0, t,c),[S,v,X]=await Promise.all([_chunkCUUCCUX2cjs.xc.call(void 0, t,c,i),n?Promise.resolve([]):_chunkCUUCCUX2cjs.fd.call(void 0, b),n?Promise.resolve([]):_chunkCUUCCUX2cjs.bd.call(void 0, b)]);await Pe({file:d,purposeSlugs:v.map($=>$.trackingType),preferenceTopics:X,sombra:S,partitionKey:a,skipExistingRecordCheck:s,forceTriggerWorkflows:n},m);let E={};y=m.getValue("fileMetadata");let x=y[d];if(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(x.pendingSafeUpdates).length} safe updates in ${d}`)),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(x.pendingConflictUpdates).length} conflict updates in ${d}`)),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(x.skippedUpdates).length} skipped updates in ${d}`)),Object.entries({...x.pendingSafeUpdates,...f?{}:_typeutils.apply.call(void 0, x.pendingConflictUpdates,({row:$})=>$)}).forEach(([$,C])=>{let A=x.timestampColum===B?new Date:new Date(C[x.timestampColum]),V=q({row:C,columnToPurposeName:x.columnToPurposeName,preferenceTopics:X,purposeSlugs:v.map(O=>O.trackingType)});E[$]={userId:$,partition:a,timestamp:A.toISOString(),purposes:Object.entries(V).map(([O,Ie])=>({...Ie,purpose:O,workflowSettings:{attributes:g,isSilent:u,skipWorkflowTrigger:o}}))}}),await m.setValue(E,"pendingUpdates"),await m.setValue({},"failingUpdates"),l){_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Dry run complete, exiting. ${Object.values(E).length} pending updates. Check file: ${h}`));return}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Uploading ${Object.values(E).length} preferences to partition: ${a}`));let Me=new Date().getTime(),Q=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),Z=0,L=Object.entries(E),xe=_chunkP7WCYR3Acjs.b.call(void 0, L,o?100:10);Q.start(L.length,0),await _bluebird.map.call(void 0, xe,async $=>{try{await S.put("v1/preferences",{json:{records:$.map(([,C])=>C),skipWorkflowTriggers:o,forceTriggerWorkflows:n}}).json()}catch(C){try{let V=JSON.parse(_optionalChain([C, 'optionalAccess', _5 => _5.response, 'optionalAccess', _6 => _6.body])||"{}");V.error&&_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Error: ${V.error}`))}catch (e2){}_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to upload ${$.length} user preferences to partition ${a}: ${_optionalChain([C, 'optionalAccess', _7 => _7.response, 'optionalAccess', _8 => _8.body])||_optionalChain([C, 'optionalAccess', _9 => _9.message])}`));let A=m.getValue("failingUpdates");$.forEach(([V,O])=>{A[V]={uploadedAt:new Date().toISOString(),update:O,error:_optionalChain([C, 'optionalAccess', _10 => _10.response, 'optionalAccess', _11 => _11.body])||_optionalChain([C, 'optionalAccess', _12 => _12.message])||"Unknown error"}}),await m.setValue(A,"failingUpdates")}Z+=$.length,Q.update(Z)},{concurrency:40}),Q.stop();let Ve=new Date().getTime()-Me;_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully uploaded ${L.length} user preferences to partition ${a} in "${Ve/1e3}" seconds!`))}var _fs = require('fs');var _path = require('path');async function jr({auth:c,partition:i,sombraAuth:h,transcendUrl:d,file:a="",directory:u,dryRun:l,skipExistingRecordCheck:o,receiptFileDir:f,skipWorkflowTriggers:s,forceTriggerWorkflows:r,skipConflictUpdates:t,isSilent:n,attributes:g,concurrency:m}){u&&a&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red("Cannot provide both a directory and a file. Please provide only one.")),this.process.exit(1)),!a&&!u&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red("A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences")),this.process.exit(1)),_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let w=[];if(u)try{let y=_fs.readdirSync.call(void 0, u).filter(b=>b.endsWith(".csv"));y.length===0&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red(`No CSV files found in directory: ${u}`)),this.process.exit(1)),w.push(...y.map(b=>_path.join.call(void 0, u,b)))}catch(P){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to read directory: ${u}`)),_chunkZUNVPK23cjs.a.error(_colors2.default.red(P.message)),this.process.exit(1)}else try{a.endsWith(".csv")||(_chunkZUNVPK23cjs.a.error(_colors2.default.red("File must be a CSV file")),this.process.exit(1)),w.push(a)}catch(P){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to access file: ${a}`)),_chunkZUNVPK23cjs.a.error(_colors2.default.red(P.message)),this.process.exit(1)}_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Processing ${w.length} consent preferences files for partition: ${i}`)),_chunkZUNVPK23cjs.a.debug(`Files to process: ${w.join(", ")}`),o&&_chunkZUNVPK23cjs.a.info(_colors2.default.bgYellow(`Skipping existing record check: ${o}`)),await _bluebird.map.call(void 0, w,async P=>{let y=_path.basename.call(void 0, P).replace(".csv","");await Ue({receiptFilepath:_path.join.call(void 0, f,`${y}-receipts.json`),auth:c,sombraAuth:h,file:P,partition:i,transcendUrl:d,skipConflictUpdates:t,skipWorkflowTriggers:s,skipExistingRecordCheck:o,isSilent:n,dryRun:l,attributes:_chunkCUUCCUX2cjs.oc.call(void 0, g),forceTriggerWorkflows:r})},{concurrency:m})}exports.uploadPreferences = jr;
-//# sourceMappingURL=impl-K4253LZW.cjs.map
+`));let b=_chunkKKAJPI6Ocjs.wc.call(void 0, t,c),[F,U,X]=await Promise.all([_chunkKKAJPI6Ocjs.xc.call(void 0, t,c,i),n?Promise.resolve([]):_chunkKKAJPI6Ocjs.fd.call(void 0, b),n?Promise.resolve([]):_chunkKKAJPI6Ocjs.bd.call(void 0, b)]);await Pe({file:d,purposeSlugs:U.map($=>$.trackingType),preferenceTopics:X,sombra:F,partitionKey:a,skipExistingRecordCheck:s,forceTriggerWorkflows:n},m);let O={};h=m.getValue("fileMetadata");let V=h[d];if(_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(V.pendingSafeUpdates).length} safe updates in ${d}`)),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(V.pendingConflictUpdates).length} conflict updates in ${d}`)),_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Found ${Object.entries(V.skippedUpdates).length} skipped updates in ${d}`)),Object.entries({...V.pendingSafeUpdates,...f?{}:_typeutils.apply.call(void 0, V.pendingConflictUpdates,({row:$})=>$)}).forEach(([$,C])=>{let A=V.timestampColum===q?new Date:new Date(C[V.timestampColum]),x=B({row:C,columnToPurposeName:V.columnToPurposeName,preferenceTopics:X,purposeSlugs:U.map(E=>E.trackingType)});O[$]={userId:$,partition:a,timestamp:A.toISOString(),purposes:Object.entries(x).map(([E,ke])=>({...ke,purpose:E,workflowSettings:{attributes:g,isSilent:u,skipWorkflowTrigger:o}}))}}),await m.setValue(O,"pendingUpdates"),await m.setValue({},"failingUpdates"),l){_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Dry run complete, exiting. ${Object.values(O).length} pending updates. Check file: ${w}`));return}_chunkZUNVPK23cjs.a.info(_colors2.default.magenta(`Uploading ${Object.values(O).length} preferences to partition: ${a}`));let Me=new Date().getTime(),W=new _cliprogress2.default.SingleBar({},_cliprogress2.default.Presets.shades_classic),Z=0,L=Object.entries(O),Ve=_chunkVY55L6P5cjs.b.call(void 0, L,o?100:10);W.start(L.length,0),await _bluebird.map.call(void 0, Ve,async $=>{try{await F.put("v1/preferences",{json:{records:$.map(([,C])=>C),skipWorkflowTriggers:o,forceTriggerWorkflows:n}}).json()}catch(C){try{let x=JSON.parse(_optionalChain([C, 'optionalAccess', _5 => _5.response, 'optionalAccess', _6 => _6.body])||"{}");x.error&&_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Error: ${x.error}`))}catch (e2){}_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to upload ${$.length} user preferences to partition ${a}: ${_optionalChain([C, 'optionalAccess', _7 => _7.response, 'optionalAccess', _8 => _8.body])||_optionalChain([C, 'optionalAccess', _9 => _9.message])}`));let A=m.getValue("failingUpdates");$.forEach(([x,E])=>{A[x]={uploadedAt:new Date().toISOString(),update:E,error:_optionalChain([C, 'optionalAccess', _10 => _10.response, 'optionalAccess', _11 => _11.body])||_optionalChain([C, 'optionalAccess', _12 => _12.message])||"Unknown error"}}),await m.setValue(A,"failingUpdates")}Z+=$.length,W.update(Z)},{concurrency:40}),W.stop();let xe=new Date().getTime()-Me;_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Successfully uploaded ${L.length} user preferences to partition ${a} in "${xe/1e3}" seconds!`))}var _fs = require('fs');var _path = require('path');async function Dr({auth:c,partition:i,sombraAuth:w,transcendUrl:d,file:a="",directory:u,dryRun:l,skipExistingRecordCheck:o,receiptFileDir:f,skipWorkflowTriggers:s,forceTriggerWorkflows:r,skipConflictUpdates:t,isSilent:n,attributes:g,concurrency:m}){u&&a&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red("Cannot provide both a directory and a file. Please provide only one.")),this.process.exit(1)),!a&&!u&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red("A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences")),this.process.exit(1)),_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let y=[];if(u)try{let h=_fs.readdirSync.call(void 0, u).filter(b=>b.endsWith(".csv"));h.length===0&&(_chunkZUNVPK23cjs.a.error(_colors2.default.red(`No CSV files found in directory: ${u}`)),this.process.exit(1)),y.push(...h.map(b=>_path.join.call(void 0, u,b)))}catch(P){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to read directory: ${u}`)),_chunkZUNVPK23cjs.a.error(_colors2.default.red(P.message)),this.process.exit(1)}else try{a.endsWith(".csv")||(_chunkZUNVPK23cjs.a.error(_colors2.default.red("File must be a CSV file")),this.process.exit(1)),y.push(a)}catch(P){_chunkZUNVPK23cjs.a.error(_colors2.default.red(`Failed to access file: ${a}`)),_chunkZUNVPK23cjs.a.error(_colors2.default.red(P.message)),this.process.exit(1)}_chunkZUNVPK23cjs.a.info(_colors2.default.green(`Processing ${y.length} consent preferences files for partition: ${i}`)),_chunkZUNVPK23cjs.a.debug(`Files to process: ${y.join(", ")}`),o&&_chunkZUNVPK23cjs.a.info(_colors2.default.bgYellow(`Skipping existing record check: ${o}`)),await _bluebird.map.call(void 0, y,async P=>{let h=_path.basename.call(void 0, P).replace(".csv","");await ve({receiptFilepath:_path.join.call(void 0, f,`${h}-receipts.json`),auth:c,sombraAuth:w,file:P,partition:i,transcendUrl:d,skipConflictUpdates:t,skipWorkflowTriggers:s,skipExistingRecordCheck:o,isSilent:n,dryRun:l,attributes:_chunkKKAJPI6Ocjs.oc.call(void 0, g),forceTriggerWorkflows:r})},{concurrency:m})}exports.uploadPreferences = Dr;
+//# sourceMappingURL=impl-TJIMDEBN.cjs.map

package/dist/{impl-K4253LZW.cjs.map → impl-TJIMDEBN.cjs.map}

@@ -1 +1 @@
-{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-K4253LZW.cjs","../src/commands/consent/upload-preferences/impl.ts","../src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts","../src/lib/preference-management/parsePreferenceManagementCsv.ts","../src/lib/preference-management/getPreferencesForIdentifiers.ts","../src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts","../src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts","../src/lib/preference-management/codecs.ts"],"names":["PreferenceRecordsQueryResponse","PreferenceQueryResponseItem","MSGS","getPreferencesForIdentifiers","sombra","identifiers","partitionKey","skipLogging","results","groupedIdentifiers","chunk_default","t0","progressBar","cliProgress","total","map","group","attempts","maxAttempts","rawResult","result","decodeCodec","err","msg","errorMessage"],"mappings":"AAAA,u/BAAyC,wDAAwG,gCAA6B,wDAAyC,wDAAuE,gCAA6B,gFCCxS,oCCQC,qGAGI,+DAEO,qJCXZ,2DCHyB,qDAKhB,IAKtBA,EAAAA,CAAmC,CAAA,CAAA,YAAA,CAAa,CAClD,CAAA,CAAA,IAAA,CAAK,CACL,KAAA,CAAS,CAAA,CAAA,KAAA,CAAMC,yCAA2B,CAC5C,CAAC,CAAA,CACC,CAAA,CAAA,OAAA,CAAQ,CAER,MAAA,CAAU,CAAA,CAAA,MACZ,CAAC,CACH,CAAC,CAAA,CAEKC,EAAAA,CAAO,CACX,WAAA,CACA,WAAA,CACA,sBAAA,CACA,sBACF,CAAA,CASA,MAAA,SAAsBC,EAAAA,CACpBC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,CAAA,CAChB,CAAA,CAWwC,CACxC,IAAMC,CAAAA,CAAyC,CAAC,CAAA,CAC1CC,CAAAA,CAAqBC,iCAAAA,CAAML,CAAa,GAAG,CAAA,CAG3CM,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CACxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CACKN,CAAAA,EACHK,CAAAA,CAAY,KAAA,CAAMP,CAAAA,CAAY,MAAA,CAAQ,CAAC,CAAA,CAGzC,IAAIS,CAAAA,CAAQ,CAAA,CACZ,MAAMC,2BAAAA,CACJN,CACA,MAAOO,CAAAA,EAAU,CAEf,IAAIC,CAAAA,CAAW,CAAA,CACTC,CAAAA,CAAc,CAAA,CACpB,GAAA,CAAA,CAAOD,CAAAA,CAAWC,CAAAA,CAAAA,CAChB,GAAI,CACF,IAAMC,CAAAA,CAAY,MAAMf,CAAAA,CACrB,IAAA,CAAK,CAAA,eAAA,EAAkBE,CAAY,CAAA,MAAA,CAAA,CAAU,CAC5C,IAAA,CAAM,CACJ,MAAA,CAAQ,CACN,WAAA,CAAaU,CACf,CAAA,CACA,KAAA,CAAOA,CAAAA,CAAM,MACf,CACF,CAAC,CAAA,CACA,IAAA,CAAK,CAAA,CAEFI,CAAAA,CAASC,oCAAAA,EAAYrB,CAAgCmB,CAAS,CAAA,CACpEX,CAAAA,CAAQ,IAAA,CAAK,GAAGY,CAAAA,CAAO,KAAK,CAAA,CAC5BN,CAAAA,EAASE,CAAAA,CAAM,MAAA,CACfJ,CAAAA,CAAY,MAAA,CAAOE,CAAK,CAAA,CACxB,KACF,CAAA,KAAA,CAASQ,CAAAA,CAAK,CACZL,CAAAA,EAAY,CAAA,CACZ,IAAMM,CAAAA,iBAAMD,CAAAA,6BAAK,QAAA,6BAAU,MAAA,kBAAQA,CAAAA,6BAAK,SAAA,EAAW,EAAA,CACnD,EAAA,CACEL,CAAAA,EAAYC,CAAAA,EACZ,CAAChB,EAAAA,CAAK,IAAA,CAAMsB,CAAAA,EAAiBD,CAAAA,CAAI,QAAA,CAASC,CAAY,CAAC,CAAA,CAEvD,MAAM,IAAI,KAAA,CACR,CAAA,oCAAA,EAAuCP,CAAQ,CAAA,WAAA,EAAcM,CAAG,CAAA,CAAA;ACclE;ACHS;ACkFjB,oGAAA;ALrGI;AAGA;AAAA;AAKQ;AAAK;AACgC,sCAAA;AAmIpB","file":"/home/runner/work/cli/cli/dist/impl-K4253LZW.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport colors from 'colors';\n\nimport { logger } from '../../../logger';\nimport { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management';\nimport { splitCsvToList } from '../../../lib/requests';\nimport { readdirSync } from 'node:fs';\nimport { map } from 'bluebird';\nimport { basename, join } from 'node:path';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadPreferencesCommandFlags {\n  auth: string;\n  partition: string;\n  sombraAuth?: string;\n  transcendUrl: string;\n  file?: string;\n  directory?: string;\n  dryRun: boolean;\n  skipExistingRecordCheck: boolean;\n  receiptFileDir: string;\n  skipWorkflowTriggers: boolean;\n  forceTriggerWorkflows: boolean;\n  skipConflictUpdates: boolean;\n  isSilent: boolean;\n  attributes: string;\n  receiptFilepath: string;\n  concurrency: number;\n}\n\nexport async function uploadPreferences(\n  this: LocalContext,\n  {\n    auth,\n    partition,\n    sombraAuth,\n    transcendUrl,\n    file = '',\n    directory,\n    dryRun,\n    skipExistingRecordCheck,\n    receiptFileDir,\n    skipWorkflowTriggers,\n    forceTriggerWorkflows,\n    skipConflictUpdates,\n    isSilent,\n    attributes,\n    concurrency,\n  }: UploadPreferencesCommandFlags,\n): Promise<void> {\n  if (!!directory && !!file) {\n    logger.error(\n      colors.red(\n        'Cannot provide both a directory and a file. Please provide only one.',\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  if (!file && !directory) {\n    logger.error(\n      colors.red(\n        'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences',\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  doneInputValidation(this.process.exit);\n\n  const files: string[] = [];\n\n  if (directory) {\n    try {\n      const filesInDirectory = readdirSync(directory);\n      const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv'));\n\n      if (csvFiles.length === 0) {\n        logger.error(\n          colors.red(`No CSV files found in directory: ${directory}`),\n        );\n        this.process.exit(1);\n      }\n\n      // Add full paths for each CSV file\n      files.push(...csvFiles.map((file) => join(directory, file)));\n    } catch (err) {\n      logger.error(colors.red(`Failed to read directory: ${directory}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  } else {\n    try {\n      // Verify file exists and is a CSV\n      if (!file.endsWith('.csv')) {\n        logger.error(colors.red('File must be a CSV file'));\n        this.process.exit(1);\n      }\n      files.push(file);\n    } catch (err) {\n      logger.error(colors.red(`Failed to access file: ${file}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  }\n\n  logger.info(\n    colors.green(\n      `Processing ${files.length} consent preferences files for partition: ${partition}`,\n    ),\n  );\n  logger.debug(`Files to process: ${files.join(', ')}`);\n\n  if (skipExistingRecordCheck) {\n    logger.info(\n      colors.bgYellow(\n        `Skipping existing record check: ${skipExistingRecordCheck}`,\n      ),\n    );\n  }\n\n  await map(\n    files,\n    async (filePath) => {\n      const fileName = basename(filePath).replace('.csv', '');\n      await uploadPreferenceManagementPreferencesInteractive({\n        receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`),\n        auth,\n        sombraAuth,\n        file: filePath,\n        partition,\n        transcendUrl,\n        skipConflictUpdates,\n        skipWorkflowTriggers,\n        skipExistingRecordCheck,\n        isSilent,\n        dryRun,\n        attributes: splitCsvToList(attributes),\n        forceTriggerWorkflows,\n      });\n    },\n    { concurrency },\n  );\n}\n","import {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllPurposes,\n  fetchAllPreferenceTopics,\n  PreferenceTopic,\n  Purpose,\n} from '../graphql';\nimport colors from 'colors';\nimport { map } from 'bluebird';\nimport { chunk } from 'lodash-es';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { parseAttributesFromString } from '../requests';\nimport { PersistedState } from '@transcend-io/persisted-state';\nimport { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv';\nimport { PreferenceState } from './codecs';\nimport { PreferenceUpdateItem } from '@transcend-io/privacy-types';\nimport { apply } from '@transcend-io/type-utils';\nimport { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\n\n/**\n * Upload a set of consent preferences\n *\n * @param options - Options\n */\nexport async function uploadPreferenceManagementPreferencesInteractive({\n  auth,\n  sombraAuth,\n  receiptFilepath,\n  file,\n  partition,\n  isSilent = true,\n  dryRun = false,\n  skipWorkflowTriggers = false,\n  skipConflictUpdates = false,\n  skipExistingRecordCheck = false,\n  attributes = [],\n  transcendUrl,\n  forceTriggerWorkflows = false,\n}: {\n  /** The Transcend API key */\n  auth: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Partition key */\n  partition: string;\n  /** File where to store receipt and continue from where left off */\n  receiptFilepath: string;\n  /** The file to process */\n  file: string;\n  /** API URL for Transcend backend */\n  transcendUrl: string;\n  /** Whether to do a dry run */\n  dryRun?: boolean;\n  /** Whether to upload as isSilent */\n  isSilent?: boolean;\n  /** Attributes string pre-parse. In format Key:Value */\n  attributes?: string[];\n  /** Skip workflow triggers */\n  skipWorkflowTriggers?: boolean;\n  /**\n   * When true, only update preferences that do not conflict with existing\n   * preferences. When false, update all preferences in CSV based on timestamp.\n   */\n  skipConflictUpdates?: boolean;\n  /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n  skipExistingRecordCheck?: boolean;\n  /** Whether to force trigger workflows */\n  forceTriggerWorkflows?: boolean;\n}): Promise<void> {\n  // Parse out the extra attributes to apply to all requests uploaded\n  const parsedAttributes = parseAttributesFromString(attributes);\n\n  // Create a new state file to store the requests from this run\n  const preferenceState = new PersistedState(receiptFilepath, PreferenceState, {\n    fileMetadata: {},\n    failingUpdates: {},\n    pendingUpdates: {},\n  });\n  const failingRequests = preferenceState.getValue('failingUpdates');\n  const pendingRequests = preferenceState.getValue('pendingUpdates');\n  let fileMetadata = preferenceState.getValue('fileMetadata');\n\n  logger.info(\n    colors.magenta(\n      'Restored cache, there are: \\n' +\n        `${\n          Object.values(failingRequests).length\n        } failing requests to be retried\\n` +\n        `${\n          Object.values(pendingRequests).length\n        } pending requests to be processed\\n` +\n        `The following files are stored in cache and will be used:\\n${Object.keys(\n          fileMetadata,\n        )\n          .map((x) => x)\n          .join('\\n')}\\n` +\n        `The following file will be processed: ${file}\\n`,\n    ),\n  );\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  const [sombra, purposes, preferenceTopics] = await Promise.all([\n    // Create sombra instance to communicate with\n    createSombraGotInstance(transcendUrl, auth, sombraAuth),\n    // get all purposes and topics\n    forceTriggerWorkflows\n      ? Promise.resolve([] as Purpose[])\n      : fetchAllPurposes(client),\n    forceTriggerWorkflows\n      ? Promise.resolve([] as PreferenceTopic[])\n      : fetchAllPreferenceTopics(client),\n  ]);\n\n  // Process the file\n  await parsePreferenceManagementCsvWithCache(\n    {\n      file,\n      purposeSlugs: purposes.map((x) => x.trackingType),\n      preferenceTopics,\n      sombra,\n      partitionKey: partition,\n      skipExistingRecordCheck,\n      forceTriggerWorkflows,\n    },\n    preferenceState,\n  );\n\n  // Construct the pending updates\n  const pendingUpdates: Record<string, PreferenceUpdateItem> = {};\n  fileMetadata = preferenceState.getValue('fileMetadata');\n  const metadata = fileMetadata[file];\n\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.pendingSafeUpdates).length\n      } safe updates in ${file}`,\n    ),\n  );\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.pendingConflictUpdates).length\n      } conflict updates in ${file}`,\n    ),\n  );\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.skippedUpdates).length\n      } skipped updates in ${file}`,\n    ),\n  );\n\n  // Update either safe updates only or safe + conflict\n  Object.entries({\n    ...metadata.pendingSafeUpdates,\n    ...(skipConflictUpdates\n      ? {}\n      : apply(metadata.pendingConflictUpdates, ({ row }) => row)),\n  }).forEach(([userId, update]) => {\n    // Determine timestamp\n    const timestamp =\n      metadata.timestampColum === NONE_PREFERENCE_MAP\n        ? new Date()\n        : new Date(update[metadata.timestampColum!]);\n\n    // Determine updates\n    const updates = getPreferenceUpdatesFromRow({\n      row: update,\n      columnToPurposeName: metadata.columnToPurposeName,\n      preferenceTopics,\n      purposeSlugs: purposes.map((x) => x.trackingType),\n    });\n    pendingUpdates[userId] = {\n      userId,\n      partition,\n      timestamp: timestamp.toISOString(),\n      purposes: Object.entries(updates).map(([purpose, value]) => ({\n        ...value,\n        purpose,\n        workflowSettings: {\n          attributes: parsedAttributes,\n          isSilent,\n          skipWorkflowTrigger: skipWorkflowTriggers,\n        },\n      })),\n    };\n  });\n  await preferenceState.setValue(pendingUpdates, 'pendingUpdates');\n  await preferenceState.setValue({}, 'failingUpdates');\n\n  // Exist early if dry run\n  if (dryRun) {\n    logger.info(\n      colors.green(\n        `Dry run complete, exiting. ${\n          Object.values(pendingUpdates).length\n        } pending updates. Check file: ${receiptFilepath}`,\n      ),\n    );\n    return;\n  }\n\n  logger.info(\n    colors.magenta(\n      `Uploading ${\n        Object.values(pendingUpdates).length\n      } preferences to partition: ${partition}`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar(\n    {},\n    cliProgress.Presets.shades_classic,\n  );\n\n  // Build a GraphQL client\n  let total = 0;\n  const updatesToRun = Object.entries(pendingUpdates);\n  const chunkedUpdates = chunk(updatesToRun, skipWorkflowTriggers ? 100 : 10);\n  progressBar.start(updatesToRun.length, 0);\n  await map(\n    chunkedUpdates,\n    async (currentChunk) => {\n      // Make the request\n      try {\n        await sombra\n          .put('v1/preferences', {\n            json: {\n              records: currentChunk.map(([, update]) => update),\n              skipWorkflowTriggers,\n              forceTriggerWorkflows,\n            },\n          })\n          .json();\n      } catch (err) {\n        try {\n          const parsed = JSON.parse(err?.response?.body || '{}');\n          if (parsed.error) {\n            logger.error(colors.red(`Error: ${parsed.error}`));\n          }\n        } catch (e) {\n          // continue\n        }\n        logger.error(\n          colors.red(\n            `Failed to upload ${\n              currentChunk.length\n            } user preferences to partition ${partition}: ${\n              err?.response?.body || err?.message\n            }`,\n          ),\n        );\n        const failingUpdates = preferenceState.getValue('failingUpdates');\n        currentChunk.forEach(([userId, update]) => {\n          failingUpdates[userId] = {\n            uploadedAt: new Date().toISOString(),\n            update,\n            error: err?.response?.body || err?.message || 'Unknown error',\n          };\n        });\n        await preferenceState.setValue(failingUpdates, 'failingUpdates');\n      }\n\n      total += currentChunk.length;\n      progressBar.update(total);\n    },\n    {\n      concurrency: 40,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n  logger.info(\n    colors.green(\n      `Successfully uploaded ${\n        updatesToRun.length\n      } user preferences to partition ${partition} in \"${\n        totalTime / 1000\n      }\" seconds!`,\n    ),\n  );\n}\n","import { PersistedState } from '@transcend-io/persisted-state';\nimport type { Got } from 'got';\nimport { keyBy } from 'lodash-es';\nimport * as t from 'io-ts';\nimport colors from 'colors';\nimport { FileMetadataState, PreferenceState } from './codecs';\nimport { logger } from '../../logger';\nimport { readCsv } from '../requests';\nimport { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers';\nimport { PreferenceTopic } from '../graphql';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\nimport { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv';\nimport { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv';\nimport { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv';\nimport { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp';\nimport { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict';\n\n/**\n * Parse a file into the cache\n *\n *\n * @param options - Options\n * @param cache - The cache to store the parsed file in\n * @returns The cache with the parsed file\n */\nexport async function parsePreferenceManagementCsvWithCache(\n  {\n    file,\n    sombra,\n    purposeSlugs,\n    preferenceTopics,\n    partitionKey,\n    skipExistingRecordCheck,\n    forceTriggerWorkflows,\n  }: {\n    /** File to parse */\n    file: string;\n    /** The purpose slugs that are allowed to be updated */\n    purposeSlugs: string[];\n    /** The preference topics */\n    preferenceTopics: PreferenceTopic[];\n    /** Sombra got instance */\n    sombra: Got;\n    /** Partition key */\n    partitionKey: string;\n    /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n    skipExistingRecordCheck: boolean;\n    /** Wheather to force workflow triggers */\n    forceTriggerWorkflows: boolean;\n  },\n  cache: PersistedState<typeof PreferenceState>,\n): Promise<void> {\n  // Start the timer\n  const t0 = new Date().getTime();\n\n  // Get the current metadata\n  const fileMetadata = cache.getValue('fileMetadata');\n\n  // Read in the file\n  logger.info(colors.magenta(`Reading in file: \"${file}\"`));\n  let preferences = readCsv(file, t.record(t.string, t.string));\n\n  // start building the cache, can use previous cache as well\n  let currentState: FileMetadataState = {\n    columnToPurposeName: {},\n    pendingSafeUpdates: {},\n    pendingConflictUpdates: {},\n    skippedUpdates: {},\n    // Load in the last fetched time\n    ...((fileMetadata[file] || {}) as Partial<FileMetadataState>),\n    lastFetchedAt: new Date().toISOString(),\n  };\n\n  // Validate that all timestamps are present in the file\n  currentState = await parsePreferenceTimestampsFromCsv(\n    preferences,\n    currentState,\n  );\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Validate that all identifiers are present and unique\n  const result = await parsePreferenceIdentifiersFromCsv(\n    preferences,\n    currentState,\n  );\n  currentState = result.currentState;\n  preferences = result.preferences;\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Ensure all other columns are mapped to purpose and preference\n  // slug values\n  currentState = await parsePreferenceAndPurposeValuesFromCsv(\n    preferences,\n    currentState,\n    {\n      preferenceTopics,\n      purposeSlugs,\n      forceTriggerWorkflows,\n    },\n  );\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Grab existing preference store records\n  const identifiers = preferences.map(\n    (pref) => pref[currentState.identifierColumn!],\n  );\n  const existingConsentRecords = skipExistingRecordCheck\n    ? []\n    : await getPreferencesForIdentifiers(sombra, {\n        identifiers: identifiers.map((x) => ({ value: x })),\n        partitionKey,\n      });\n  const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId');\n\n  // Clear out previous updates\n  currentState.pendingConflictUpdates = {};\n  currentState.pendingSafeUpdates = {};\n  currentState.skippedUpdates = {};\n\n  // Process each row\n  preferences.forEach((pref) => {\n    // Grab unique Id for the user\n    const userId = pref[currentState.identifierColumn!];\n\n    // determine updates for user\n    const pendingUpdates = getPreferenceUpdatesFromRow({\n      row: pref,\n      columnToPurposeName: currentState.columnToPurposeName,\n      preferenceTopics,\n      purposeSlugs,\n    });\n\n    // Grab current state of the update\n    const currentConsentRecord = consentRecordByIdentifier[userId];\n    if (forceTriggerWorkflows && !currentConsentRecord) {\n      throw new Error(\n        `No existing consent record found for user with id: ${userId}. \n        When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`,\n      );\n    }\n    // Check if the update can be skipped\n    // this is the case if a record exists, and the purpose\n    // and preference values are all in sync\n    if (\n      currentConsentRecord &&\n      checkIfPendingPreferenceUpdatesAreNoOp({\n        currentConsentRecord,\n        pendingUpdates,\n        preferenceTopics,\n      }) &&\n      !forceTriggerWorkflows\n    ) {\n      currentState.skippedUpdates[userId] = pref;\n      return;\n    }\n\n    // Determine if there are any conflicts\n    if (\n      currentConsentRecord &&\n      checkIfPendingPreferenceUpdatesCauseConflict({\n        currentConsentRecord,\n        pendingUpdates,\n        preferenceTopics,\n      })\n    ) {\n      currentState.pendingConflictUpdates[userId] = {\n        row: pref,\n        record: currentConsentRecord,\n      };\n      return;\n    }\n\n    // Add to pending updates\n    currentState.pendingSafeUpdates[userId] = pref;\n  });\n\n  // Read in the file\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n  const t1 = new Date().getTime();\n  logger.info(\n    colors.green(\n      `Successfully pre-processed file: \"${file}\" in ${(t1 - t0) / 1000}s`,\n    ),\n  );\n}\n","import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\nimport colors from 'colors';\nimport cliProgress from 'cli-progress';\nimport { chunk } from 'lodash-es';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport * as t from 'io-ts';\nimport { map } from 'bluebird';\nimport { logger } from '../../logger';\n\nconst PreferenceRecordsQueryResponse = t.intersection([\n  t.type({\n    nodes: t.array(PreferenceQueryResponseItem),\n  }),\n  t.partial({\n    /** The base64 encoded(PreferenceStorePaginationKey) cursor for pagination */\n    cursor: t.string,\n  }),\n]);\n\nconst MSGS = [\n  'ENOTFOUND',\n  'ETIMEDOUT',\n  '504 Gateway Time-out',\n  'Task timed out after',\n];\n\n/**\n * Grab the current consent preference values for a list of identifiers\n *\n * @param sombra - Backend to make API call to\n * @param options - Options\n * @returns Plaintext context information\n */\nexport async function getPreferencesForIdentifiers(\n  sombra: Got,\n  {\n    identifiers,\n    partitionKey,\n    skipLogging = false,\n  }: {\n    /** The list of identifiers to look up */\n    identifiers: {\n      /** The value of the identifier */\n      value: string;\n    }[];\n    /** The partition key to look up */\n    partitionKey: string;\n    /** Whether to skip logging */\n    skipLogging?: boolean;\n  },\n): Promise<PreferenceQueryResponseItem[]> {\n  const results: PreferenceQueryResponseItem[] = [];\n  const groupedIdentifiers = chunk(identifiers, 100);\n\n  // create a new progress bar instance and use shades_classic theme\n  const t0 = new Date().getTime();\n  const progressBar = new cliProgress.SingleBar(\n    {},\n    cliProgress.Presets.shades_classic,\n  );\n  if (!skipLogging) {\n    progressBar.start(identifiers.length, 0);\n  }\n\n  let total = 0;\n  await map(\n    groupedIdentifiers,\n    async (group) => {\n      // Make the request with retry logic\n      let attempts = 0;\n      const maxAttempts = 3;\n      while (attempts < maxAttempts) {\n        try {\n          const rawResult = await sombra\n            .post(`v1/preferences/${partitionKey}/query`, {\n              json: {\n                filter: {\n                  identifiers: group,\n                },\n                limit: group.length,\n              },\n            })\n            .json();\n\n          const result = decodeCodec(PreferenceRecordsQueryResponse, rawResult);\n          results.push(...result.nodes);\n          total += group.length;\n          progressBar.update(total);\n          break; // Exit loop if successful\n        } catch (err) {\n          attempts += 1;\n          const msg = err?.response?.body || err?.message || '';\n          if (\n            attempts >= maxAttempts ||\n            !MSGS.some((errorMessage) => msg.includes(errorMessage))\n          ) {\n            throw new Error(\n              `Received an error from server after ${attempts} attempts: ${msg}`,\n            );\n          }\n\n          logger.warn(\n            colors.yellow(\n              `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` +\n                `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`,\n            ),\n          );\n        }\n      }\n    },\n    {\n      concurrency: 40,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  if (!skipLogging) {\n    // Log completion time\n    logger.info(\n      colors.green(`Completed download in \"${totalTime / 1000}\" seconds.`),\n    );\n  }\n\n  return results;\n}\n","import { uniq, groupBy, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { inquirerConfirmBoolean } from '../helpers';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse identifiers from a CSV list of preferences\n *\n * Ensures that all rows have a valid identifier\n * and that all identifiers are unique.\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceIdentifiersFromCsv(\n  preferences: Record<string, string>[],\n  currentState: FileMetadataState,\n): Promise<{\n  /** The updated state */\n  currentState: FileMetadataState;\n  /** The updated preferences */\n  preferences: Record<string, string>[];\n}> {\n  // Determine columns to map\n  const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n  // Determine the columns that could potentially be used for identifier\n  const remainingColumnsForIdentifier = difference(columnNames, [\n    ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n    ...Object.keys(currentState.columnToPurposeName),\n  ]);\n\n  // Determine the identifier column to work off of\n  if (!currentState.identifierColumn) {\n    const { identifierName } = await inquirer.prompt<{\n      /** Identifier name */\n      identifierName: string;\n    }>([\n      {\n        name: 'identifierName',\n        message:\n          'Choose the column that will be used as the identifier to upload consent preferences by',\n        type: 'list',\n        default:\n          remainingColumnsForIdentifier.find((col) =>\n            col.toLowerCase().includes('email'),\n          ) || remainingColumnsForIdentifier[0],\n        choices: remainingColumnsForIdentifier,\n      },\n    ]);\n    currentState.identifierColumn = identifierName;\n  }\n  logger.info(\n    colors.magenta(\n      `Using identifier column \"${currentState.identifierColumn}\"`,\n    ),\n  );\n\n  // Validate that the identifier column is present for all rows and unique\n  const identifierColumnsMissing = preferences\n    .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind]))\n    .filter((x): x is number[] => !!x)\n    .flat();\n  if (identifierColumnsMissing.length > 0) {\n    const msg = `The identifier column \"${\n      currentState.identifierColumn\n    }\" is missing a value for the following rows: ${identifierColumnsMissing.join(\n      ', ',\n    )}`;\n    logger.warn(colors.yellow(msg));\n\n    // Ask user if they would like to skip rows missing an identifier\n    const skip = await inquirerConfirmBoolean({\n      message: 'Would you like to skip rows missing an identifier?',\n    });\n    if (!skip) {\n      throw new Error(msg);\n    }\n\n    // Filter out rows missing an identifier\n    const previous = preferences.length;\n    preferences = preferences.filter(\n      (pref) => pref[currentState.identifierColumn!],\n    );\n    logger.info(\n      colors.yellow(\n        `Skipped ${previous - preferences.length} rows missing an identifier`,\n      ),\n    );\n  }\n  logger.info(\n    colors.magenta(\n      `The identifier column \"${currentState.identifierColumn}\" is present for all rows`,\n    ),\n  );\n\n  // Validate that all identifiers are unique\n  const rowsByUserId = groupBy(preferences, currentState.identifierColumn);\n  const duplicateIdentifiers = Object.entries(rowsByUserId).filter(\n    ([, rows]) => rows.length > 1,\n  );\n  if (duplicateIdentifiers.length > 0) {\n    const msg = `The identifier column \"${\n      currentState.identifierColumn\n    }\" has duplicate values for the following rows: ${duplicateIdentifiers\n      .slice(0, 10)\n      .map(([userId, rows]) => `${userId} (${rows.length})`)\n      .join('\\n')}`;\n    logger.warn(colors.yellow(msg));\n\n    // Ask user if they would like to take the most recent update\n    // for each duplicate identifier\n    const skip = await inquirerConfirmBoolean({\n      message: 'Would you like to automatically take the latest update?',\n    });\n    if (!skip) {\n      throw new Error(msg);\n    }\n    preferences = Object.entries(rowsByUserId)\n      .map(([, rows]) => {\n        const sorted = rows.sort(\n          (a, b) =>\n            new Date(b[currentState.timestampColum!]).getTime() -\n            new Date(a[currentState.timestampColum!]).getTime(),\n        );\n        return sorted[0];\n      })\n      .filter((x) => x);\n  }\n\n  return { currentState, preferences };\n}\n/* eslint-enable no-param-reassign */\n","import { uniq, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { mapSeries } from 'bluebird';\nimport { PreferenceTopic } from '../graphql';\nimport { PreferenceTopicType } from '@transcend-io/privacy-types';\nimport { splitCsvToList } from '../requests';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse out the purpose.enabled and preference values from a CSV file\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @param options - Options\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceAndPurposeValuesFromCsv(\n  preferences: Record<string, string>[],\n  currentState: FileMetadataState,\n  {\n    purposeSlugs,\n    preferenceTopics,\n    forceTriggerWorkflows,\n  }: {\n    /** The purpose slugs that are allowed to be updated */\n    purposeSlugs: string[];\n    /** The preference topics */\n    preferenceTopics: PreferenceTopic[];\n    /** Force workflow triggers */\n    forceTriggerWorkflows: boolean;\n  },\n): Promise<FileMetadataState> {\n  // Determine columns to map\n  const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n  // Determine the columns that could potentially be used for identifier\n  const otherColumns = difference(columnNames, [\n    ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n    ...(currentState.timestampColum ? [currentState.timestampColum] : []),\n  ]);\n  if (otherColumns.length === 0) {\n    if (forceTriggerWorkflows) {\n      return currentState;\n    }\n    throw new Error('No other columns to process');\n  }\n\n  // The purpose and preferences to map to\n  const purposeNames = [\n    ...purposeSlugs,\n    ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`),\n  ];\n\n  // Ensure all columns are accounted for\n  await mapSeries(otherColumns, async (col) => {\n    // Determine the unique values to map in this column\n    const uniqueValues = uniq(preferences.map((x) => x[col]));\n\n    // Map the column to a purpose\n    let purposeMapping = currentState.columnToPurposeName[col];\n    if (purposeMapping) {\n      logger.info(\n        colors.magenta(\n          `Column \"${col}\" is associated with purpose \"${purposeMapping.purpose}\"`,\n        ),\n      );\n    } else {\n      const { purposeName } = await inquirer.prompt<{\n        /** purpose name */\n        purposeName: string;\n      }>([\n        {\n          name: 'purposeName',\n          message: `Choose the purpose that column ${col} is associated with`,\n          type: 'list',\n          default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])),\n          choices: purposeNames,\n        },\n      ]);\n      const [purposeSlug, preferenceSlug] = purposeName.split('->');\n      purposeMapping = {\n        purpose: purposeSlug,\n        preference: preferenceSlug || null,\n        valueMapping: {},\n      };\n    }\n\n    // map each value to the purpose value\n    await mapSeries(uniqueValues, async (value) => {\n      if (purposeMapping.valueMapping[value] !== undefined) {\n        logger.info(\n          colors.magenta(\n            `Value \"${value}\" is associated with purpose value \"${purposeMapping.valueMapping[value]}\"`,\n          ),\n        );\n        return;\n      }\n      // if preference is null, this column is just for the purpose\n      if (purposeMapping.preference === null) {\n        const { purposeValue } = await inquirer.prompt<{\n          /** purpose value */\n          purposeValue: boolean;\n        }>([\n          {\n            name: 'purposeValue',\n            message: `Choose the purpose value for value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n            type: 'confirm',\n            default: value !== 'false',\n          },\n        ]);\n        purposeMapping.valueMapping[value] = purposeValue;\n      }\n\n      // if preference is not null, this column is for a specific preference\n      if (purposeMapping.preference !== null) {\n        const preferenceTopic = preferenceTopics.find(\n          (x) => x.slug === purposeMapping.preference,\n        );\n        if (!preferenceTopic) {\n          logger.error(\n            colors.red(\n              `Preference topic \"${purposeMapping.preference}\" not found`,\n            ),\n          );\n          return;\n        }\n        const preferenceOptions = preferenceTopic.preferenceOptionValues.map(\n          ({ slug }) => slug,\n        );\n\n        if (preferenceTopic.type === PreferenceTopicType.Boolean) {\n          const { preferenceValue } = await inquirer.prompt<{\n            /** purpose value */\n            preferenceValue: boolean;\n          }>([\n            {\n              name: 'preferenceValue',\n              message:\n                // eslint-disable-next-line max-len\n                `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n              type: 'confirm',\n              default: value !== 'false',\n            },\n          ]);\n          purposeMapping.valueMapping[value] = preferenceValue;\n          return;\n        }\n\n        if (preferenceTopic.type === PreferenceTopicType.Select) {\n          const { preferenceValue } = await inquirer.prompt<{\n            /** purpose value */\n            preferenceValue: boolean;\n          }>([\n            {\n              name: 'preferenceValue',\n              // eslint-disable-next-line max-len\n              message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n              type: 'list',\n              choices: preferenceOptions,\n              default: preferenceOptions.find((x) => x === value),\n            },\n          ]);\n          purposeMapping.valueMapping[value] = preferenceValue;\n          return;\n        }\n\n        if (preferenceTopic.type === PreferenceTopicType.MultiSelect) {\n          const parsedValues = splitCsvToList(value);\n          // need to do this serially\n          await mapSeries(parsedValues, async (parsedValue) => {\n            // if we already have a value, skip re-processing it again\n            if (purposeMapping.valueMapping[parsedValue] !== undefined) {\n              return;\n            }\n            const { preferenceValue } = await inquirer.prompt<{\n              /** purpose value */\n              preferenceValue: boolean;\n            }>([\n              {\n                name: 'preferenceValue',\n                // eslint-disable-next-line max-len\n                message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${parsedValue}\" associated with purpose \"${purposeMapping.purpose}\"`,\n                type: 'list',\n                choices: preferenceOptions,\n                default: preferenceOptions.find((x) => x === parsedValue),\n              },\n            ]);\n            purposeMapping.valueMapping[parsedValue] = preferenceValue;\n          });\n          return;\n        }\n\n        throw new Error(\n          `Unknown preference topic type: ${preferenceTopic.type}`,\n        );\n      }\n    });\n\n    currentState.columnToPurposeName[col] = purposeMapping;\n  });\n\n  return currentState;\n}\n/* eslint-enable no-param-reassign */\n","import {\n  PreferenceQueryResponseItem,\n  PreferenceUpdateItem,\n} from '@transcend-io/privacy-types';\nimport * as t from 'io-ts';\n\nexport const PurposeRowMapping = t.type({\n  /**\n   * The slug or trackingType of the purpose to map to\n   *\n   * e.g. `Marketing`\n   */\n  purpose: t.string,\n  /**\n   * If the column maps to a preference instead of a purpose\n   * this is the slug of the purpose.\n   *\n   * null value indicates that this column maps to the true/false\n   * value of the purpose\n   */\n  preference: t.union([t.string, t.null]),\n  /**\n   * The mapping between each row value and purpose/preference value.\n   *\n   * e.g. for a boolean preference or purpose\n   * {\n   *  'true': true,\n   *  'false': false,\n   *   '': true,\n   * }\n   *\n   * or for a single or multi select preference\n   * {\n   *   '': true,\n   *   'value1': 'Value1',\n   *   'value2': 'Value2',\n   * }\n   */\n  valueMapping: t.record(\n    t.string,\n    t.union([t.string, t.boolean, t.null, t.undefined]),\n  ),\n});\n\n/** Override type */\nexport type PurposeRowMapping = t.TypeOf<typeof PurposeRowMapping>;\n\n/**\n * Mapping of column name to purpose row mapping.\n * This is used to map each column in the CSV to the relevant purpose and preference definitions in\n * transcend.\n */\nexport const ColumnPurposeMap = t.record(t.string, PurposeRowMapping);\n\n/** Override type */\nexport type ColumnPurposeMap = t.TypeOf<typeof ColumnPurposeMap>;\n\nexport const IdentifierMetadataForPreference = t.type({\n  /** The identifier name */\n  name: t.string,\n  /** Is unique on preference store */\n  isUniqueOnPreferenceStore: t.boolean,\n});\n\n/** Override type */\nexport type IdentifierMetadataForPreference = t.TypeOf<\n  typeof IdentifierMetadataForPreference\n>;\n\n/**\n * Mapping of identifier name to the column name in the CSV file.\n * This is used to map each identifier name to the column in the CSV file.\n */\nexport const ColumnIdentifierMap = t.record(\n  t.string,\n  IdentifierMetadataForPreference,\n);\n\n/** Override type */\nexport type ColumnIdentifierMap = t.TypeOf<typeof ColumnIdentifierMap>;\n\nexport const FileMetadataState = t.intersection([\n  t.type({\n    /**\n     * Definition of how to map each column in the CSV to\n     * the relevant purpose and preference definitions in transcend\n     */\n    columnToPurposeName: t.record(t.string, PurposeRowMapping),\n    /** Last time the file was last parsed at */\n    lastFetchedAt: t.string,\n    /**\n     * Mapping of userId to the rows in the file that need to be uploaded\n     * These uploads are overwriting non-existent preferences and are safe\n     */\n    pendingSafeUpdates: t.record(t.string, t.record(t.string, t.string)),\n    /**\n     * Mapping of userId to the rows in the file that need to be uploaded\n     * these records have conflicts with existing consent preferences\n     */\n    pendingConflictUpdates: t.record(\n      t.string,\n      t.type({\n        record: PreferenceQueryResponseItem,\n        row: t.record(t.string, t.string),\n      }),\n    ),\n    /**\n     * Mapping of userId to the rows in the file that can be skipped because\n     * their preferences are already in the store\n     */\n    skippedUpdates: t.record(t.string, t.record(t.string, t.string)),\n  }),\n  t.partial({\n    /** Determine which column name in file maps to consent record identifier to upload on  */\n    identifierColumn: t.string,\n    /** Determine which column name in file maps to the timestamp  */\n    timestampColum: t.string,\n  }),\n]);\n\n/** Override type */\nexport type FileMetadataState = t.TypeOf<typeof FileMetadataState>;\n\n/**\n * This is the type of the receipts that are stored in the file\n * that is used to track the state of the upload process.\n * It is used to resume the upload process from where it left off.\n * It is used to persist the state of the upload process across multiple runs.\n */\nexport const PreferenceUpdateMap = t.record(\n  t.string,\n  // This can either be true to indicate the record is pending\n  // or it can be an object showing the object\n  // We only return a fixed number of results to avoid\n  // making the JSON file too large\n  t.union([t.boolean, PreferenceUpdateItem]),\n);\n\n/** Override type */\nexport type PreferenceUpdateMap = t.TypeOf<typeof PreferenceUpdateMap>;\n\n/**\n * This is the type of the pending updates that are safe to run without\n * conflicts with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is safe to upload.\n */\nexport const PendingSafePreferenceUpdates = t.record(\n  t.string,\n  // This can either be true to indicate the record is safe\n  // or it can be an object showing the object\n  // We only return a fixed number of results to avoid\n  // making the JSON file too large\n  t.union([t.boolean, t.record(t.string, t.string)]),\n);\n\n/** Override type */\nexport type PendingSafePreferenceUpdates = t.TypeOf<\n  typeof PendingSafePreferenceUpdates\n>;\n\n/**\n * These are the updates that failed to be uploaded to the API.\n */\nexport const FailingPreferenceUpdates = t.record(\n  t.string,\n  t.type({\n    /** Time upload ran at */\n    uploadedAt: t.string,\n    /** Attempts to upload that resulted in an error */\n    error: t.string,\n    /** The update body */\n    update: PreferenceUpdateItem,\n  }),\n);\n\n/** Override type */\nexport type FailingPreferenceUpdates = t.TypeOf<\n  typeof FailingPreferenceUpdates\n>;\n\n/**\n * This is the type of the pending updates that are in conflict with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is pending upload.\n */\nexport const PendingWithConflictPreferenceUpdates = t.record(\n  t.string,\n  // We always return the conflicts for investigation\n  t.type({\n    /** Record to be inserted to transcend v1/preferences API */\n    record: PreferenceQueryResponseItem,\n    /** The row in the file that is pending upload */\n    row: t.record(t.string, t.string),\n  }),\n);\n\n/** Override type */\nexport type PendingWithConflictPreferenceUpdates = t.TypeOf<\n  typeof PendingWithConflictPreferenceUpdates\n>;\n\n/**\n * The set of preference updates that are skipped\n * Key is primaryKey and value is the row in the CSV\n * that is skipped.\n *\n * This is usually because the preferences are already in the store\n * or there are duplicate rows in the CSV file that are identical.\n */\nexport const SkippedPreferenceUpdates = t.record(\n  t.string,\n  t.record(t.string, t.string),\n);\n\n/** Override type */\nexport type SkippedPreferenceUpdates = t.TypeOf<\n  typeof SkippedPreferenceUpdates\n>;\n\n/** Persist this data between runs of the script */\nexport const PreferenceState = t.type({\n  /**\n   * Store a cache of previous files read in\n   */\n  fileMetadata: t.record(t.string, FileMetadataState),\n  /**\n   * The set of successful uploads to Transcend\n   * Mapping from userId to the upload metadata\n   */\n  failingUpdates: t.record(\n    t.string,\n    t.type({\n      /** Time upload ran at */\n      uploadedAt: t.string,\n      /** Attempts to upload that resulted in an error */\n      error: t.string,\n      /** The update body */\n      update: PreferenceUpdateItem,\n    }),\n  ),\n  /**\n   * The set of pending uploads to Transcend\n   * Mapping from userId to the upload metadata\n   */\n  pendingUpdates: t.record(t.string, PreferenceUpdateItem),\n});\n\n/** Override type */\nexport type PreferenceState = t.TypeOf<typeof PreferenceState>;\n"]}
+{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-TJIMDEBN.cjs","../src/commands/consent/upload-preferences/impl.ts","../src/lib/preference-management/uploadPreferenceManagementPreferencesInteractive.ts","../src/lib/preference-management/parsePreferenceManagementCsv.ts","../src/lib/preference-management/getPreferencesForIdentifiers.ts","../src/lib/preference-management/parsePreferenceIdentifiersFromCsv.ts","../src/lib/preference-management/parsePreferenceAndPurposeValuesFromCsv.ts","../src/lib/preference-management/codecs.ts"],"names":["PreferenceRecordsQueryResponse","PreferenceQueryResponseItem","MSGS","getPreferencesForIdentifiers","sombra","identifiers","partitionKey","skipLogging","results","groupedIdentifiers","chunk_default","t0","progressBar","cliProgress","total","map","group","attempts","maxAttempts","rawResult","result","decodeCodec","err","msg","errorMessage"],"mappings":"AAAA,u/BAAyC,wDAAwG,gCAA6B,wDAAyC,wDAAuE,gCAA6B,gFCCxS,oCCQC,qGAGI,+DAEO,qJCXZ,2DCHyB,qDAKhB,IAKtBA,EAAAA,CAAmC,CAAA,CAAA,YAAA,CAAa,CAClD,CAAA,CAAA,IAAA,CAAK,CACL,KAAA,CAAS,CAAA,CAAA,KAAA,CAAMC,yCAA2B,CAC5C,CAAC,CAAA,CACC,CAAA,CAAA,OAAA,CAAQ,CAER,MAAA,CAAU,CAAA,CAAA,MACZ,CAAC,CACH,CAAC,CAAA,CAEKC,EAAAA,CAAO,CACX,WAAA,CACA,WAAA,CACA,sBAAA,CACA,sBACF,CAAA,CASA,MAAA,SAAsBC,EAAAA,CACpBC,CAAAA,CACA,CACE,WAAA,CAAAC,CAAAA,CACA,YAAA,CAAAC,CAAAA,CACA,WAAA,CAAAC,CAAAA,CAAc,CAAA,CAChB,CAAA,CAWwC,CACxC,IAAMC,CAAAA,CAAyC,CAAC,CAAA,CAC1CC,CAAAA,CAAqBC,iCAAAA,CAAML,CAAa,GAAG,CAAA,CAG3CM,CAAAA,CAAK,IAAI,IAAA,CAAK,CAAA,CAAE,OAAA,CAAQ,CAAA,CACxBC,CAAAA,CAAc,IAAIC,qBAAAA,CAAY,SAAA,CAClC,CAAC,CAAA,CACDA,qBAAAA,CAAY,OAAA,CAAQ,cACtB,CAAA,CACKN,CAAAA,EACHK,CAAAA,CAAY,KAAA,CAAMP,CAAAA,CAAY,MAAA,CAAQ,CAAC,CAAA,CAGzC,IAAIS,CAAAA,CAAQ,CAAA,CACZ,MAAMC,2BAAAA,CACJN,CACA,MAAOO,CAAAA,EAAU,CAEf,IAAIC,CAAAA,CAAW,CAAA,CACTC,CAAAA,CAAc,CAAA,CACpB,GAAA,CAAA,CAAOD,CAAAA,CAAWC,CAAAA,CAAAA,CAChB,GAAI,CACF,IAAMC,CAAAA,CAAY,MAAMf,CAAAA,CACrB,IAAA,CAAK,CAAA,eAAA,EAAkBE,CAAY,CAAA,MAAA,CAAA,CAAU,CAC5C,IAAA,CAAM,CACJ,MAAA,CAAQ,CACN,WAAA,CAAaU,CACf,CAAA,CACA,KAAA,CAAOA,CAAAA,CAAM,MACf,CACF,CAAC,CAAA,CACA,IAAA,CAAK,CAAA,CAEFI,CAAAA,CAASC,oCAAAA,EAAYrB,CAAgCmB,CAAS,CAAA,CACpEX,CAAAA,CAAQ,IAAA,CAAK,GAAGY,CAAAA,CAAO,KAAK,CAAA,CAC5BN,CAAAA,EAASE,CAAAA,CAAM,MAAA,CACfJ,CAAAA,CAAY,MAAA,CAAOE,CAAK,CAAA,CACxB,KACF,CAAA,KAAA,CAASQ,CAAAA,CAAK,CACZL,CAAAA,EAAY,CAAA,CACZ,IAAMM,CAAAA,iBAAMD,CAAAA,6BAAK,QAAA,6BAAU,MAAA,kBAAQA,CAAAA,6BAAK,SAAA,EAAW,EAAA,CACnD,EAAA,CACEL,CAAAA,EAAYC,CAAAA,EACZ,CAAChB,EAAAA,CAAK,IAAA,CAAMsB,CAAAA,EAAiBD,CAAAA,CAAI,QAAA,CAASC,CAAY,CAAC,CAAA,CAEvD,MAAM,IAAI,KAAA,CACR,CAAA,oCAAA,EAAuCP,CAAQ,CAAA,WAAA,EAAcM,CAAG,CAAA,CAAA;ACclE;ACHS;ACkFjB,oGAAA;ALrGI;AAGA;AAAA;AAKQ;AAAK;AACgC,sCAAA;AAmIpB","file":"/home/runner/work/cli/cli/dist/impl-TJIMDEBN.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport colors from 'colors';\n\nimport { logger } from '../../../logger';\nimport { uploadPreferenceManagementPreferencesInteractive } from '../../../lib/preference-management';\nimport { splitCsvToList } from '../../../lib/requests';\nimport { readdirSync } from 'node:fs';\nimport { map } from 'bluebird';\nimport { basename, join } from 'node:path';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface UploadPreferencesCommandFlags {\n  auth: string;\n  partition: string;\n  sombraAuth?: string;\n  transcendUrl: string;\n  file?: string;\n  directory?: string;\n  dryRun: boolean;\n  skipExistingRecordCheck: boolean;\n  receiptFileDir: string;\n  skipWorkflowTriggers: boolean;\n  forceTriggerWorkflows: boolean;\n  skipConflictUpdates: boolean;\n  isSilent: boolean;\n  attributes: string;\n  receiptFilepath: string;\n  concurrency: number;\n}\n\nexport async function uploadPreferences(\n  this: LocalContext,\n  {\n    auth,\n    partition,\n    sombraAuth,\n    transcendUrl,\n    file = '',\n    directory,\n    dryRun,\n    skipExistingRecordCheck,\n    receiptFileDir,\n    skipWorkflowTriggers,\n    forceTriggerWorkflows,\n    skipConflictUpdates,\n    isSilent,\n    attributes,\n    concurrency,\n  }: UploadPreferencesCommandFlags,\n): Promise<void> {\n  if (!!directory && !!file) {\n    logger.error(\n      colors.red(\n        'Cannot provide both a directory and a file. Please provide only one.',\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  if (!file && !directory) {\n    logger.error(\n      colors.red(\n        'A file or directory must be provided. Please provide one using --file=./preferences.csv or --directory=./preferences',\n      ),\n    );\n    this.process.exit(1);\n  }\n\n  doneInputValidation(this.process.exit);\n\n  const files: string[] = [];\n\n  if (directory) {\n    try {\n      const filesInDirectory = readdirSync(directory);\n      const csvFiles = filesInDirectory.filter((file) => file.endsWith('.csv'));\n\n      if (csvFiles.length === 0) {\n        logger.error(\n          colors.red(`No CSV files found in directory: ${directory}`),\n        );\n        this.process.exit(1);\n      }\n\n      // Add full paths for each CSV file\n      files.push(...csvFiles.map((file) => join(directory, file)));\n    } catch (err) {\n      logger.error(colors.red(`Failed to read directory: ${directory}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  } else {\n    try {\n      // Verify file exists and is a CSV\n      if (!file.endsWith('.csv')) {\n        logger.error(colors.red('File must be a CSV file'));\n        this.process.exit(1);\n      }\n      files.push(file);\n    } catch (err) {\n      logger.error(colors.red(`Failed to access file: ${file}`));\n      logger.error(colors.red((err as Error).message));\n      this.process.exit(1);\n    }\n  }\n\n  logger.info(\n    colors.green(\n      `Processing ${files.length} consent preferences files for partition: ${partition}`,\n    ),\n  );\n  logger.debug(`Files to process: ${files.join(', ')}`);\n\n  if (skipExistingRecordCheck) {\n    logger.info(\n      colors.bgYellow(\n        `Skipping existing record check: ${skipExistingRecordCheck}`,\n      ),\n    );\n  }\n\n  await map(\n    files,\n    async (filePath) => {\n      const fileName = basename(filePath).replace('.csv', '');\n      await uploadPreferenceManagementPreferencesInteractive({\n        receiptFilepath: join(receiptFileDir, `${fileName}-receipts.json`),\n        auth,\n        sombraAuth,\n        file: filePath,\n        partition,\n        transcendUrl,\n        skipConflictUpdates,\n        skipWorkflowTriggers,\n        skipExistingRecordCheck,\n        isSilent,\n        dryRun,\n        attributes: splitCsvToList(attributes),\n        forceTriggerWorkflows,\n      });\n    },\n    { concurrency },\n  );\n}\n","import {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllPurposes,\n  fetchAllPreferenceTopics,\n  PreferenceTopic,\n  Purpose,\n} from '../graphql';\nimport colors from 'colors';\nimport { map } from 'bluebird';\nimport { chunk } from 'lodash-es';\nimport { logger } from '../../logger';\nimport cliProgress from 'cli-progress';\nimport { parseAttributesFromString } from '../requests';\nimport { PersistedState } from '@transcend-io/persisted-state';\nimport { parsePreferenceManagementCsvWithCache } from './parsePreferenceManagementCsv';\nimport { PreferenceState } from './codecs';\nimport { PreferenceUpdateItem } from '@transcend-io/privacy-types';\nimport { apply } from '@transcend-io/type-utils';\nimport { NONE_PREFERENCE_MAP } from './parsePreferenceTimestampsFromCsv';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\n\n/**\n * Upload a set of consent preferences\n *\n * @param options - Options\n */\nexport async function uploadPreferenceManagementPreferencesInteractive({\n  auth,\n  sombraAuth,\n  receiptFilepath,\n  file,\n  partition,\n  isSilent = true,\n  dryRun = false,\n  skipWorkflowTriggers = false,\n  skipConflictUpdates = false,\n  skipExistingRecordCheck = false,\n  attributes = [],\n  transcendUrl,\n  forceTriggerWorkflows = false,\n}: {\n  /** The Transcend API key */\n  auth: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Partition key */\n  partition: string;\n  /** File where to store receipt and continue from where left off */\n  receiptFilepath: string;\n  /** The file to process */\n  file: string;\n  /** API URL for Transcend backend */\n  transcendUrl: string;\n  /** Whether to do a dry run */\n  dryRun?: boolean;\n  /** Whether to upload as isSilent */\n  isSilent?: boolean;\n  /** Attributes string pre-parse. In format Key:Value */\n  attributes?: string[];\n  /** Skip workflow triggers */\n  skipWorkflowTriggers?: boolean;\n  /**\n   * When true, only update preferences that do not conflict with existing\n   * preferences. When false, update all preferences in CSV based on timestamp.\n   */\n  skipConflictUpdates?: boolean;\n  /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n  skipExistingRecordCheck?: boolean;\n  /** Whether to force trigger workflows */\n  forceTriggerWorkflows?: boolean;\n}): Promise<void> {\n  // Parse out the extra attributes to apply to all requests uploaded\n  const parsedAttributes = parseAttributesFromString(attributes);\n\n  // Create a new state file to store the requests from this run\n  const preferenceState = new PersistedState(receiptFilepath, PreferenceState, {\n    fileMetadata: {},\n    failingUpdates: {},\n    pendingUpdates: {},\n  });\n  const failingRequests = preferenceState.getValue('failingUpdates');\n  const pendingRequests = preferenceState.getValue('pendingUpdates');\n  let fileMetadata = preferenceState.getValue('fileMetadata');\n\n  logger.info(\n    colors.magenta(\n      'Restored cache, there are: \\n' +\n        `${\n          Object.values(failingRequests).length\n        } failing requests to be retried\\n` +\n        `${\n          Object.values(pendingRequests).length\n        } pending requests to be processed\\n` +\n        `The following files are stored in cache and will be used:\\n${Object.keys(\n          fileMetadata,\n        )\n          .map((x) => x)\n          .join('\\n')}\\n` +\n        `The following file will be processed: ${file}\\n`,\n    ),\n  );\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  const [sombra, purposes, preferenceTopics] = await Promise.all([\n    // Create sombra instance to communicate with\n    createSombraGotInstance(transcendUrl, auth, sombraAuth),\n    // get all purposes and topics\n    forceTriggerWorkflows\n      ? Promise.resolve([] as Purpose[])\n      : fetchAllPurposes(client),\n    forceTriggerWorkflows\n      ? Promise.resolve([] as PreferenceTopic[])\n      : fetchAllPreferenceTopics(client),\n  ]);\n\n  // Process the file\n  await parsePreferenceManagementCsvWithCache(\n    {\n      file,\n      purposeSlugs: purposes.map((x) => x.trackingType),\n      preferenceTopics,\n      sombra,\n      partitionKey: partition,\n      skipExistingRecordCheck,\n      forceTriggerWorkflows,\n    },\n    preferenceState,\n  );\n\n  // Construct the pending updates\n  const pendingUpdates: Record<string, PreferenceUpdateItem> = {};\n  fileMetadata = preferenceState.getValue('fileMetadata');\n  const metadata = fileMetadata[file];\n\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.pendingSafeUpdates).length\n      } safe updates in ${file}`,\n    ),\n  );\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.pendingConflictUpdates).length\n      } conflict updates in ${file}`,\n    ),\n  );\n  logger.info(\n    colors.magenta(\n      `Found ${\n        Object.entries(metadata.skippedUpdates).length\n      } skipped updates in ${file}`,\n    ),\n  );\n\n  // Update either safe updates only or safe + conflict\n  Object.entries({\n    ...metadata.pendingSafeUpdates,\n    ...(skipConflictUpdates\n      ? {}\n      : apply(metadata.pendingConflictUpdates, ({ row }) => row)),\n  }).forEach(([userId, update]) => {\n    // Determine timestamp\n    const timestamp =\n      metadata.timestampColum === NONE_PREFERENCE_MAP\n        ? new Date()\n        : new Date(update[metadata.timestampColum!]);\n\n    // Determine updates\n    const updates = getPreferenceUpdatesFromRow({\n      row: update,\n      columnToPurposeName: metadata.columnToPurposeName,\n      preferenceTopics,\n      purposeSlugs: purposes.map((x) => x.trackingType),\n    });\n    pendingUpdates[userId] = {\n      userId,\n      partition,\n      timestamp: timestamp.toISOString(),\n      purposes: Object.entries(updates).map(([purpose, value]) => ({\n        ...value,\n        purpose,\n        workflowSettings: {\n          attributes: parsedAttributes,\n          isSilent,\n          skipWorkflowTrigger: skipWorkflowTriggers,\n        },\n      })),\n    };\n  });\n  await preferenceState.setValue(pendingUpdates, 'pendingUpdates');\n  await preferenceState.setValue({}, 'failingUpdates');\n\n  // Exist early if dry run\n  if (dryRun) {\n    logger.info(\n      colors.green(\n        `Dry run complete, exiting. ${\n          Object.values(pendingUpdates).length\n        } pending updates. Check file: ${receiptFilepath}`,\n      ),\n    );\n    return;\n  }\n\n  logger.info(\n    colors.magenta(\n      `Uploading ${\n        Object.values(pendingUpdates).length\n      } preferences to partition: ${partition}`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar(\n    {},\n    cliProgress.Presets.shades_classic,\n  );\n\n  // Build a GraphQL client\n  let total = 0;\n  const updatesToRun = Object.entries(pendingUpdates);\n  const chunkedUpdates = chunk(updatesToRun, skipWorkflowTriggers ? 100 : 10);\n  progressBar.start(updatesToRun.length, 0);\n  await map(\n    chunkedUpdates,\n    async (currentChunk) => {\n      // Make the request\n      try {\n        await sombra\n          .put('v1/preferences', {\n            json: {\n              records: currentChunk.map(([, update]) => update),\n              skipWorkflowTriggers,\n              forceTriggerWorkflows,\n            },\n          })\n          .json();\n      } catch (err) {\n        try {\n          const parsed = JSON.parse(err?.response?.body || '{}');\n          if (parsed.error) {\n            logger.error(colors.red(`Error: ${parsed.error}`));\n          }\n        } catch (e) {\n          // continue\n        }\n        logger.error(\n          colors.red(\n            `Failed to upload ${\n              currentChunk.length\n            } user preferences to partition ${partition}: ${\n              err?.response?.body || err?.message\n            }`,\n          ),\n        );\n        const failingUpdates = preferenceState.getValue('failingUpdates');\n        currentChunk.forEach(([userId, update]) => {\n          failingUpdates[userId] = {\n            uploadedAt: new Date().toISOString(),\n            update,\n            error: err?.response?.body || err?.message || 'Unknown error',\n          };\n        });\n        await preferenceState.setValue(failingUpdates, 'failingUpdates');\n      }\n\n      total += currentChunk.length;\n      progressBar.update(total);\n    },\n    {\n      concurrency: 40,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n  logger.info(\n    colors.green(\n      `Successfully uploaded ${\n        updatesToRun.length\n      } user preferences to partition ${partition} in \"${\n        totalTime / 1000\n      }\" seconds!`,\n    ),\n  );\n}\n","import { PersistedState } from '@transcend-io/persisted-state';\nimport type { Got } from 'got';\nimport { keyBy } from 'lodash-es';\nimport * as t from 'io-ts';\nimport colors from 'colors';\nimport { FileMetadataState, PreferenceState } from './codecs';\nimport { logger } from '../../logger';\nimport { readCsv } from '../requests';\nimport { getPreferencesForIdentifiers } from './getPreferencesForIdentifiers';\nimport { PreferenceTopic } from '../graphql';\nimport { getPreferenceUpdatesFromRow } from './getPreferenceUpdatesFromRow';\nimport { parsePreferenceTimestampsFromCsv } from './parsePreferenceTimestampsFromCsv';\nimport { parsePreferenceIdentifiersFromCsv } from './parsePreferenceIdentifiersFromCsv';\nimport { parsePreferenceAndPurposeValuesFromCsv } from './parsePreferenceAndPurposeValuesFromCsv';\nimport { checkIfPendingPreferenceUpdatesAreNoOp } from './checkIfPendingPreferenceUpdatesAreNoOp';\nimport { checkIfPendingPreferenceUpdatesCauseConflict } from './checkIfPendingPreferenceUpdatesCauseConflict';\n\n/**\n * Parse a file into the cache\n *\n *\n * @param options - Options\n * @param cache - The cache to store the parsed file in\n * @returns The cache with the parsed file\n */\nexport async function parsePreferenceManagementCsvWithCache(\n  {\n    file,\n    sombra,\n    purposeSlugs,\n    preferenceTopics,\n    partitionKey,\n    skipExistingRecordCheck,\n    forceTriggerWorkflows,\n  }: {\n    /** File to parse */\n    file: string;\n    /** The purpose slugs that are allowed to be updated */\n    purposeSlugs: string[];\n    /** The preference topics */\n    preferenceTopics: PreferenceTopic[];\n    /** Sombra got instance */\n    sombra: Got;\n    /** Partition key */\n    partitionKey: string;\n    /** Whether to skip the check for existing records. SHOULD ONLY BE USED FOR INITIAL UPLOAD */\n    skipExistingRecordCheck: boolean;\n    /** Wheather to force workflow triggers */\n    forceTriggerWorkflows: boolean;\n  },\n  cache: PersistedState<typeof PreferenceState>,\n): Promise<void> {\n  // Start the timer\n  const t0 = new Date().getTime();\n\n  // Get the current metadata\n  const fileMetadata = cache.getValue('fileMetadata');\n\n  // Read in the file\n  logger.info(colors.magenta(`Reading in file: \"${file}\"`));\n  let preferences = readCsv(file, t.record(t.string, t.string));\n\n  // start building the cache, can use previous cache as well\n  let currentState: FileMetadataState = {\n    columnToPurposeName: {},\n    pendingSafeUpdates: {},\n    pendingConflictUpdates: {},\n    skippedUpdates: {},\n    // Load in the last fetched time\n    ...((fileMetadata[file] || {}) as Partial<FileMetadataState>),\n    lastFetchedAt: new Date().toISOString(),\n  };\n\n  // Validate that all timestamps are present in the file\n  currentState = await parsePreferenceTimestampsFromCsv(\n    preferences,\n    currentState,\n  );\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Validate that all identifiers are present and unique\n  const result = await parsePreferenceIdentifiersFromCsv(\n    preferences,\n    currentState,\n  );\n  currentState = result.currentState;\n  preferences = result.preferences;\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Ensure all other columns are mapped to purpose and preference\n  // slug values\n  currentState = await parsePreferenceAndPurposeValuesFromCsv(\n    preferences,\n    currentState,\n    {\n      preferenceTopics,\n      purposeSlugs,\n      forceTriggerWorkflows,\n    },\n  );\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n\n  // Grab existing preference store records\n  const identifiers = preferences.map(\n    (pref) => pref[currentState.identifierColumn!],\n  );\n  const existingConsentRecords = skipExistingRecordCheck\n    ? []\n    : await getPreferencesForIdentifiers(sombra, {\n        identifiers: identifiers.map((x) => ({ value: x })),\n        partitionKey,\n      });\n  const consentRecordByIdentifier = keyBy(existingConsentRecords, 'userId');\n\n  // Clear out previous updates\n  currentState.pendingConflictUpdates = {};\n  currentState.pendingSafeUpdates = {};\n  currentState.skippedUpdates = {};\n\n  // Process each row\n  preferences.forEach((pref) => {\n    // Grab unique Id for the user\n    const userId = pref[currentState.identifierColumn!];\n\n    // determine updates for user\n    const pendingUpdates = getPreferenceUpdatesFromRow({\n      row: pref,\n      columnToPurposeName: currentState.columnToPurposeName,\n      preferenceTopics,\n      purposeSlugs,\n    });\n\n    // Grab current state of the update\n    const currentConsentRecord = consentRecordByIdentifier[userId];\n    if (forceTriggerWorkflows && !currentConsentRecord) {\n      throw new Error(\n        `No existing consent record found for user with id: ${userId}. \n        When 'forceTriggerWorkflows' is set all the user identifiers should contain a consent record`,\n      );\n    }\n    // Check if the update can be skipped\n    // this is the case if a record exists, and the purpose\n    // and preference values are all in sync\n    if (\n      currentConsentRecord &&\n      checkIfPendingPreferenceUpdatesAreNoOp({\n        currentConsentRecord,\n        pendingUpdates,\n        preferenceTopics,\n      }) &&\n      !forceTriggerWorkflows\n    ) {\n      currentState.skippedUpdates[userId] = pref;\n      return;\n    }\n\n    // Determine if there are any conflicts\n    if (\n      currentConsentRecord &&\n      checkIfPendingPreferenceUpdatesCauseConflict({\n        currentConsentRecord,\n        pendingUpdates,\n        preferenceTopics,\n      })\n    ) {\n      currentState.pendingConflictUpdates[userId] = {\n        row: pref,\n        record: currentConsentRecord,\n      };\n      return;\n    }\n\n    // Add to pending updates\n    currentState.pendingSafeUpdates[userId] = pref;\n  });\n\n  // Read in the file\n  fileMetadata[file] = currentState;\n  await cache.setValue(fileMetadata, 'fileMetadata');\n  const t1 = new Date().getTime();\n  logger.info(\n    colors.green(\n      `Successfully pre-processed file: \"${file}\" in ${(t1 - t0) / 1000}s`,\n    ),\n  );\n}\n","import { PreferenceQueryResponseItem } from '@transcend-io/privacy-types';\nimport type { Got } from 'got';\nimport colors from 'colors';\nimport cliProgress from 'cli-progress';\nimport { chunk } from 'lodash-es';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport * as t from 'io-ts';\nimport { map } from 'bluebird';\nimport { logger } from '../../logger';\n\nconst PreferenceRecordsQueryResponse = t.intersection([\n  t.type({\n    nodes: t.array(PreferenceQueryResponseItem),\n  }),\n  t.partial({\n    /** The base64 encoded(PreferenceStorePaginationKey) cursor for pagination */\n    cursor: t.string,\n  }),\n]);\n\nconst MSGS = [\n  'ENOTFOUND',\n  'ETIMEDOUT',\n  '504 Gateway Time-out',\n  'Task timed out after',\n];\n\n/**\n * Grab the current consent preference values for a list of identifiers\n *\n * @param sombra - Backend to make API call to\n * @param options - Options\n * @returns Plaintext context information\n */\nexport async function getPreferencesForIdentifiers(\n  sombra: Got,\n  {\n    identifiers,\n    partitionKey,\n    skipLogging = false,\n  }: {\n    /** The list of identifiers to look up */\n    identifiers: {\n      /** The value of the identifier */\n      value: string;\n    }[];\n    /** The partition key to look up */\n    partitionKey: string;\n    /** Whether to skip logging */\n    skipLogging?: boolean;\n  },\n): Promise<PreferenceQueryResponseItem[]> {\n  const results: PreferenceQueryResponseItem[] = [];\n  const groupedIdentifiers = chunk(identifiers, 100);\n\n  // create a new progress bar instance and use shades_classic theme\n  const t0 = new Date().getTime();\n  const progressBar = new cliProgress.SingleBar(\n    {},\n    cliProgress.Presets.shades_classic,\n  );\n  if (!skipLogging) {\n    progressBar.start(identifiers.length, 0);\n  }\n\n  let total = 0;\n  await map(\n    groupedIdentifiers,\n    async (group) => {\n      // Make the request with retry logic\n      let attempts = 0;\n      const maxAttempts = 3;\n      while (attempts < maxAttempts) {\n        try {\n          const rawResult = await sombra\n            .post(`v1/preferences/${partitionKey}/query`, {\n              json: {\n                filter: {\n                  identifiers: group,\n                },\n                limit: group.length,\n              },\n            })\n            .json();\n\n          const result = decodeCodec(PreferenceRecordsQueryResponse, rawResult);\n          results.push(...result.nodes);\n          total += group.length;\n          progressBar.update(total);\n          break; // Exit loop if successful\n        } catch (err) {\n          attempts += 1;\n          const msg = err?.response?.body || err?.message || '';\n          if (\n            attempts >= maxAttempts ||\n            !MSGS.some((errorMessage) => msg.includes(errorMessage))\n          ) {\n            throw new Error(\n              `Received an error from server after ${attempts} attempts: ${msg}`,\n            );\n          }\n\n          logger.warn(\n            colors.yellow(\n              `[RETRYING FAILED REQUEST - Attempt ${attempts}] ` +\n                `Failed to fetch ${group.length} user preferences from partition ${partitionKey}: ${msg}`,\n            ),\n          );\n        }\n      }\n    },\n    {\n      concurrency: 40,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  if (!skipLogging) {\n    // Log completion time\n    logger.info(\n      colors.green(`Completed download in \"${totalTime / 1000}\" seconds.`),\n    );\n  }\n\n  return results;\n}\n","import { uniq, groupBy, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { inquirerConfirmBoolean } from '../helpers';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse identifiers from a CSV list of preferences\n *\n * Ensures that all rows have a valid identifier\n * and that all identifiers are unique.\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceIdentifiersFromCsv(\n  preferences: Record<string, string>[],\n  currentState: FileMetadataState,\n): Promise<{\n  /** The updated state */\n  currentState: FileMetadataState;\n  /** The updated preferences */\n  preferences: Record<string, string>[];\n}> {\n  // Determine columns to map\n  const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n  // Determine the columns that could potentially be used for identifier\n  const remainingColumnsForIdentifier = difference(columnNames, [\n    ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n    ...Object.keys(currentState.columnToPurposeName),\n  ]);\n\n  // Determine the identifier column to work off of\n  if (!currentState.identifierColumn) {\n    const { identifierName } = await inquirer.prompt<{\n      /** Identifier name */\n      identifierName: string;\n    }>([\n      {\n        name: 'identifierName',\n        message:\n          'Choose the column that will be used as the identifier to upload consent preferences by',\n        type: 'list',\n        default:\n          remainingColumnsForIdentifier.find((col) =>\n            col.toLowerCase().includes('email'),\n          ) || remainingColumnsForIdentifier[0],\n        choices: remainingColumnsForIdentifier,\n      },\n    ]);\n    currentState.identifierColumn = identifierName;\n  }\n  logger.info(\n    colors.magenta(\n      `Using identifier column \"${currentState.identifierColumn}\"`,\n    ),\n  );\n\n  // Validate that the identifier column is present for all rows and unique\n  const identifierColumnsMissing = preferences\n    .map((pref, ind) => (pref[currentState.identifierColumn!] ? null : [ind]))\n    .filter((x): x is number[] => !!x)\n    .flat();\n  if (identifierColumnsMissing.length > 0) {\n    const msg = `The identifier column \"${\n      currentState.identifierColumn\n    }\" is missing a value for the following rows: ${identifierColumnsMissing.join(\n      ', ',\n    )}`;\n    logger.warn(colors.yellow(msg));\n\n    // Ask user if they would like to skip rows missing an identifier\n    const skip = await inquirerConfirmBoolean({\n      message: 'Would you like to skip rows missing an identifier?',\n    });\n    if (!skip) {\n      throw new Error(msg);\n    }\n\n    // Filter out rows missing an identifier\n    const previous = preferences.length;\n    preferences = preferences.filter(\n      (pref) => pref[currentState.identifierColumn!],\n    );\n    logger.info(\n      colors.yellow(\n        `Skipped ${previous - preferences.length} rows missing an identifier`,\n      ),\n    );\n  }\n  logger.info(\n    colors.magenta(\n      `The identifier column \"${currentState.identifierColumn}\" is present for all rows`,\n    ),\n  );\n\n  // Validate that all identifiers are unique\n  const rowsByUserId = groupBy(preferences, currentState.identifierColumn);\n  const duplicateIdentifiers = Object.entries(rowsByUserId).filter(\n    ([, rows]) => rows.length > 1,\n  );\n  if (duplicateIdentifiers.length > 0) {\n    const msg = `The identifier column \"${\n      currentState.identifierColumn\n    }\" has duplicate values for the following rows: ${duplicateIdentifiers\n      .slice(0, 10)\n      .map(([userId, rows]) => `${userId} (${rows.length})`)\n      .join('\\n')}`;\n    logger.warn(colors.yellow(msg));\n\n    // Ask user if they would like to take the most recent update\n    // for each duplicate identifier\n    const skip = await inquirerConfirmBoolean({\n      message: 'Would you like to automatically take the latest update?',\n    });\n    if (!skip) {\n      throw new Error(msg);\n    }\n    preferences = Object.entries(rowsByUserId)\n      .map(([, rows]) => {\n        const sorted = rows.sort(\n          (a, b) =>\n            new Date(b[currentState.timestampColum!]).getTime() -\n            new Date(a[currentState.timestampColum!]).getTime(),\n        );\n        return sorted[0];\n      })\n      .filter((x) => x);\n  }\n\n  return { currentState, preferences };\n}\n/* eslint-enable no-param-reassign */\n","import { uniq, difference } from 'lodash-es';\nimport colors from 'colors';\nimport inquirer from 'inquirer';\nimport { FileMetadataState } from './codecs';\nimport { logger } from '../../logger';\nimport { mapSeries } from 'bluebird';\nimport { PreferenceTopic } from '../graphql';\nimport { PreferenceTopicType } from '@transcend-io/privacy-types';\nimport { splitCsvToList } from '../requests';\n\n/* eslint-disable no-param-reassign */\n\n/**\n * Parse out the purpose.enabled and preference values from a CSV file\n *\n * @param preferences - List of preferences\n * @param currentState - The current file metadata state for parsing this list\n * @param options - Options\n * @returns The updated file metadata state\n */\nexport async function parsePreferenceAndPurposeValuesFromCsv(\n  preferences: Record<string, string>[],\n  currentState: FileMetadataState,\n  {\n    purposeSlugs,\n    preferenceTopics,\n    forceTriggerWorkflows,\n  }: {\n    /** The purpose slugs that are allowed to be updated */\n    purposeSlugs: string[];\n    /** The preference topics */\n    preferenceTopics: PreferenceTopic[];\n    /** Force workflow triggers */\n    forceTriggerWorkflows: boolean;\n  },\n): Promise<FileMetadataState> {\n  // Determine columns to map\n  const columnNames = uniq(preferences.map((x) => Object.keys(x)).flat());\n\n  // Determine the columns that could potentially be used for identifier\n  const otherColumns = difference(columnNames, [\n    ...(currentState.identifierColumn ? [currentState.identifierColumn] : []),\n    ...(currentState.timestampColum ? [currentState.timestampColum] : []),\n  ]);\n  if (otherColumns.length === 0) {\n    if (forceTriggerWorkflows) {\n      return currentState;\n    }\n    throw new Error('No other columns to process');\n  }\n\n  // The purpose and preferences to map to\n  const purposeNames = [\n    ...purposeSlugs,\n    ...preferenceTopics.map((x) => `${x.purpose.trackingType}->${x.slug}`),\n  ];\n\n  // Ensure all columns are accounted for\n  await mapSeries(otherColumns, async (col) => {\n    // Determine the unique values to map in this column\n    const uniqueValues = uniq(preferences.map((x) => x[col]));\n\n    // Map the column to a purpose\n    let purposeMapping = currentState.columnToPurposeName[col];\n    if (purposeMapping) {\n      logger.info(\n        colors.magenta(\n          `Column \"${col}\" is associated with purpose \"${purposeMapping.purpose}\"`,\n        ),\n      );\n    } else {\n      const { purposeName } = await inquirer.prompt<{\n        /** purpose name */\n        purposeName: string;\n      }>([\n        {\n          name: 'purposeName',\n          message: `Choose the purpose that column ${col} is associated with`,\n          type: 'list',\n          default: purposeNames.find((x) => x.startsWith(purposeSlugs[0])),\n          choices: purposeNames,\n        },\n      ]);\n      const [purposeSlug, preferenceSlug] = purposeName.split('->');\n      purposeMapping = {\n        purpose: purposeSlug,\n        preference: preferenceSlug || null,\n        valueMapping: {},\n      };\n    }\n\n    // map each value to the purpose value\n    await mapSeries(uniqueValues, async (value) => {\n      if (purposeMapping.valueMapping[value] !== undefined) {\n        logger.info(\n          colors.magenta(\n            `Value \"${value}\" is associated with purpose value \"${purposeMapping.valueMapping[value]}\"`,\n          ),\n        );\n        return;\n      }\n      // if preference is null, this column is just for the purpose\n      if (purposeMapping.preference === null) {\n        const { purposeValue } = await inquirer.prompt<{\n          /** purpose value */\n          purposeValue: boolean;\n        }>([\n          {\n            name: 'purposeValue',\n            message: `Choose the purpose value for value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n            type: 'confirm',\n            default: value !== 'false',\n          },\n        ]);\n        purposeMapping.valueMapping[value] = purposeValue;\n      }\n\n      // if preference is not null, this column is for a specific preference\n      if (purposeMapping.preference !== null) {\n        const preferenceTopic = preferenceTopics.find(\n          (x) => x.slug === purposeMapping.preference,\n        );\n        if (!preferenceTopic) {\n          logger.error(\n            colors.red(\n              `Preference topic \"${purposeMapping.preference}\" not found`,\n            ),\n          );\n          return;\n        }\n        const preferenceOptions = preferenceTopic.preferenceOptionValues.map(\n          ({ slug }) => slug,\n        );\n\n        if (preferenceTopic.type === PreferenceTopicType.Boolean) {\n          const { preferenceValue } = await inquirer.prompt<{\n            /** purpose value */\n            preferenceValue: boolean;\n          }>([\n            {\n              name: 'preferenceValue',\n              message:\n                // eslint-disable-next-line max-len\n                `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n              type: 'confirm',\n              default: value !== 'false',\n            },\n          ]);\n          purposeMapping.valueMapping[value] = preferenceValue;\n          return;\n        }\n\n        if (preferenceTopic.type === PreferenceTopicType.Select) {\n          const { preferenceValue } = await inquirer.prompt<{\n            /** purpose value */\n            preferenceValue: boolean;\n          }>([\n            {\n              name: 'preferenceValue',\n              // eslint-disable-next-line max-len\n              message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${value}\" associated with purpose \"${purposeMapping.purpose}\"`,\n              type: 'list',\n              choices: preferenceOptions,\n              default: preferenceOptions.find((x) => x === value),\n            },\n          ]);\n          purposeMapping.valueMapping[value] = preferenceValue;\n          return;\n        }\n\n        if (preferenceTopic.type === PreferenceTopicType.MultiSelect) {\n          const parsedValues = splitCsvToList(value);\n          // need to do this serially\n          await mapSeries(parsedValues, async (parsedValue) => {\n            // if we already have a value, skip re-processing it again\n            if (purposeMapping.valueMapping[parsedValue] !== undefined) {\n              return;\n            }\n            const { preferenceValue } = await inquirer.prompt<{\n              /** purpose value */\n              preferenceValue: boolean;\n            }>([\n              {\n                name: 'preferenceValue',\n                // eslint-disable-next-line max-len\n                message: `Choose the preference value for \"${preferenceTopic.slug}\" value \"${parsedValue}\" associated with purpose \"${purposeMapping.purpose}\"`,\n                type: 'list',\n                choices: preferenceOptions,\n                default: preferenceOptions.find((x) => x === parsedValue),\n              },\n            ]);\n            purposeMapping.valueMapping[parsedValue] = preferenceValue;\n          });\n          return;\n        }\n\n        throw new Error(\n          `Unknown preference topic type: ${preferenceTopic.type}`,\n        );\n      }\n    });\n\n    currentState.columnToPurposeName[col] = purposeMapping;\n  });\n\n  return currentState;\n}\n/* eslint-enable no-param-reassign */\n","import {\n  PreferenceQueryResponseItem,\n  PreferenceUpdateItem,\n} from '@transcend-io/privacy-types';\nimport * as t from 'io-ts';\n\nexport const PurposeRowMapping = t.type({\n  /**\n   * The slug or trackingType of the purpose to map to\n   *\n   * e.g. `Marketing`\n   */\n  purpose: t.string,\n  /**\n   * If the column maps to a preference instead of a purpose\n   * this is the slug of the purpose.\n   *\n   * null value indicates that this column maps to the true/false\n   * value of the purpose\n   */\n  preference: t.union([t.string, t.null]),\n  /**\n   * The mapping between each row value and purpose/preference value.\n   *\n   * e.g. for a boolean preference or purpose\n   * {\n   *  'true': true,\n   *  'false': false,\n   *   '': true,\n   * }\n   *\n   * or for a single or multi select preference\n   * {\n   *   '': true,\n   *   'value1': 'Value1',\n   *   'value2': 'Value2',\n   * }\n   */\n  valueMapping: t.record(\n    t.string,\n    t.union([t.string, t.boolean, t.null, t.undefined]),\n  ),\n});\n\n/** Override type */\nexport type PurposeRowMapping = t.TypeOf<typeof PurposeRowMapping>;\n\n/**\n * Mapping of column name to purpose row mapping.\n * This is used to map each column in the CSV to the relevant purpose and preference definitions in\n * transcend.\n */\nexport const ColumnPurposeMap = t.record(t.string, PurposeRowMapping);\n\n/** Override type */\nexport type ColumnPurposeMap = t.TypeOf<typeof ColumnPurposeMap>;\n\nexport const IdentifierMetadataForPreference = t.type({\n  /** The identifier name */\n  name: t.string,\n  /** Is unique on preference store */\n  isUniqueOnPreferenceStore: t.boolean,\n});\n\n/** Override type */\nexport type IdentifierMetadataForPreference = t.TypeOf<\n  typeof IdentifierMetadataForPreference\n>;\n\n/**\n * Mapping of identifier name to the column name in the CSV file.\n * This is used to map each identifier name to the column in the CSV file.\n */\nexport const ColumnIdentifierMap = t.record(\n  t.string,\n  IdentifierMetadataForPreference,\n);\n\n/** Override type */\nexport type ColumnIdentifierMap = t.TypeOf<typeof ColumnIdentifierMap>;\n\nexport const FileMetadataState = t.intersection([\n  t.type({\n    /**\n     * Definition of how to map each column in the CSV to\n     * the relevant purpose and preference definitions in transcend\n     */\n    columnToPurposeName: t.record(t.string, PurposeRowMapping),\n    /** Last time the file was last parsed at */\n    lastFetchedAt: t.string,\n    /**\n     * Mapping of userId to the rows in the file that need to be uploaded\n     * These uploads are overwriting non-existent preferences and are safe\n     */\n    pendingSafeUpdates: t.record(t.string, t.record(t.string, t.string)),\n    /**\n     * Mapping of userId to the rows in the file that need to be uploaded\n     * these records have conflicts with existing consent preferences\n     */\n    pendingConflictUpdates: t.record(\n      t.string,\n      t.type({\n        record: PreferenceQueryResponseItem,\n        row: t.record(t.string, t.string),\n      }),\n    ),\n    /**\n     * Mapping of userId to the rows in the file that can be skipped because\n     * their preferences are already in the store\n     */\n    skippedUpdates: t.record(t.string, t.record(t.string, t.string)),\n  }),\n  t.partial({\n    /** Determine which column name in file maps to consent record identifier to upload on  */\n    identifierColumn: t.string,\n    /** Determine which column name in file maps to the timestamp  */\n    timestampColum: t.string,\n  }),\n]);\n\n/** Override type */\nexport type FileMetadataState = t.TypeOf<typeof FileMetadataState>;\n\n/**\n * This is the type of the receipts that are stored in the file\n * that is used to track the state of the upload process.\n * It is used to resume the upload process from where it left off.\n * It is used to persist the state of the upload process across multiple runs.\n */\nexport const PreferenceUpdateMap = t.record(\n  t.string,\n  // This can either be true to indicate the record is pending\n  // or it can be an object showing the object\n  // We only return a fixed number of results to avoid\n  // making the JSON file too large\n  t.union([t.boolean, PreferenceUpdateItem]),\n);\n\n/** Override type */\nexport type PreferenceUpdateMap = t.TypeOf<typeof PreferenceUpdateMap>;\n\n/**\n * This is the type of the pending updates that are safe to run without\n * conflicts with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is safe to upload.\n */\nexport const PendingSafePreferenceUpdates = t.record(\n  t.string,\n  // This can either be true to indicate the record is safe\n  // or it can be an object showing the object\n  // We only return a fixed number of results to avoid\n  // making the JSON file too large\n  t.union([t.boolean, t.record(t.string, t.string)]),\n);\n\n/** Override type */\nexport type PendingSafePreferenceUpdates = t.TypeOf<\n  typeof PendingSafePreferenceUpdates\n>;\n\n/**\n * These are the updates that failed to be uploaded to the API.\n */\nexport const FailingPreferenceUpdates = t.record(\n  t.string,\n  t.type({\n    /** Time upload ran at */\n    uploadedAt: t.string,\n    /** Attempts to upload that resulted in an error */\n    error: t.string,\n    /** The update body */\n    update: PreferenceUpdateItem,\n  }),\n);\n\n/** Override type */\nexport type FailingPreferenceUpdates = t.TypeOf<\n  typeof FailingPreferenceUpdates\n>;\n\n/**\n * This is the type of the pending updates that are in conflict with existing consent preferences.\n *\n * Key is primaryKey of the record in the file.\n * The value is the row in the file that is pending upload.\n */\nexport const PendingWithConflictPreferenceUpdates = t.record(\n  t.string,\n  // We always return the conflicts for investigation\n  t.type({\n    /** Record to be inserted to transcend v1/preferences API */\n    record: PreferenceQueryResponseItem,\n    /** The row in the file that is pending upload */\n    row: t.record(t.string, t.string),\n  }),\n);\n\n/** Override type */\nexport type PendingWithConflictPreferenceUpdates = t.TypeOf<\n  typeof PendingWithConflictPreferenceUpdates\n>;\n\n/**\n * The set of preference updates that are skipped\n * Key is primaryKey and value is the row in the CSV\n * that is skipped.\n *\n * This is usually because the preferences are already in the store\n * or there are duplicate rows in the CSV file that are identical.\n */\nexport const SkippedPreferenceUpdates = t.record(\n  t.string,\n  t.record(t.string, t.string),\n);\n\n/** Override type */\nexport type SkippedPreferenceUpdates = t.TypeOf<\n  typeof SkippedPreferenceUpdates\n>;\n\n/** Persist this data between runs of the script */\nexport const PreferenceState = t.type({\n  /**\n   * Store a cache of previous files read in\n   */\n  fileMetadata: t.record(t.string, FileMetadataState),\n  /**\n   * The set of successful uploads to Transcend\n   * Mapping from userId to the upload metadata\n   */\n  failingUpdates: t.record(\n    t.string,\n    t.type({\n      /** Time upload ran at */\n      uploadedAt: t.string,\n      /** Attempts to upload that resulted in an error */\n      error: t.string,\n      /** The update body */\n      update: PreferenceUpdateItem,\n    }),\n  ),\n  /**\n   * The set of pending uploads to Transcend\n   * Mapping from userId to the upload metadata\n   */\n  pendingUpdates: t.record(t.string, PreferenceUpdateItem),\n});\n\n/** Override type */\nexport type PreferenceState = t.TypeOf<typeof PreferenceState>;\n"]}

package/dist/{impl-YIRKVDLY.cjs → impl-TXQ7EFQE.cjs}

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkCUUCCUX2cjs = require('./chunk-CUUCCUX2.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-P7WCYR3A.cjs');require('./chunk-Q7I37FJV.cjs');async function a({auth:o,transcendUrl:r,enricherIds:e}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkCUUCCUX2cjs.Lf.call(void 0, {transcendUrl:r,auth:o,enricherIds:e})}exports.skipPreflightJobs = a;
-//# sourceMappingURL=impl-YIRKVDLY.cjs.map
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function a({auth:o,transcendUrl:r,enricherIds:e}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit),await _chunkKKAJPI6Ocjs.Lf.call(void 0, {transcendUrl:r,auth:o,enricherIds:e})}exports.skipPreflightJobs = a;
+//# sourceMappingURL=impl-TXQ7EFQE.cjs.map

package/dist/{impl-YIRKVDLY.cjs.map → impl-TXQ7EFQE.cjs.map}

@@ -1 +1 @@
-{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-YIRKVDLY.cjs","../src/commands/request/skip-preflight-jobs/impl.ts"],"names":["skipPreflightJobs","auth","transcendUrl","enricherIds","doneInputValidation"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCUtM,SAAsBA,CAAAA,CAEpB,CAAE,IAAA,CAAAC,CAAAA,CAAM,YAAA,CAAAC,CAAAA,CAAc,WAAA,CAAAC,CAAY,CAAA,CACnB,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMJ,kCAAAA,CACJ,YAAA,CAAAE,CAAAA,CACA,IAAA,CAAAD,CAAAA,CACA,WAAA,CAAAE,CACF,CAAC,CACH,CAAA,8BAAA","file":"/home/runner/work/cli/cli/dist/impl-YIRKVDLY.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { skipPreflightJobs as skipPreflightJobsHelper } from '../../../lib/requests';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface SkipPreflightJobsCommandFlags {\n  auth: string;\n  enricherIds: string[];\n  transcendUrl: string;\n}\n\nexport async function skipPreflightJobs(\n  this: LocalContext,\n  { auth, transcendUrl, enricherIds }: SkipPreflightJobsCommandFlags,\n): Promise<void> {\n  doneInputValidation(this.process.exit);\n\n  await skipPreflightJobsHelper({\n    transcendUrl,\n    auth,\n    enricherIds,\n  });\n}\n"]}
+{"version":3,"sources":["/home/runner/work/cli/cli/dist/impl-TXQ7EFQE.cjs","../src/commands/request/skip-preflight-jobs/impl.ts"],"names":["skipPreflightJobs","auth","transcendUrl","enricherIds","doneInputValidation"],"mappings":"AAAA,iIAAwC,wDAA0C,gCAA6B,gCAA6B,gCAA6B,gCAA6B,MCUtM,SAAsBA,CAAAA,CAEpB,CAAE,IAAA,CAAAC,CAAAA,CAAM,YAAA,CAAAC,CAAAA,CAAc,WAAA,CAAAC,CAAY,CAAA,CACnB,CACfC,iCAAAA,IAAoB,CAAK,OAAA,CAAQ,IAAI,CAAA,CAErC,MAAMJ,kCAAAA,CACJ,YAAA,CAAAE,CAAAA,CACA,IAAA,CAAAD,CAAAA,CACA,WAAA,CAAAE,CACF,CAAC,CACH,CAAA,8BAAA","file":"/home/runner/work/cli/cli/dist/impl-TXQ7EFQE.cjs","sourcesContent":[null,"import type { LocalContext } from '../../../context';\nimport { skipPreflightJobs as skipPreflightJobsHelper } from '../../../lib/requests';\nimport { doneInputValidation } from '../../../lib/cli/done-input-validation';\n\nexport interface SkipPreflightJobsCommandFlags {\n  auth: string;\n  enricherIds: string[];\n  transcendUrl: string;\n}\n\nexport async function skipPreflightJobs(\n  this: LocalContext,\n  { auth, transcendUrl, enricherIds }: SkipPreflightJobsCommandFlags,\n): Promise<void> {\n  doneInputValidation(this.process.exit);\n\n  await skipPreflightJobsHelper({\n    transcendUrl,\n    auth,\n    enricherIds,\n  });\n}\n"]}

package/dist/{impl-GOJPWPSQ.cjs → impl-VKEICKZ7.cjs}

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkOZ5HYZX2cjs = require('./chunk-OZ5HYZX2.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkCUUCCUX2cjs = require('./chunk-CUUCCUX2.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-P7WCYR3A.cjs');require('./chunk-Q7I37FJV.cjs');async function g({base64EncryptionKey:t,base64SigningKey:s,partition:i,file:a,consentUrl:c,concurrency:p}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let m=_chunkCUUCCUX2cjs.rc.call(void 0, a,_chunkOZ5HYZX2cjs.b);await _chunkOZ5HYZX2cjs.f.call(void 0, {base64EncryptionKey:t,base64SigningKey:s,preferences:m,partition:i,concurrency:p,transcendUrl:c})}exports.uploadConsentPreferences = g;
-//# sourceMappingURL=impl-GOJPWPSQ.cjs.map
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});var _chunkZUT7UIQUcjs = require('./chunk-ZUT7UIQU.cjs');var _chunkWKCTKYN4cjs = require('./chunk-WKCTKYN4.cjs');var _chunkKKAJPI6Ocjs = require('./chunk-KKAJPI6O.cjs');require('./chunk-LCDYXJN6.cjs');require('./chunk-ZUNVPK23.cjs');require('./chunk-VY55L6P5.cjs');require('./chunk-Q7I37FJV.cjs');async function g({base64EncryptionKey:t,base64SigningKey:s,partition:i,file:a,consentUrl:c,concurrency:p}){_chunkWKCTKYN4cjs.a.call(void 0, this.process.exit);let m=_chunkKKAJPI6Ocjs.rc.call(void 0, a,_chunkZUT7UIQUcjs.b);await _chunkZUT7UIQUcjs.f.call(void 0, {base64EncryptionKey:t,base64SigningKey:s,preferences:m,partition:i,concurrency:p,transcendUrl:c})}exports.uploadConsentPreferences = g;
+//# sourceMappingURL=impl-VKEICKZ7.cjs.map