npm - @transcend-io/cli - Versions diffs - 10.27.3 → 10.27.5 - Mend

@transcend-io/cli 10.27.3 → 10.27.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (355) hide show

package/dist/{mergeTranscendInputs-DjPe49yR.mjs.map → mergeTranscendInputs-CkpitZbH.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"mergeTranscendInputs-DjPe49yR.mjs","names":[],"sources":["../src/lib/preference-management/validatePreferenceManagementSlugs.ts","../src/lib/graphql/ensureAllDataSubjectsExist.ts","../src/lib/graphql/normalizeDeletionDependencies.ts","../src/lib/graphql/syncDataSilos.ts","../src/lib/graphql/syncConfigurationToTranscend.ts","../src/lib/mergeTranscendInputs.ts"],"sourcesContent":["import type { TranscendInput } from '../../codecs.js';\n\n/** Slugs for preference topics and option values must be alphabetical letters only. /\nexport const PREFERENCE_SLUG_REGEX = /^[A-Za-z]+$/;\n\n/\n Validate preference topic and option value slugs before push.\n \n @param input - Parsed transcend.yml input\n * @returns List of validation error messages (empty when valid)\n /\nexport function validatePreferenceManagementSlugs(input: TranscendInput): string[] {\n const errors: string[] = [];\n\n const validateSlug = (slug: string, context: string): void => {\n if (!PREFERENCE_SLUG_REGEX.test(slug)) {\n errors.push(\n `${context}: slug \"${slug}\" is invalid — slugs must contain only alphabetical letters (A-Z, a-z). ` +\n 'Topic slugs are normalized to PascalCase server-side, so YAML slugs should already be PascalCase ' +\n 'or matching will break on subsequent pulls.',\n );\n }\n };\n\n for (const option of input['preference-options'] ?? []) {\n validateSlug(option.slug, 'preference-options');\n }\n\n for (const purpose of input.purposes ?? []) {\n for (const topic of purpose['preference-topics'] ?? []) {\n validateSlug(topic.slug, `purposes (topic \"${topic.title}\")`);\n for (const option of topic.options ?? []) {\n validateSlug(option.slug, `purposes (topic \"${topic.title}\" option \"${option.title}\")`);\n }\n }\n }\n\n return errors;\n}\n","import { createDataSubject, fetchAllDataSubjects, type DataSubject } from '@transcend-io/sdk';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { keyBy, flatten, uniq, difference } from 'lodash-es';\n\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\n/\n Fetch all of the data subjects in the organization\n \n @param input - Input to fetch\n * @param client - GraphQL client\n * @param fetchAll - When true, always fetch all subjects\n * @returns The list of data subjects\n /\nexport async function ensureAllDataSubjectsExist(\n {\n 'data-silos': dataSilos = [],\n 'data-subjects': dataSubjects = [],\n 'processing-activities': processingActivities = [],\n enrichers = [],\n }: TranscendInput,\n client: GraphQLClient,\n fetchAll = false,\n): Promise<{ [type in string]: DataSubject }> {\n const expectedDataSubjects = uniq([\n ...flatten(dataSilos.map((silo) => silo['data-subjects'] \|\| []) \|\| []),\n ...flatten(processingActivities.map(({ dataSubjectTypes }) => dataSubjectTypes ?? []) ?? []),\n ...flatten(enrichers.map((enricher) => enricher['data-subjects'] \|\| []) \|\| []),\n ...dataSubjects.map((subject) => subject.type),\n ]);\n if (expectedDataSubjects.length === 0 && !fetchAll) {\n return {};\n }\n\n const internalSubjects = await fetchAllDataSubjects(client, { logger });\n const dataSubjectByName = keyBy(internalSubjects, 'type');\n\n const missingDataSubjects = difference(\n expectedDataSubjects,\n internalSubjects.map(({ type }) => type),\n );\n\n if (missingDataSubjects.length > 0) {\n logger.info(colors.magenta(`Creating ${missingDataSubjects.length} new data subjects...`));\n for (const dataSubjectType of missingDataSubjects) {\n logger.info(colors.magenta(`Creating data subject ${dataSubjectType}...`));\n const created = await createDataSubject(client, { input: dataSubjectType, logger });\n logger.info(colors.green(`Created data subject ${dataSubjectType}!`));\n dataSubjectByName[dataSubjectType] = created;\n }\n }\n\n return dataSubjectByName;\n}\n","import type { DependedOnDataSiloInput } from '@transcend-io/sdk';\nimport { uniq } from 'lodash-es';\n\nimport type { DeletionDependencies, DeletionDependencyInput } from '../../codecs.js';\n\n/\n Convert the `deletion-dependencies` entries for a single data silo into the\n * `dependedOnDataSilos` input accepted by the `updateDataSilos` mutation.\n \n A list of titles becomes a single global entry. A list of objects may declare\n * at most one global configuration (`{ titles }`) plus per-workflow overrides;\n * workflows that are absent from the list are left untouched.\n \n @param dependencies - The `deletion-dependencies` entries from transcend.yml\n * @param dataSiloTitle - Title of the data silo being synced, used in error messages\n * @returns The dependency entries to send to the API\n /\nexport function normalizeDeletionDependencies(\n dependencies: DeletionDependencies,\n dataSiloTitle: string,\n): DependedOnDataSiloInput[] {\n if (dependencies.length === 0) {\n return [];\n }\n\n // Legacy / no-overrides form: a flat list of data silo titles\n if (typeof dependencies[0] === 'string') {\n return [{ titles: uniq(dependencies as string[]) }];\n }\n\n let globalTitles: string[] \| undefined;\n const workflowOverrides: DependedOnDataSiloInput[] = [];\n const seenWorkflows = new Set<string>();\n\n /\n Record an override for a single workflow, rejecting duplicates before the API does\n * so that the error names the data silo\n \n @param workflow - Internal name of the workflow being overridden\n * @param override - Titles or reset flag to send for that workflow\n /\n const addWorkflowOverride = (\n workflow: string,\n override: Omit<DependedOnDataSiloInput, 'workflowConfigInternalName'>,\n ): void => {\n if (seenWorkflows.has(workflow)) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has multiple deletion-dependencies entries for workflow ` +\n `\"${workflow}\". Combine them into a single entry.`,\n );\n }\n seenWorkflows.add(workflow);\n workflowOverrides.push({ workflowConfigInternalName: workflow, ...override });\n };\n\n (dependencies as DeletionDependencyInput[]).forEach((dependency) => {\n // io-ts codecs allow extra properties, so this combination decodes cleanly\n // even though the two halves contradict each other\n if ('reset-to-global' in dependency && 'titles' in dependency) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has a deletion-dependencies entry for workflow ` +\n `\"${dependency.workflow}\" that sets both \"reset-to-global\" and \"titles\". ` +\n 'Use \"titles: []\" to override the global configuration with no dependencies, ' +\n 'or \"reset-to-global: true\" to fall back to the global configuration.',\n );\n }\n\n if ('reset-to-global' in dependency) {\n addWorkflowOverride(dependency.workflow, { resetToGlobal: true });\n return;\n }\n\n if (dependency.workflow) {\n addWorkflowOverride(dependency.workflow, { titles: dependency.titles });\n return;\n }\n\n if (globalTitles !== undefined) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has multiple global deletion-dependencies entries. ` +\n 'Combine them into a single `{ titles: [...] }` entry.',\n );\n }\n globalTitles = dependency.titles;\n });\n\n return [\n ...(globalTitles !== undefined ? [{ titles: uniq(globalTitles) }] : []),\n ...workflowOverrides,\n ];\n}\n","import {\n makeGraphQLRequest,\n ApiKey,\n convertToDataSubjectBlockList,\n type DataSubject,\n fetchAllDataSilos,\n type DataSilo,\n CREATE_DATA_SILOS,\n UPDATE_DATA_SILOS,\n UPDATE_OR_CREATE_DATA_POINT,\n} from '@transcend-io/sdk';\nimport { apply } from '@transcend-io/type-utils';\nimport { mapSeries, map } from '@transcend-io/utils';\n/ eslint-disable max-lines /\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { chunk, keyBy } from 'lodash-es';\n\nimport { DataSiloInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nconst BATCH_SILOS_LIMIT = 20;\n\n/\n Sync a data silo configuration\n \n @param dataSilos - Data silos to sync\n * @param client - GraphQL client\n * @param options - Options\n * @returns Data silo info\n /\nexport async function syncDataSilos(\n dataSilos: DataSiloInput[],\n client: GraphQLClient,\n {\n pageSize,\n dataSubjectsByName,\n apiKeysByTitle,\n }: {\n /* Page size /\n pageSize: number;\n /* The data subjects in the organization /\n dataSubjectsByName: { [type in string]: DataSubject };\n /* API key title to API key /\n apiKeysByTitle: { [title in string]: ApiKey };\n },\n): Promise<{\n /* Whether successfully updated /\n success: boolean;\n /* A mapping between data silo title to data silo ID /\n dataSiloTitleToId: { [k in string]: string };\n}> {\n let encounteredError = false;\n\n // Time duration\n const t0 = new Date().getTime();\n logger.info(colors.magenta(`Syncing \"${dataSilos.length}\" data silos...`));\n\n // Determine the set of data silos that already exist\n const existingDataSilos = await fetchAllDataSilos(client, {\n titles: dataSilos.map(({ title }) => title),\n pageSize,\n logger,\n });\n\n // Create a mapping of title -> existing silo, if it exists\n const existingDataSiloByTitle = keyBy<Pick<DataSilo, 'id' \| 'title'>>(existingDataSilos, 'title');\n\n // Create new silos that do not exist\n const newDataSiloInputs = dataSilos.filter(({ title }) => !existingDataSiloByTitle[title]);\n if (newDataSiloInputs.length > 0) {\n logger.info(\n colors.magenta(`Creating \"${newDataSiloInputs.length}\" data silos that did not exist...`),\n );\n\n // Batch the creation\n const chunked = chunk(newDataSiloInputs, BATCH_SILOS_LIMIT);\n await mapSeries(chunked, async (dependencyUpdateChunk) => {\n const {\n createDataSilos: { dataSilos },\n } = await makeGraphQLRequest<{\n /* Mutation result /\n createDataSilos: {\n /* New data silos /\n dataSilos: Pick<DataSilo, 'id' \| 'title'>[];\n };\n }>(client, CREATE_DATA_SILOS, {\n variables: {\n input: dependencyUpdateChunk.map((input) => ({\n name: input['outer-type'] \|\| input.integrationName,\n title: input.title,\n country: input.country,\n countrySubDivision: input.countrySubDivision,\n sombraId: input['sombra-id'],\n })),\n },\n logger,\n });\n\n // save mapping of title and id\n dataSilos.forEach((silo) => {\n existingDataSiloByTitle[silo.title] = silo;\n });\n });\n\n logger.info(colors.green(`Successfully created \"${newDataSiloInputs.length}\" data silos!`));\n }\n\n // Batch the updates\n const chunkedUpdates = chunk(dataSilos, BATCH_SILOS_LIMIT);\n await mapSeries(chunkedUpdates, async (dataSiloUpdateChunk, ind) => {\n logger.info(\n colors.magenta(\n `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing \"${\n dataSiloUpdateChunk.length\n }\" data silos`,\n ),\n );\n await makeGraphQLRequest<{\n /* Mutation result /\n updateDataSilos: {\n /* New data silos /\n dataSilos: Pick<DataSilo, 'id' \| 'title'>[];\n };\n }>(client, UPDATE_DATA_SILOS, {\n variables: {\n input: {\n dataSilos: dataSiloUpdateChunk.map((input) => ({\n id: existingDataSiloByTitle[input.title].id,\n country: input.country,\n countrySubDivision: input.countrySubDivision,\n url: input.url,\n headers: input.headers,\n description: input.description,\n identifiers: input['identity-keys'],\n isLive: !input.disabled,\n ownerEmails: input.owners,\n teamNames: input.teams,\n // clear out the global config if not specified, otherwise the update needs to be\n // applied after all data silos are created\n dependedOnDataSilos: input['deletion-dependencies'] ? undefined : [{ titles: [] }],\n apiKeyId: input['api-key-title']\n ? apiKeysByTitle[input['api-key-title']].id\n : undefined,\n dataSubjectBlockListIds: input['data-subjects']\n ? convertToDataSubjectBlockList(input['data-subjects'], dataSubjectsByName)\n : undefined,\n attributes: input.attributes,\n businessEntityTitles: input.businessEntityTitles,\n sombraId: input['sombra-id'],\n // AVC settings\n notifyEmailAddress: input['email-settings']?.['notify-email-address'],\n promptAVendorEmailSendFrequency: input['email-settings']?.['send-frequency'],\n promptAVendorEmailSendType: input['email-settings']?.['send-type'],\n promptAVendorEmailIncludeIdentifiersAttachment:\n input['email-settings']?.['include-identifiers-attachment'],\n promptAVendorEmailCompletionLinkType: input['email-settings']?.['completion-link-type'],\n manualWorkRetryFrequency: input['email-settings']?.['manual-work-retry-frequency'],\n })),\n },\n },\n logger,\n });\n logger.info(\n colors.green(\n `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced \"${\n dataSiloUpdateChunk.length\n }\" data silos!`,\n ),\n );\n });\n\n // Sync datapoints\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n const dataSilosWithDataPoints = dataSilos.filter(({ datapoints = [] }) => datapoints.length > 0);\n const totalDataPoints = dataSilos\n .map(({ datapoints = [] }) => datapoints.length)\n .reduce((acc, count) => acc + count, 0);\n logger.info(\n colors.magenta(\n `Syncing \"${totalDataPoints}\" datapoints from \"${dataSilosWithDataPoints.length}\" data silos...`,\n ),\n );\n progressBar.start(totalDataPoints, 0);\n let total = 0;\n\n await map(\n dataSilosWithDataPoints,\n async ({ datapoints, title }) => {\n if (datapoints) {\n await mapSeries(datapoints, async (datapoint) => {\n const fields = datapoint.fields\n ? datapoint.fields.map(\n ({ key, description, categories, purposes, attributes, ...rest }) =>\n // TODO: Support setting title separately from the 'key/name'\n ({\n name: key,\n description,\n categories: !categories\n ? undefined\n : categories.map((category) => ({\n ...category,\n name: category.name \|\| 'Other',\n })),\n purposes: !purposes\n ? undefined\n : purposes.map((purpose) => ({\n ...purpose,\n name: purpose.name \|\| 'Other',\n })),\n attributes,\n accessRequestVisibilityEnabled: rest['access-request-visibility-enabled'],\n erasureRequestRedactionEnabled: rest['erasure-request-redaction-enabled'],\n }),\n )\n : undefined;\n\n const payload = {\n dataSiloId: existingDataSiloByTitle[title].id,\n path: datapoint.path,\n name: datapoint.key,\n title: datapoint.title,\n description: datapoint.description,\n ...(datapoint.owners\n ? {\n ownerEmails: datapoint.owners,\n }\n : {}),\n ...(datapoint.teams\n ? {\n teamNames: datapoint.teams,\n }\n : {}),\n ...(datapoint['data-collection-tag']\n ? { dataCollectionTag: datapoint['data-collection-tag'] }\n : {}),\n querySuggestions: !datapoint['privacy-action-queries']\n ? undefined\n : Object.entries(datapoint['privacy-action-queries']).map(([key, value]) => ({\n requestType: key,\n suggestedQuery: value,\n })),\n enabledActions: datapoint['privacy-actions'] \|\| [], // clear out when not specified\n subDataPoints: fields,\n };\n\n // Ensure no duplicate sub-datapoints are provided\n const subDataPointsToUpdate = (payload.subDataPoints \|\| []).map(({ name }) => name);\n const duplicateDataPoints = subDataPointsToUpdate.filter(\n (name, index) => subDataPointsToUpdate.indexOf(name) !== index,\n );\n if (duplicateDataPoints.length > 0) {\n logger.info(\n colors.red(\n `\\nCannot update datapoint \"${\n datapoint.key\n }\" as it has duplicate sub-datapoints with the same name: \\n${duplicateDataPoints.join(\n '\\n',\n )}`,\n ),\n );\n encounteredError = true;\n } else {\n try {\n await makeGraphQLRequest(client, UPDATE_OR_CREATE_DATA_POINT, {\n variables: payload,\n logger,\n });\n } catch (err) {\n logger.info(\n colors.red(\n `\\nFailed to update datapoint \"${datapoint.key}\" for data silo \"${title}\"! - \\n${err.message}`,\n ),\n );\n encounteredError = true;\n }\n }\n total += 1;\n progressBar.update(total);\n });\n }\n },\n {\n concurrency: 10,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Synced \"${dataSilos.length}\" data silos and \"${totalDataPoints}\" datapoints in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n return {\n success: !encounteredError,\n dataSiloTitleToId: apply(existingDataSiloByTitle, ({ id }) => id),\n };\n}\n/ eslint-enable max-lines /\n","import {\n fetchAllActions,\n fetchAllAttributes,\n fetchAllDataSubjects,\n fetchApiKeys,\n fetchIdentifiersAndCreateMissing,\n syncAction,\n syncActionItemCollections,\n syncActionItems,\n syncAttribute,\n syncBusinessEntities,\n syncDataCategories,\n syncConsentManager,\n syncCookies,\n syncDataFlows,\n syncDataSubject,\n syncDataSiloDependencies,\n syncEnricher,\n syncIdentifier,\n syncIntlMessages,\n syncPartitions,\n syncPolicies,\n syncPrivacyCenter,\n syncProcessingActivities,\n syncProcessingPurposes,\n syncConsentWorkflowTriggers,\n syncWorkflowConfigs,\n syncPreferenceOptionValues,\n syncPurposes,\n syncPreferenceTopics,\n syncTeams,\n syncTemplate,\n syncVendors,\n type DependedOnDataSiloInput,\n type Identifier,\n} from '@transcend-io/sdk';\nimport { map, type Logger, type SyncError, type SyncResult } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\n\n/ eslint-disable max-lines /\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { validatePreferenceManagementSlugs } from '../preference-management/validatePreferenceManagementSlugs.js';\nimport { ensureAllDataSubjectsExist } from './ensureAllDataSubjectsExist.js';\nimport { normalizeDeletionDependencies } from './normalizeDeletionDependencies.js';\nimport { syncDataSilos } from './syncDataSilos.js';\n\nconst CONCURRENCY = 10;\n\n/\n Sync the yaml input back to Transcend using the GraphQL APIs\n \n @param input - The yml input\n * @param client - GraphQL client\n * @param pageSize - Page size\n * @returns Structured sync result with per-resource errors\n /\nexport async function syncConfigurationToTranscend(\n input: TranscendInput,\n client: GraphQLClient,\n {\n pageSize = 50,\n // TODO: https://transcend.height.app/T-23779\n publishToPrivacyCenter = true,\n classifyService = false,\n deleteExtraAttributeValues = false,\n logger: syncLogger,\n warnings = [],\n }: {\n /* Page size /\n pageSize?: number;\n /* When true, skip publishing to privacy center /\n publishToPrivacyCenter?: boolean;\n /* When true, delete any attributes being synced up /\n deleteExtraAttributeValues?: boolean;\n /* classify data flow service if missing /\n classifyService?: boolean;\n /* Optional logger (e.g. collecting logger for MCP debug responses) /\n logger?: Logger;\n /* Non-fatal warnings to include in the result /\n warnings?: string[];\n },\n): Promise<SyncResult> {\n const activeLogger = syncLogger ?? logger;\n const errors: SyncError[] = [];\n let encounteredError = false;\n\n const recordError = (resource: string, message: string, item?: string): void => {\n errors.push({ resource, item, message });\n encounteredError = true;\n };\n\n activeLogger.info(colors.magenta(`Fetching data with page size ${pageSize}...`));\n\n const {\n templates,\n attributes,\n actions,\n identifiers,\n 'data-subjects': dataSubjects,\n 'business-entities': businessEntities,\n enrichers,\n cookies,\n 'consent-manager': consentManager,\n 'data-silos': dataSilos,\n 'data-flows': dataFlows,\n vendors,\n 'data-categories': dataCategories,\n 'processing-activities': processingActivities,\n 'processing-purposes': processingPurposes,\n 'action-items': actionItems,\n 'action-item-collections': actionItemCollections,\n teams,\n 'privacy-center': privacyCenter,\n messages,\n policies,\n partitions,\n 'preference-workflow-configs': preferenceWorkflowConfigs,\n 'workflow-configs': workflowConfigs,\n } = input;\n\n const preferenceOptions = input['preference-options'];\n const purposes = input.purposes;\n\n const preferenceSlugErrors = validatePreferenceManagementSlugs(input);\n if (preferenceSlugErrors.length > 0) {\n for (const message of preferenceSlugErrors) {\n recordError('preference-slugs', message);\n }\n return {\n success: false,\n errors,\n warnings: warnings.length > 0 ? warnings : undefined,\n };\n }\n\n const [identifierByName, dataSubjectsByName, apiKeyTitleMap] = await Promise.all([\n // Ensure all identifiers are created and create a map from name -> identifier.id\n enrichers \|\| identifiers\n ? fetchIdentifiersAndCreateMissing(client, {\n input,\n skipPublish: !publishToPrivacyCenter,\n logger,\n })\n : ({} as { [k in string]: Identifier }),\n // Grab all data subjects in the organization\n dataSilos \|\| dataSubjects \|\| enrichers \|\| processingActivities\n ? ensureAllDataSubjectsExist(input, client)\n : {},\n // Grab API keys\n dataSilos &&\n dataSilos\n .map((dataSilo) => dataSilo['api-key-title'] \|\| [])\n .reduce((acc, lst) => acc + lst.length, 0) > 0\n ? fetchApiKeys(client, { apiKeyInputs: input, logger })\n : {},\n ]);\n\n if (preferenceWorkflowConfigs?.length) {\n const preferenceWorkflowConfigsSuccess = await syncConsentWorkflowTriggers(\n client,\n preferenceWorkflowConfigs,\n {\n logger: activeLogger,\n pageSize,\n },\n );\n if (!preferenceWorkflowConfigsSuccess) {\n recordError('preference-workflow-configs', 'Failed to sync preference workflow configs');\n }\n }\n\n if (workflowConfigs?.length) {\n const workflowConfigsSuccess = await syncWorkflowConfigs(client, workflowConfigs, {\n logger: activeLogger,\n });\n if (!workflowConfigsSuccess) {\n recordError('workflow-configs', 'Failed to sync workflow configs');\n }\n }\n\n if (preferenceOptions?.length) {\n const preferenceOptionsSuccess = await syncPreferenceOptionValues(client, preferenceOptions, {\n logger: activeLogger,\n });\n if (!preferenceOptionsSuccess) {\n recordError('preference-options', 'Failed to sync preference option values');\n }\n }\n\n // Sync consent purposes and their nested preference topics.\n // Order matters: option values (above) exist first, then purposes, then topics\n // (which link to their purpose by ID and to option values by slug).\n if (purposes?.length) {\n const { success: purposesSuccess, purposeIdByTrackingType } = await syncPurposes(\n client,\n purposes,\n { logger: activeLogger },\n );\n if (!purposesSuccess) {\n recordError('purposes', 'Failed to sync one or more purposes');\n }\n\n const topics = purposes.flatMap((purpose) =>\n (purpose['preference-topics'] ?? []).map((topic) => ({\n ...topic,\n 'tracking-type': purpose.trackingType,\n })),\n );\n if (topics.length > 0) {\n const topicsSuccess = await syncPreferenceTopics(client, topics, {\n logger: activeLogger,\n purposeIdByTrackingType,\n });\n if (!topicsSuccess) {\n recordError('preference-topics', 'Failed to sync one or more preference topics');\n }\n }\n }\n\n // Sync consent manager\n if (consentManager) {\n activeLogger.info(colors.magenta('Syncing consent manager...'));\n try {\n await syncConsentManager(client, consentManager, { logger: activeLogger });\n activeLogger.info(colors.green('Successfully synced consent manager!'));\n } catch (err) {\n recordError('consent-manager', (err as Error).message);\n activeLogger.error(colors.red(`Failed to sync consent manager! - ${(err as Error).message}`));\n }\n }\n\n if (teams) {\n const teamsSuccess = await syncTeams(client, teams, { logger });\n encounteredError = encounteredError \|\| !teamsSuccess;\n }\n\n // Sync email templates\n if (templates) {\n logger.info(colors.magenta(`Syncing \"${templates.length}\" email templates...`));\n await map(\n templates,\n async (template) => {\n logger.info(colors.magenta(`Syncing template \"${template.title}\"...`));\n try {\n await syncTemplate(client, template, { logger });\n logger.info(colors.green(`Successfully synced template \"${template.title}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync template \"${template.title}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${templates.length}\" email templates!`));\n }\n\n // Sync business entities\n if (businessEntities) {\n const businessEntitySuccess = await syncBusinessEntities(client, businessEntities, { logger });\n encounteredError = encounteredError \|\| !businessEntitySuccess;\n }\n\n // Sync vendors\n if (vendors) {\n const vendorsSuccess = await syncVendors(client, vendors, { logger });\n encounteredError = encounteredError \|\| !vendorsSuccess;\n }\n\n // Sync data categories\n if (dataCategories) {\n const dataCategoriesSuccess = await syncDataCategories(client, dataCategories, { logger });\n encounteredError = encounteredError \|\| !dataCategoriesSuccess;\n }\n\n // Sync processing purposes\n if (processingPurposes) {\n const processingPurposesSuccess = await syncProcessingPurposes(client, processingPurposes, {\n logger,\n });\n encounteredError = encounteredError \|\| !processingPurposesSuccess;\n }\n\n // Sync partitions\n if (partitions) {\n const partitionsSuccess = await syncPartitions(client, partitions, { logger });\n encounteredError = encounteredError \|\| !partitionsSuccess;\n }\n\n // Sync cookies\n if (cookies) {\n const cookiesSuccess = await syncCookies(client, cookies, { logger });\n encounteredError = encounteredError \|\| !cookiesSuccess;\n }\n\n // Sync action item collections\n if (actionItemCollections) {\n const actionItemCollectionsSuccess = await syncActionItemCollections(\n client,\n actionItemCollections,\n { logger },\n );\n encounteredError = encounteredError \|\| !actionItemCollectionsSuccess;\n }\n\n // Sync attributes\n if (attributes) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${attributes.length}\" attributes...`));\n const existingAttributes = await fetchAllAttributes(client, { logger });\n await map(\n attributes,\n async (attribute) => {\n const existing = existingAttributes.find((attr) => attr.name === attribute.name);\n\n logger.info(colors.magenta(`Syncing attribute \"${attribute.name}\"...`));\n try {\n await syncAttribute(client, attribute, {\n existingAttribute: existing,\n deleteExtraAttributeValues,\n logger,\n });\n logger.info(colors.green(`Successfully synced attribute \"${attribute.name}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(\n colors.red(`Failed to sync attribute \"${attribute.name}\"! - ${err.message}`),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${attributes.length}\" attributes!`));\n }\n\n // Sync action items\n if (actionItems) {\n const actionItemsSuccess = await syncActionItems(client, actionItems, { logger });\n encounteredError = encounteredError \|\| !actionItemsSuccess;\n }\n\n // Sync enrichers\n if (enrichers) {\n logger.info(colors.magenta(`Syncing \"${enrichers.length}\" enrichers...`));\n await map(\n enrichers,\n async (enricher) => {\n logger.info(colors.magenta(`Syncing enricher \"${enricher.title}\"...`));\n try {\n await syncEnricher(client, {\n input: enricher,\n identifierByName,\n dataSubjectsByName,\n logger,\n });\n logger.info(colors.green(`Successfully synced enricher \"${enricher.title}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync enricher \"${enricher.title}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${enrichers.length}\" enrichers!`));\n }\n\n // Sync identifiers\n if (identifiers) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${identifiers.length}\" identifiers...`));\n await map(\n identifiers,\n async (identifier) => {\n const existing = identifierByName[identifier.name];\n if (!existing) {\n throw new Error(\n `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing identifier \"${identifier.type}\"...`));\n try {\n await syncIdentifier(client, {\n input: identifier,\n dataSubjectsByName,\n identifierId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n logger,\n });\n logger.info(colors.green(`Successfully synced identifier \"${identifier.type}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.info(\n colors.red(`Failed to sync identifier \"${identifier.type}\"! - ${err.message}`),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${identifiers.length}\" identifiers!`));\n }\n\n // Sync actions\n if (actions) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${actions.length}\" actions...`));\n const existingActions = await fetchAllActions(client, { logger });\n await map(\n actions,\n async (action) => {\n const existing = existingActions.find((act) => act.type === action.type);\n if (!existing) {\n throw new Error(\n `Failed to find action with type: ${action.type}. Should have already existing in the organization.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing action \"${action.type}\"...`));\n try {\n await syncAction(\n client,\n {\n action,\n actionId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n },\n { logger },\n );\n logger.info(colors.green(`Successfully synced action \"${action.type}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync action \"${action.type}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${actions.length}\" actions!`));\n }\n\n // Sync data subjects\n if (dataSubjects) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${dataSubjects.length}\" data subjects...`));\n const existingDataSubjects = await fetchAllDataSubjects(client, { logger });\n await map(\n dataSubjects,\n async (dataSubject) => {\n const existing = existingDataSubjects.find((subj) => subj.type === dataSubject.type);\n if (!existing) {\n throw new Error(\n `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing data subject \"${dataSubject.type}\"...`));\n try {\n await syncDataSubject(client, {\n input: dataSubject,\n dataSubjectId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n logger,\n });\n logger.info(colors.green(`Successfully synced data subject \"${dataSubject.type}\"!`));\n } catch (err) {\n encounteredError = true;\n // supportsAuthorizedAgent is gated on the Authorized Agents feature being\n // enabled for the organization; surface a clearer hint than the raw API error\n const hint =\n dataSubject.supportsAuthorizedAgent !== undefined &&\n /authorized.agent/i.test(err.message)\n ? ' (enable the Authorized Agents feature for this organization before setting \"supportsAuthorizedAgent\")'\n : '';\n logger.info(\n colors.red(\n `Failed to sync data subject \"${dataSubject.type}\"! - ${err.message}${hint}`,\n ),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${dataSubjects.length}\" data subjects!`));\n }\n\n // Sync data flows\n if (dataFlows) {\n const syncedDataFlows = await syncDataFlows(client, dataFlows, { classifyService, logger });\n encounteredError = encounteredError \|\| !syncedDataFlows;\n }\n\n // Sync privacy center\n if (privacyCenter) {\n try {\n const privacyCenterSuccess = await syncPrivacyCenter(client, privacyCenter, {\n logger,\n skipPublish: !publishToPrivacyCenter,\n });\n if (!privacyCenterSuccess) {\n recordError('privacy-center', 'Failed to sync privacy center');\n }\n } catch (err) {\n recordError('privacy-center', (err as Error).message);\n }\n }\n\n // Sync messages\n if (messages) {\n const messagesSuccess = await syncIntlMessages(client, messages, { logger });\n encounteredError = encounteredError \|\| !messagesSuccess;\n }\n\n // Sync policies\n if (policies) {\n const policiesSuccess = await syncPolicies(client, policies, { logger });\n encounteredError = encounteredError \|\| !policiesSuccess;\n }\n\n // Store dependency updates\n const dependencyUpdates: [string, DependedOnDataSiloInput[]][] = [];\n // Sync data silos\n if (dataSilos) {\n const { success, dataSiloTitleToId } = await syncDataSilos(dataSilos, client, {\n dataSubjectsByName,\n apiKeysByTitle: apiKeyTitleMap,\n pageSize,\n });\n dataSilos?.forEach((dataSilo) => {\n // Queue up dependency update\n const dependencies = dataSilo['deletion-dependencies'];\n if (dependencies) {\n try {\n dependencyUpdates.push([\n dataSiloTitleToId[dataSilo.title],\n normalizeDeletionDependencies(dependencies, dataSilo.title),\n ]);\n } catch (err) {\n recordError('data-silos', (err as Error).message, dataSilo.title);\n activeLogger.error(colors.red((err as Error).message));\n }\n }\n });\n encounteredError = encounteredError \|\| !success;\n }\n\n // Dependencies updated at the end after all data silos are created\n if (dependencyUpdates.length > 0) {\n const dependenciesSuccess = await syncDataSiloDependencies(client, {\n input: dependencyUpdates,\n logger,\n });\n encounteredError = encounteredError \|\| !dependenciesSuccess;\n }\n\n // Update processing activities\n if (processingActivities) {\n const processingActivitySuccess = await syncProcessingActivities(client, processingActivities, {\n logger,\n });\n encounteredError = encounteredError \|\| !processingActivitySuccess;\n }\n\n if (publishToPrivacyCenter) {\n // TODO: https://transcend.height.app/T-23779\n }\n\n return {\n success: !encounteredError,\n errors,\n warnings: warnings.length > 0 ? warnings : undefined,\n };\n}\n/ eslint-enable max-lines /\n","import { getEntries } from '@transcend-io/type-utils';\n\nimport { TranscendInput } from '../codecs.js';\n\n/\n Combine a set of TranscendInput yaml files into a single yaml\n \n @param base - Base input\n * @param inputs - The list of inputs\n * @returns Merged input\n */\nexport function mergeTranscendInputs(\n base: TranscendInput,\n ...inputs: TranscendInput[]\n): TranscendInput {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const cloned: any = JSON.parse(JSON.stringify(base));\n inputs.forEach((input) => {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n getEntries(input).forEach(([key, value]: [any, any]) => {\n if (cloned[key] === undefined) {\n cloned[key] = value;\n } else if (Array.isArray(value)) {\n cloned[key] = [...cloned[key], ...value];\n } else {\n cloned[key] = value;\n }\n });\n });\n return cloned;\n}\n"],"mappings":"yvCAGA,MAAa,EAAwB,cAQrC,SAAgB,GAAkC,EAAiC,CACjF,IAAM,EAAmB,EAAE,CAErB,GAAgB,EAAc,IAA0B,CACvD,EAAsB,KAAK,EAAK,EACnC,EAAO,KACL,GAAG,EAAQ,UAAU,EAAK,sNAG3B,EAIL,IAAK,IAAM,KAAU,EAAM,uBAAyB,EAAE,CACpD,EAAa,EAAO,KAAM,qBAAqB,CAGjD,IAAK,IAAM,KAAW,EAAM,UAAY,EAAE,CACxC,IAAK,IAAM,KAAS,EAAQ,sBAAwB,EAAE,CAAE,CACtD,EAAa,EAAM,KAAM,oBAAoB,EAAM,MAAM,IAAI,CAC7D,IAAK,IAAM,KAAU,EAAM,SAAW,EAAE,CACtC,EAAa,EAAO,KAAM,oBAAoB,EAAM,MAAM,YAAY,EAAO,MAAM,IAAI,CAK7F,OAAO,ECrBT,eAAsB,EACpB,CACE,aAAc,EAAY,EAAE,CAC5B,gBAAiB,EAAe,EAAE,CAClC,wBAAyB,EAAuB,EAAE,CAClD,YAAY,EAAE,EAEhB,EACA,EAAW,GACiC,CAC5C,IAAM,EAAuB,EAAK,CAChC,GAAG,EAAQ,EAAU,IAAK,GAAS,EAAK,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CACtE,GAAG,EAAQ,EAAqB,KAAK,CAAE,sBAAuB,GAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC5F,GAAG,EAAQ,EAAU,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC9E,GAAG,EAAa,IAAK,GAAY,EAAQ,KAAK,CAC/C,CAAC,CACF,GAAI,EAAqB,SAAW,GAAK,CAAC,EACxC,MAAO,EAAE,CAGX,IAAM,EAAmB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CACjE,EAAoB,EAAM,EAAkB,OAAO,CAEnD,EAAsB,EAC1B,EACA,EAAiB,KAAK,CAAE,UAAW,EAAK,CACzC,CAED,GAAI,EAAoB,OAAS,EAAG,CAClC,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAoB,OAAO,uBAAuB,CAAC,CAC1F,IAAK,IAAM,KAAmB,EAAqB,CACjD,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAgB,KAAK,CAAC,CAC1E,IAAM,EAAU,MAAM,EAAkB,EAAQ,CAAE,MAAO,EAAiB,SAAQ,CAAC,CACnF,EAAO,KAAK,EAAO,MAAM,wBAAwB,EAAgB,GAAG,CAAC,CACrE,EAAkB,GAAmB,GAIzC,OAAO,ECrCT,SAAgB,GACd,EACA,EAC2B,CAC3B,GAAI,EAAa,SAAW,EAC1B,MAAO,EAAE,CAIX,GAAI,OAAO,EAAa,IAAO,SAC7B,MAAO,CAAC,CAAE,OAAQ,EAAK,EAAyB,CAAE,CAAC,CAGrD,IAAI,EACE,EAA+C,EAAE,CACjD,EAAgB,IAAI,IASpB,GACJ,EACA,IACS,CACT,GAAI,EAAc,IAAI,EAAS,CAC7B,MAAU,MACR,cAAc,EAAc,6DACtB,EAAS,sCAChB,CAEH,EAAc,IAAI,EAAS,CAC3B,EAAkB,KAAK,CAAE,2BAA4B,EAAU,GAAG,EAAU,CAAC,EAkC/E,OA/BC,EAA2C,QAAS,GAAe,CAGlE,GAAI,oBAAqB,GAAc,WAAY,EACjD,MAAU,MACR,cAAc,EAAc,oDACtB,EAAW,SAAS,mMAG3B,CAGH,GAAI,oBAAqB,EAAY,CACnC,EAAoB,EAAW,SAAU,CAAE,cAAe,GAAM,CAAC,CACjE,OAGF,GAAI,EAAW,SAAU,CACvB,EAAoB,EAAW,SAAU,CAAE,OAAQ,EAAW,OAAQ,CAAC,CACvE,OAGF,GAAI,IAAiB,IAAA,GACnB,MAAU,MACR,cAAc,EAAc,8GAE7B,CAEH,EAAe,EAAW,QAC1B,CAEK,CACL,GAAI,IAAiB,IAAA,GAA+C,EAAE,CAArC,CAAC,CAAE,OAAQ,EAAK,EAAa,CAAE,CAAC,CACjE,GAAG,EACJ,CCzDH,eAAsB,EACpB,EACA,EACA,CACE,WACA,qBACA,kBAcD,CACD,IAAI,EAAmB,GAGjB,EAAK,IAAI,MAAM,CAAC,SAAS,CAC/B,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,iBAAiB,CAAC,CAU1E,IAAM,EAA0B,EAAsC,MAPtC,EAAkB,EAAQ,CACxD,OAAQ,EAAU,KAAK,CAAE,WAAY,EAAM,CAC3C,WACA,SACD,CAAC,CAGuF,QAAQ,CAG3F,EAAoB,EAAU,QAAQ,CAAE,WAAY,CAAC,EAAwB,GAAO,CACtF,EAAkB,OAAS,IAC7B,EAAO,KACL,EAAO,QAAQ,aAAa,EAAkB,OAAO,oCAAoC,CAC1F,CAID,MAAM,EADU,EAAM,EAAmB,GAClB,CAAE,KAAO,IAA0B,CACxD,GAAM,CACJ,gBAAiB,CAAE,cACjB,MAAM,EAMP,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,EAAsB,IAAK,IAAW,CAC3C,KAAM,EAAM,eAAiB,EAAM,gBACnC,MAAO,EAAM,MACb,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,SAAU,EAAM,aACjB,EAAE,CACJ,CACD,SACD,CAAC,CAGF,EAAU,QAAS,GAAS,CAC1B,EAAwB,EAAK,OAAS,GACtC,EACF,CAEF,EAAO,KAAK,EAAO,MAAM,yBAAyB,EAAkB,OAAO,eAAe,CAAC,EAI7F,IAAM,EAAiB,EAAM,EAAW,GAAkB,CAC1D,MAAM,EAAU,EAAgB,MAAO,EAAqB,IAAQ,CAClE,EAAO,KACL,EAAO,QACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,aACzC,EAAoB,OACrB,cACF,CACF,CACD,MAAM,EAMH,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,CACL,UAAW,EAAoB,IAAK,IAAW,CAC7C,GAAI,EAAwB,EAAM,OAAO,GACzC,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,IAAK,EAAM,IACX,QAAS,EAAM,QACf,YAAa,EAAM,YACnB,YAAa,EAAM,iBACnB,OAAQ,CAAC,EAAM,SACf,YAAa,EAAM,OACnB,UAAW,EAAM,MAGjB,oBAAqB,EAAM,yBAA2B,IAAA,GAAY,CAAC,CAAE,OAAQ,EAAE,CAAE,CAAC,CAClF,SAAU,EAAM,iBACZ,EAAe,EAAM,kBAAkB,GACvC,IAAA,GACJ,wBAAyB,EAAM,iBAC3B,EAA8B,EAAM,iBAAkB,EAAmB,CACzE,IAAA,GACJ,WAAY,EAAM,WAClB,qBAAsB,EAAM,qBAC5B,SAAU,EAAM,aAEhB,mBAAoB,EAAM,oBAAoB,wBAC9C,gCAAiC,EAAM,oBAAoB,kBAC3D,2BAA4B,EAAM,oBAAoB,aACtD,+CACE,EAAM,oBAAoB,kCAC5B,qCAAsC,EAAM,oBAAoB,wBAChE,yBAA0B,EAAM,oBAAoB,+BACrD,EAAE,CACJ,CACF,CACD,SACD,CAAC,CACF,EAAO,KACL,EAAO,MACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,YACzC,EAAoB,OACrB,eACF,CACF,EACD,CAKF,IAAM,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAA0B,EAAU,QAAQ,CAAE,aAAa,EAAE,IAAO,EAAW,OAAS,EAAE,CAC1F,EAAkB,EACrB,KAAK,CAAE,aAAa,EAAE,IAAO,EAAW,OAAO,CAC/C,QAAQ,EAAK,IAAU,EAAM,EAAO,EAAE,CACzC,EAAO,KACL,EAAO,QACL,YAAY,EAAgB,qBAAqB,EAAwB,OAAO,iBACjF,CACF,CACD,EAAY,MAAM,EAAiB,EAAE,CACrC,IAAI,EAAQ,EAEZ,MAAM,EACJ,EACA,MAAO,CAAE,aAAY,WAAY,CAC3B,GACF,MAAM,EAAU,EAAY,KAAO,IAAc,CAC/C,IAAM,EAAS,EAAU,OACrB,EAAU,OAAO,KACd,CAAE,MAAK,cAAa,aAAY,WAAU,aAAY,GAAG,MAEvD,CACC,KAAM,EACN,cACA,WAAa,EAET,EAAW,IAAK,IAAc,CAC5B,GAAG,EACH,KAAM,EAAS,MAAQ,QACxB,EAAE,CAJH,IAAA,GAKJ,SAAW,EAEP,EAAS,IAAK,IAAa,CACzB,GAAG,EACH,KAAM,EAAQ,MAAQ,QACvB,EAAE,CAJH,IAAA,GAKJ,aACA,+BAAgC,EAAK,qCACrC,+BAAgC,EAAK,qCACtC,EACJ,CACD,IAAA,GAEE,EAAU,CACd,WAAY,EAAwB,GAAO,GAC3C,KAAM,EAAU,KAChB,KAAM,EAAU,IAChB,MAAO,EAAU,MACjB,YAAa,EAAU,YACvB,GAAI,EAAU,OACV,CACE,YAAa,EAAU,OACxB,CACD,EAAE,CACN,GAAI,EAAU,MACV,CACE,UAAW,EAAU,MACtB,CACD,EAAE,CACN,GAAI,EAAU,uBACV,CAAE,kBAAmB,EAAU,uBAAwB,CACvD,EAAE,CACN,iBAAmB,EAAU,0BAEzB,OAAO,QAAQ,EAAU,0BAA0B,CAAC,KAAK,CAAC,EAAK,MAAY,CACzE,YAAa,EACb,eAAgB,EACjB,EAAE,CAJH,IAAA,GAKJ,eAAgB,EAAU,oBAAsB,EAAE,CAClD,cAAe,EAChB,CAGK,GAAyB,EAAQ,eAAiB,EAAE,EAAE,KAAK,CAAE,UAAW,EAAK,CAC7E,EAAsB,EAAsB,QAC/C,EAAM,IAAU,EAAsB,QAAQ,EAAK,GAAK,EAC1D,CACD,GAAI,EAAoB,OAAS,EAC/B,EAAO,KACL,EAAO,IACL,8BACE,EAAU,IACX,6DAA6D,EAAoB,KAChF;EACD,GACF,CACF,CACD,EAAmB,QAEnB,GAAI,CACF,MAAM,EAAmB,EAAQ,EAA6B,CAC5D,UAAW,EACX,SACD,CAAC,OACK,EAAK,CACZ,EAAO,KACL,EAAO,IACL,iCAAiC,EAAU,IAAI,mBAAmB,EAAM,SAAS,EAAI,UACtF,CACF,CACD,EAAmB,GAGvB,GAAS,EACT,EAAY,OAAO,EAAM,EACzB,EAGN,CACE,YAAa,GACd,CACF,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EASvB,OAPA,EAAO,KACL,EAAO,MACL,WAAW,EAAU,OAAO,oBAAoB,EAAgB,mBAC9D,EAAY,IACb,YACF,CACF,CACM,CACL,QAAS,CAAC,EACV,kBAAmB,EAAM,GAA0B,CAAE,QAAS,EAAG,CAClE,CCtPH,eAAsB,EACpB,EACA,EACA,CACE,WAAW,GAEX,yBAAyB,GACzB,kBAAkB,GAClB,6BAA6B,GAC7B,OAAQ,EACR,WAAW,EAAE,EAeM,CACrB,IAAM,EAAe,GAAc,EAC7B,EAAsB,EAAE,CAC1B,EAAmB,GAEjB,GAAe,EAAkB,EAAiB,IAAwB,CAC9E,EAAO,KAAK,CAAE,WAAU,OAAM,UAAS,CAAC,CACxC,EAAmB,IAGrB,EAAa,KAAK,EAAO,QAAQ,gCAAgC,EAAS,KAAK,CAAC,CAEhF,GAAM,CACJ,YACA,aACA,UACA,cACA,gBAAiB,EACjB,oBAAqB,EACrB,YACA,UACA,kBAAmB,EACnB,aAAc,EACd,aAAc,EACd,UACA,kBAAmB,EACnB,wBAAyB,EACzB,sBAAuB,EACvB,eAAgB,EAChB,0BAA2B,EAC3B,QACA,iBAAkB,EAClB,WACA,WACA,aACA,8BAA+B,GAC/B,mBAAoB,GAClB,EAEE,GAAoB,EAAM,sBAC1B,EAAW,EAAM,SAEjB,GAAuB,GAAkC,EAAM,CACrE,GAAI,GAAqB,OAAS,EAAG,CACnC,IAAK,IAAM,KAAW,GACpB,EAAY,mBAAoB,EAAQ,CAE1C,MAAO,CACL,QAAS,GACT,SACA,SAAU,EAAS,OAAS,EAAI,EAAW,IAAA,GAC5C,CAGH,GAAM,CAAC,EAAkB,EAAoB,IAAkB,MAAM,QAAQ,IAAI,CAE/E,GAAa,EACT,EAAiC,EAAQ,CACvC,QACA,YAAa,CAAC,EACd,SACD,CAAC,CACD,EAAE,CAEP,GAAa,GAAgB,GAAa,EACtC,EAA2B,EAAO,EAAO,CACzC,EAAE,CAEN,GACA,EACG,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,CAClD,QAAQ,EAAK,IAAQ,EAAM,EAAI,OAAQ,EAAE,CAAG,EAC3C,EAAa,EAAQ,CAAE,aAAc,EAAO,SAAQ,CAAC,CACrD,EAAE,CACP,CAAC,CAqCF,GAnCI,IAA2B,SASxB,MAR0C,GAC7C,EACA,GACA,CACE,OAAQ,EACR,WACD,CACF,EAEC,EAAY,8BAA+B,6CAA6C,EAIxF,GAAiB,SAId,MAHgC,GAAoB,EAAQ,EAAiB,CAChF,OAAQ,EACT,CAAC,EAEA,EAAY,mBAAoB,kCAAkC,EAIlE,IAAmB,SAIhB,MAHkC,GAA2B,EAAQ,GAAmB,CAC3F,OAAQ,EACT,CAAC,EAEA,EAAY,qBAAsB,0CAA0C,EAO5E,GAAU,OAAQ,CACpB,GAAM,CAAE,QAAS,EAAiB,2BAA4B,MAAM,GAClE,EACA,EACA,CAAE,OAAQ,EAAc,CACzB,CACI,GACH,EAAY,WAAY,sCAAsC,CAGhE,IAAM,EAAS,EAAS,QAAS,IAC9B,EAAQ,sBAAwB,EAAE,EAAE,IAAK,IAAW,CACnD,GAAG,EACH,gBAAiB,EAAQ,aAC1B,EAAE,CACJ,CACG,EAAO,OAAS,IAKb,MAJuB,GAAqB,EAAQ,EAAQ,CAC/D,OAAQ,EACR,0BACD,CAAC,EAEA,EAAY,oBAAqB,+CAA+C,EAMtF,GAAI,EAAgB,CAClB,EAAa,KAAK,EAAO,QAAQ,6BAA6B,CAAC,CAC/D,GAAI,CACF,MAAM,GAAmB,EAAQ,EAAgB,CAAE,OAAQ,EAAc,CAAC,CAC1E,EAAa,KAAK,EAAO,MAAM,uCAAuC,CAAC,OAChE,EAAK,CACZ,EAAY,kBAAoB,EAAc,QAAQ,CACtD,EAAa,MAAM,EAAO,IAAI,qCAAsC,EAAc,UAAU,CAAC,EAIjG,GAAI,EAAO,CACT,IAAM,EAAe,MAAM,GAAU,EAAQ,EAAO,CAAE,SAAQ,CAAC,CAC/D,IAAuC,CAAC,EA0B1C,GAtBI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,sBAAsB,CAAC,CAC/E,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAChD,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,oBAAoB,CAAC,EAIxE,EAAkB,CACpB,IAAM,EAAwB,MAAM,EAAqB,EAAQ,EAAkB,CAAE,SAAQ,CAAC,CAC9F,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAoB,CACtB,IAAM,EAA4B,MAAM,GAAuB,EAAQ,EAAoB,CACzF,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,GAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAuB,CACzB,IAAM,EAA+B,MAAM,EACzC,EACA,EACA,CAAE,SAAQ,CACX,CACD,IAAuC,CAAC,EAI1C,GAAI,EAAY,CAEd,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAW,OAAO,iBAAiB,CAAC,CAC3E,IAAM,EAAqB,MAAM,EAAmB,EAAQ,CAAE,SAAQ,CAAC,CACvE,MAAM,EACJ,EACA,KAAO,IAAc,CACnB,IAAM,EAAW,EAAmB,KAAM,GAAS,EAAK,OAAS,EAAU,KAAK,CAEhF,EAAO,KAAK,EAAO,QAAQ,sBAAsB,EAAU,KAAK,MAAM,CAAC,CACvE,GAAI,CACF,MAAM,EAAc,EAAQ,EAAW,CACrC,kBAAmB,EACnB,6BACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,kCAAkC,EAAU,KAAK,IAAI,CAAC,OACxE,EAAK,CACZ,EAAmB,GACnB,EAAO,MACL,EAAO,IAAI,6BAA6B,EAAU,KAAK,OAAO,EAAI,UAAU,CAC7E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAW,OAAO,eAAe,CAAC,CAIxE,GAAI,EAAa,CACf,IAAM,EAAqB,MAAM,EAAgB,EAAQ,EAAa,CAAE,SAAQ,CAAC,CACjF,IAAuC,CAAC,EAqE1C,GAjEI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,gBAAgB,CAAC,CACzE,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,CACzB,MAAO,EACP,mBACA,qBACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,cAAc,CAAC,EAIlE,IAEF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAY,OAAO,kBAAkB,CAAC,CAC7E,MAAM,EACJ,EACA,KAAO,IAAe,CACpB,IAAM,EAAW,EAAiB,EAAW,MAC7C,GAAI,CAAC,EACH,MAAU,MACR,wCAAwC,EAAW,KAAK,yCACzD,CAGH,EAAO,KAAK,EAAO,QAAQ,uBAAuB,EAAW,KAAK,MAAM,CAAC,CACzE,GAAI,CACF,MAAM,GAAe,EAAQ,CAC3B,MAAO,EACP,qBACA,aAAc,EAAS,GACvB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,mCAAmC,EAAW,KAAK,IAAI,CAAC,OAC1E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,8BAA8B,EAAW,KAAK,OAAO,EAAI,UAAU,CAC/E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAY,OAAO,gBAAgB,CAAC,EAItE,EAAS,CAEX,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAQ,OAAO,cAAc,CAAC,CACrE,IAAM,EAAkB,MAAM,EAAgB,EAAQ,CAAE,SAAQ,CAAC,CACjE,MAAM,EACJ,EACA,KAAO,IAAW,CAChB,IAAM,EAAW,EAAgB,KAAM,GAAQ,EAAI,OAAS,EAAO,KAAK,CACxE,GAAI,CAAC,EACH,MAAU,MACR,oCAAoC,EAAO,KAAK,qDACjD,CAGH,EAAO,KAAK,EAAO,QAAQ,mBAAmB,EAAO,KAAK,MAAM,CAAC,CACjE,GAAI,CACF,MAAM,EACJ,EACA,CACE,SACA,SAAU,EAAS,GACnB,YAAa,CAAC,EACf,CACD,CAAE,SAAQ,CACX,CACD,EAAO,KAAK,EAAO,MAAM,+BAA+B,EAAO,KAAK,IAAI,CAAC,OAClE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,0BAA0B,EAAO,KAAK,OAAO,EAAI,UAAU,CAAC,GAGxF,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAQ,OAAO,YAAY,CAAC,CAIlE,GAAI,EAAc,CAEhB,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAa,OAAO,oBAAoB,CAAC,CAChF,IAAM,EAAuB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CAC3E,MAAM,EACJ,EACA,KAAO,IAAgB,CACrB,IAAM,EAAW,EAAqB,KAAM,GAAS,EAAK,OAAS,EAAY,KAAK,CACpF,GAAI,CAAC,EACH,MAAU,MACR,0CAA0C,EAAY,KAAK,qDAC5D,CAGH,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAY,KAAK,MAAM,CAAC,CAC5E,GAAI,CACF,MAAM,GAAgB,EAAQ,CAC5B,MAAO,EACP,cAAe,EAAS,GACxB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,qCAAqC,EAAY,KAAK,IAAI,CAAC,OAC7E,EAAK,CACZ,EAAmB,GAGnB,IAAM,EACJ,EAAY,0BAA4B,IAAA,IACxC,oBAAoB,KAAK,EAAI,QAAQ,CACjC,yGACA,GACN,EAAO,KACL,EAAO,IACL,gCAAgC,EAAY,KAAK,OAAO,EAAI,UAAU,IACvE,CACF,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAa,OAAO,kBAAkB,CAAC,CAI7E,GAAI,EAAW,CACb,IAAM,EAAkB,MAAM,GAAc,EAAQ,EAAW,CAAE,kBAAiB,SAAQ,CAAC,CAC3F,IAAuC,CAAC,EAI1C,GAAI,EACF,GAAI,CAKG,MAJ8B,GAAkB,EAAQ,EAAe,CAC1E,SACA,YAAa,CAAC,EACf,CAAC,EAEA,EAAY,iBAAkB,gCAAgC,OAEzD,EAAK,CACZ,EAAY,iBAAmB,EAAc,QAAQ,CAKzD,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAiB,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAC5E,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CACxE,IAAuC,CAAC,EAI1C,IAAM,EAA2D,EAAE,CAEnE,GAAI,EAAW,CACb,GAAM,CAAE,UAAS,qBAAsB,MAAM,EAAc,EAAW,EAAQ,CAC5E,qBACA,eAAgB,GAChB,WACD,CAAC,CACF,GAAW,QAAS,GAAa,CAE/B,IAAM,EAAe,EAAS,yBAC9B,GAAI,EACF,GAAI,CACF,EAAkB,KAAK,CACrB,EAAkB,EAAS,OAC3B,GAA8B,EAAc,EAAS,MAAM,CAC5D,CAAC,OACK,EAAK,CACZ,EAAY,aAAe,EAAc,QAAS,EAAS,MAAM,CACjE,EAAa,MAAM,EAAO,IAAK,EAAc,QAAQ,CAAC,GAG1D,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAkB,OAAS,EAAG,CAChC,IAAM,EAAsB,MAAM,GAAyB,EAAQ,CACjE,MAAO,EACP,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAsB,CACxB,IAAM,EAA4B,MAAM,GAAyB,EAAQ,EAAsB,CAC7F,SACD,CAAC,CACF,IAAuC,CAAC,EAO1C,MAAO,CACL,QAAS,CAAC,EACV,SACA,SAAU,EAAS,OAAS,EAAI,EAAW,IAAA,GAC5C,CC3jBH,SAAgB,EACd,EACA,GAAG,EACa,CAEhB,IAAM,EAAc,KAAK,MAAM,KAAK,UAAU,EAAK,CAAC,CAapD,OAZA,EAAO,QAAS,GAAU,CAExB,EAAW,EAAM,CAAC,SAAS,CAAC,EAAK,KAAuB,CAClD,EAAO,KAAS,IAAA,GAClB,EAAO,GAAO,EACL,MAAM,QAAQ,EAAM,CAC7B,EAAO,GAAO,CAAC,GAAG,EAAO,GAAM,GAAG,EAAM,CAExC,EAAO,GAAO,GAEhB,EACF,CACK"}
1	+ {"version":3,"file":"mergeTranscendInputs-CkpitZbH.mjs","names":[],"sources":["../src/lib/preference-management/validatePreferenceManagementSlugs.ts","../src/lib/graphql/ensureAllDataSubjectsExist.ts","../src/lib/graphql/normalizeDeletionDependencies.ts","../src/lib/graphql/syncDataSilos.ts","../src/lib/graphql/syncConfigurationToTranscend.ts","../src/lib/mergeTranscendInputs.ts"],"sourcesContent":["import type { TranscendInput } from '../../codecs.js';\n\n/** Slugs for preference topics and option values must be alphabetical letters only. /\nexport const PREFERENCE_SLUG_REGEX = /^[A-Za-z]+$/;\n\n/\n Validate preference topic and option value slugs before push.\n \n @param input - Parsed transcend.yml input\n * @returns List of validation error messages (empty when valid)\n /\nexport function validatePreferenceManagementSlugs(input: TranscendInput): string[] {\n const errors: string[] = [];\n\n const validateSlug = (slug: string, context: string): void => {\n if (!PREFERENCE_SLUG_REGEX.test(slug)) {\n errors.push(\n `${context}: slug \"${slug}\" is invalid — slugs must contain only alphabetical letters (A-Z, a-z). ` +\n 'Topic slugs are normalized to PascalCase server-side, so YAML slugs should already be PascalCase ' +\n 'or matching will break on subsequent pulls.',\n );\n }\n };\n\n for (const option of input['preference-options'] ?? []) {\n validateSlug(option.slug, 'preference-options');\n }\n\n for (const purpose of input.purposes ?? []) {\n for (const topic of purpose['preference-topics'] ?? []) {\n validateSlug(topic.slug, `purposes (topic \"${topic.title}\")`);\n for (const option of topic.options ?? []) {\n validateSlug(option.slug, `purposes (topic \"${topic.title}\" option \"${option.title}\")`);\n }\n }\n }\n\n return errors;\n}\n","import { createDataSubject, fetchAllDataSubjects, type DataSubject } from '@transcend-io/sdk';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { keyBy, flatten, uniq, difference } from 'lodash-es';\n\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\n/\n Fetch all of the data subjects in the organization\n \n @param input - Input to fetch\n * @param client - GraphQL client\n * @param fetchAll - When true, always fetch all subjects\n * @returns The list of data subjects\n /\nexport async function ensureAllDataSubjectsExist(\n {\n 'data-silos': dataSilos = [],\n 'data-subjects': dataSubjects = [],\n 'processing-activities': processingActivities = [],\n enrichers = [],\n }: TranscendInput,\n client: GraphQLClient,\n fetchAll = false,\n): Promise<{ [type in string]: DataSubject }> {\n const expectedDataSubjects = uniq([\n ...flatten(dataSilos.map((silo) => silo['data-subjects'] \|\| []) \|\| []),\n ...flatten(processingActivities.map(({ dataSubjectTypes }) => dataSubjectTypes ?? []) ?? []),\n ...flatten(enrichers.map((enricher) => enricher['data-subjects'] \|\| []) \|\| []),\n ...dataSubjects.map((subject) => subject.type),\n ]);\n if (expectedDataSubjects.length === 0 && !fetchAll) {\n return {};\n }\n\n const internalSubjects = await fetchAllDataSubjects(client, { logger });\n const dataSubjectByName = keyBy(internalSubjects, 'type');\n\n const missingDataSubjects = difference(\n expectedDataSubjects,\n internalSubjects.map(({ type }) => type),\n );\n\n if (missingDataSubjects.length > 0) {\n logger.info(colors.magenta(`Creating ${missingDataSubjects.length} new data subjects...`));\n for (const dataSubjectType of missingDataSubjects) {\n logger.info(colors.magenta(`Creating data subject ${dataSubjectType}...`));\n const created = await createDataSubject(client, { input: dataSubjectType, logger });\n logger.info(colors.green(`Created data subject ${dataSubjectType}!`));\n dataSubjectByName[dataSubjectType] = created;\n }\n }\n\n return dataSubjectByName;\n}\n","import type { DependedOnDataSiloInput } from '@transcend-io/sdk';\nimport { uniq } from 'lodash-es';\n\nimport type { DeletionDependencies, DeletionDependencyInput } from '../../codecs.js';\n\n/\n Convert the `deletion-dependencies` entries for a single data silo into the\n * `dependedOnDataSilos` input accepted by the `updateDataSilos` mutation.\n \n A list of titles becomes a single global entry. A list of objects may declare\n * at most one global configuration (`{ titles }`) plus per-workflow overrides;\n * workflows that are absent from the list are left untouched.\n \n @param dependencies - The `deletion-dependencies` entries from transcend.yml\n * @param dataSiloTitle - Title of the data silo being synced, used in error messages\n * @returns The dependency entries to send to the API\n /\nexport function normalizeDeletionDependencies(\n dependencies: DeletionDependencies,\n dataSiloTitle: string,\n): DependedOnDataSiloInput[] {\n if (dependencies.length === 0) {\n return [];\n }\n\n // Legacy / no-overrides form: a flat list of data silo titles\n if (typeof dependencies[0] === 'string') {\n return [{ titles: uniq(dependencies as string[]) }];\n }\n\n let globalTitles: string[] \| undefined;\n const workflowOverrides: DependedOnDataSiloInput[] = [];\n const seenWorkflows = new Set<string>();\n\n /\n Record an override for a single workflow, rejecting duplicates before the API does\n * so that the error names the data silo\n \n @param workflow - Internal name of the workflow being overridden\n * @param override - Titles or reset flag to send for that workflow\n /\n const addWorkflowOverride = (\n workflow: string,\n override: Omit<DependedOnDataSiloInput, 'workflowConfigInternalName'>,\n ): void => {\n if (seenWorkflows.has(workflow)) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has multiple deletion-dependencies entries for workflow ` +\n `\"${workflow}\". Combine them into a single entry.`,\n );\n }\n seenWorkflows.add(workflow);\n workflowOverrides.push({ workflowConfigInternalName: workflow, ...override });\n };\n\n (dependencies as DeletionDependencyInput[]).forEach((dependency) => {\n // io-ts codecs allow extra properties, so this combination decodes cleanly\n // even though the two halves contradict each other\n if ('reset-to-global' in dependency && 'titles' in dependency) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has a deletion-dependencies entry for workflow ` +\n `\"${dependency.workflow}\" that sets both \"reset-to-global\" and \"titles\". ` +\n 'Use \"titles: []\" to override the global configuration with no dependencies, ' +\n 'or \"reset-to-global: true\" to fall back to the global configuration.',\n );\n }\n\n if ('reset-to-global' in dependency) {\n addWorkflowOverride(dependency.workflow, { resetToGlobal: true });\n return;\n }\n\n if (dependency.workflow) {\n addWorkflowOverride(dependency.workflow, { titles: dependency.titles });\n return;\n }\n\n if (globalTitles !== undefined) {\n throw new Error(\n `Data silo \"${dataSiloTitle}\" has multiple global deletion-dependencies entries. ` +\n 'Combine them into a single `{ titles: [...] }` entry.',\n );\n }\n globalTitles = dependency.titles;\n });\n\n return [\n ...(globalTitles !== undefined ? [{ titles: uniq(globalTitles) }] : []),\n ...workflowOverrides,\n ];\n}\n","import {\n makeGraphQLRequest,\n ApiKey,\n convertToDataSubjectBlockList,\n type DataSubject,\n fetchAllDataSilos,\n type DataSilo,\n CREATE_DATA_SILOS,\n UPDATE_DATA_SILOS,\n UPDATE_OR_CREATE_DATA_POINT,\n} from '@transcend-io/sdk';\nimport { apply } from '@transcend-io/type-utils';\nimport { mapSeries, map } from '@transcend-io/utils';\n/ eslint-disable max-lines /\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { chunk, keyBy } from 'lodash-es';\n\nimport { DataSiloInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nconst BATCH_SILOS_LIMIT = 20;\n\n/\n Sync a data silo configuration\n \n @param dataSilos - Data silos to sync\n * @param client - GraphQL client\n * @param options - Options\n * @returns Data silo info\n /\nexport async function syncDataSilos(\n dataSilos: DataSiloInput[],\n client: GraphQLClient,\n {\n pageSize,\n dataSubjectsByName,\n apiKeysByTitle,\n }: {\n /* Page size /\n pageSize: number;\n /* The data subjects in the organization /\n dataSubjectsByName: { [type in string]: DataSubject };\n /* API key title to API key /\n apiKeysByTitle: { [title in string]: ApiKey };\n },\n): Promise<{\n /* Whether successfully updated /\n success: boolean;\n /* A mapping between data silo title to data silo ID /\n dataSiloTitleToId: { [k in string]: string };\n}> {\n let encounteredError = false;\n\n // Time duration\n const t0 = new Date().getTime();\n logger.info(colors.magenta(`Syncing \"${dataSilos.length}\" data silos...`));\n\n // Determine the set of data silos that already exist\n const existingDataSilos = await fetchAllDataSilos(client, {\n titles: dataSilos.map(({ title }) => title),\n pageSize,\n logger,\n });\n\n // Create a mapping of title -> existing silo, if it exists\n const existingDataSiloByTitle = keyBy<Pick<DataSilo, 'id' \| 'title'>>(existingDataSilos, 'title');\n\n // Create new silos that do not exist\n const newDataSiloInputs = dataSilos.filter(({ title }) => !existingDataSiloByTitle[title]);\n if (newDataSiloInputs.length > 0) {\n logger.info(\n colors.magenta(`Creating \"${newDataSiloInputs.length}\" data silos that did not exist...`),\n );\n\n // Batch the creation\n const chunked = chunk(newDataSiloInputs, BATCH_SILOS_LIMIT);\n await mapSeries(chunked, async (dependencyUpdateChunk) => {\n const {\n createDataSilos: { dataSilos },\n } = await makeGraphQLRequest<{\n /* Mutation result /\n createDataSilos: {\n /* New data silos /\n dataSilos: Pick<DataSilo, 'id' \| 'title'>[];\n };\n }>(client, CREATE_DATA_SILOS, {\n variables: {\n input: dependencyUpdateChunk.map((input) => ({\n name: input['outer-type'] \|\| input.integrationName,\n title: input.title,\n country: input.country,\n countrySubDivision: input.countrySubDivision,\n sombraId: input['sombra-id'],\n })),\n },\n logger,\n });\n\n // save mapping of title and id\n dataSilos.forEach((silo) => {\n existingDataSiloByTitle[silo.title] = silo;\n });\n });\n\n logger.info(colors.green(`Successfully created \"${newDataSiloInputs.length}\" data silos!`));\n }\n\n // Batch the updates\n const chunkedUpdates = chunk(dataSilos, BATCH_SILOS_LIMIT);\n await mapSeries(chunkedUpdates, async (dataSiloUpdateChunk, ind) => {\n logger.info(\n colors.magenta(\n `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing \"${\n dataSiloUpdateChunk.length\n }\" data silos`,\n ),\n );\n await makeGraphQLRequest<{\n /* Mutation result /\n updateDataSilos: {\n /* New data silos /\n dataSilos: Pick<DataSilo, 'id' \| 'title'>[];\n };\n }>(client, UPDATE_DATA_SILOS, {\n variables: {\n input: {\n dataSilos: dataSiloUpdateChunk.map((input) => ({\n id: existingDataSiloByTitle[input.title].id,\n country: input.country,\n countrySubDivision: input.countrySubDivision,\n url: input.url,\n headers: input.headers,\n description: input.description,\n identifiers: input['identity-keys'],\n isLive: !input.disabled,\n ownerEmails: input.owners,\n teamNames: input.teams,\n // clear out the global config if not specified, otherwise the update needs to be\n // applied after all data silos are created\n dependedOnDataSilos: input['deletion-dependencies'] ? undefined : [{ titles: [] }],\n apiKeyId: input['api-key-title']\n ? apiKeysByTitle[input['api-key-title']].id\n : undefined,\n dataSubjectBlockListIds: input['data-subjects']\n ? convertToDataSubjectBlockList(input['data-subjects'], dataSubjectsByName)\n : undefined,\n attributes: input.attributes,\n businessEntityTitles: input.businessEntityTitles,\n sombraId: input['sombra-id'],\n // AVC settings\n notifyEmailAddress: input['email-settings']?.['notify-email-address'],\n promptAVendorEmailSendFrequency: input['email-settings']?.['send-frequency'],\n promptAVendorEmailSendType: input['email-settings']?.['send-type'],\n promptAVendorEmailIncludeIdentifiersAttachment:\n input['email-settings']?.['include-identifiers-attachment'],\n promptAVendorEmailCompletionLinkType: input['email-settings']?.['completion-link-type'],\n manualWorkRetryFrequency: input['email-settings']?.['manual-work-retry-frequency'],\n })),\n },\n },\n logger,\n });\n logger.info(\n colors.green(\n `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced \"${\n dataSiloUpdateChunk.length\n }\" data silos!`,\n ),\n );\n });\n\n // Sync datapoints\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n const dataSilosWithDataPoints = dataSilos.filter(({ datapoints = [] }) => datapoints.length > 0);\n const totalDataPoints = dataSilos\n .map(({ datapoints = [] }) => datapoints.length)\n .reduce((acc, count) => acc + count, 0);\n logger.info(\n colors.magenta(\n `Syncing \"${totalDataPoints}\" datapoints from \"${dataSilosWithDataPoints.length}\" data silos...`,\n ),\n );\n progressBar.start(totalDataPoints, 0);\n let total = 0;\n\n await map(\n dataSilosWithDataPoints,\n async ({ datapoints, title }) => {\n if (datapoints) {\n await mapSeries(datapoints, async (datapoint) => {\n const fields = datapoint.fields\n ? datapoint.fields.map(\n ({ key, description, categories, purposes, attributes, ...rest }) =>\n // TODO: Support setting title separately from the 'key/name'\n ({\n name: key,\n description,\n categories: !categories\n ? undefined\n : categories.map((category) => ({\n ...category,\n name: category.name \|\| 'Other',\n })),\n purposes: !purposes\n ? undefined\n : purposes.map((purpose) => ({\n ...purpose,\n name: purpose.name \|\| 'Other',\n })),\n attributes,\n accessRequestVisibilityEnabled: rest['access-request-visibility-enabled'],\n erasureRequestRedactionEnabled: rest['erasure-request-redaction-enabled'],\n }),\n )\n : undefined;\n\n const payload = {\n dataSiloId: existingDataSiloByTitle[title].id,\n path: datapoint.path,\n name: datapoint.key,\n title: datapoint.title,\n description: datapoint.description,\n ...(datapoint.owners\n ? {\n ownerEmails: datapoint.owners,\n }\n : {}),\n ...(datapoint.teams\n ? {\n teamNames: datapoint.teams,\n }\n : {}),\n ...(datapoint['data-collection-tag']\n ? { dataCollectionTag: datapoint['data-collection-tag'] }\n : {}),\n querySuggestions: !datapoint['privacy-action-queries']\n ? undefined\n : Object.entries(datapoint['privacy-action-queries']).map(([key, value]) => ({\n requestType: key,\n suggestedQuery: value,\n })),\n enabledActions: datapoint['privacy-actions'] \|\| [], // clear out when not specified\n subDataPoints: fields,\n };\n\n // Ensure no duplicate sub-datapoints are provided\n const subDataPointsToUpdate = (payload.subDataPoints \|\| []).map(({ name }) => name);\n const duplicateDataPoints = subDataPointsToUpdate.filter(\n (name, index) => subDataPointsToUpdate.indexOf(name) !== index,\n );\n if (duplicateDataPoints.length > 0) {\n logger.info(\n colors.red(\n `\\nCannot update datapoint \"${\n datapoint.key\n }\" as it has duplicate sub-datapoints with the same name: \\n${duplicateDataPoints.join(\n '\\n',\n )}`,\n ),\n );\n encounteredError = true;\n } else {\n try {\n await makeGraphQLRequest(client, UPDATE_OR_CREATE_DATA_POINT, {\n variables: payload,\n logger,\n });\n } catch (err) {\n logger.info(\n colors.red(\n `\\nFailed to update datapoint \"${datapoint.key}\" for data silo \"${title}\"! - \\n${err.message}`,\n ),\n );\n encounteredError = true;\n }\n }\n total += 1;\n progressBar.update(total);\n });\n }\n },\n {\n concurrency: 10,\n },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Synced \"${dataSilos.length}\" data silos and \"${totalDataPoints}\" datapoints in \"${\n totalTime / 1000\n }\" seconds!`,\n ),\n );\n return {\n success: !encounteredError,\n dataSiloTitleToId: apply(existingDataSiloByTitle, ({ id }) => id),\n };\n}\n/ eslint-enable max-lines /\n","import {\n fetchAllActions,\n fetchAllAttributes,\n fetchAllDataSubjects,\n fetchApiKeys,\n fetchIdentifiersAndCreateMissing,\n syncAction,\n syncActionItemCollections,\n syncActionItems,\n syncAttribute,\n syncBusinessEntities,\n syncDataCategories,\n syncConsentManager,\n syncCookies,\n syncDataFlows,\n syncDataSubject,\n syncDataSiloDependencies,\n syncEnricher,\n syncIdentifier,\n syncIntlMessages,\n syncPartitions,\n syncPolicies,\n syncPrivacyCenter,\n syncProcessingActivities,\n syncProcessingPurposes,\n syncConsentWorkflowTriggers,\n syncWorkflowConfigs,\n syncPreferenceOptionValues,\n syncPurposes,\n syncPreferenceTopics,\n syncTeams,\n syncTemplate,\n syncVendors,\n type DependedOnDataSiloInput,\n type Identifier,\n} from '@transcend-io/sdk';\nimport { map, type Logger, type SyncError, type SyncResult } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\n\n/ eslint-disable max-lines /\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { validatePreferenceManagementSlugs } from '../preference-management/validatePreferenceManagementSlugs.js';\nimport { ensureAllDataSubjectsExist } from './ensureAllDataSubjectsExist.js';\nimport { normalizeDeletionDependencies } from './normalizeDeletionDependencies.js';\nimport { syncDataSilos } from './syncDataSilos.js';\n\nconst CONCURRENCY = 10;\n\n/\n Sync the yaml input back to Transcend using the GraphQL APIs\n \n @param input - The yml input\n * @param client - GraphQL client\n * @param pageSize - Page size\n * @returns Structured sync result with per-resource errors\n /\nexport async function syncConfigurationToTranscend(\n input: TranscendInput,\n client: GraphQLClient,\n {\n pageSize = 50,\n // TODO: https://transcend.height.app/T-23779\n publishToPrivacyCenter = true,\n classifyService = false,\n deleteExtraAttributeValues = false,\n logger: syncLogger,\n warnings = [],\n }: {\n /* Page size /\n pageSize?: number;\n /* When true, skip publishing to privacy center /\n publishToPrivacyCenter?: boolean;\n /* When true, delete any attributes being synced up /\n deleteExtraAttributeValues?: boolean;\n /* classify data flow service if missing /\n classifyService?: boolean;\n /* Optional logger (e.g. collecting logger for MCP debug responses) /\n logger?: Logger;\n /* Non-fatal warnings to include in the result /\n warnings?: string[];\n },\n): Promise<SyncResult> {\n const activeLogger = syncLogger ?? logger;\n const errors: SyncError[] = [];\n let encounteredError = false;\n\n const recordError = (resource: string, message: string, item?: string): void => {\n errors.push({ resource, item, message });\n encounteredError = true;\n };\n\n activeLogger.info(colors.magenta(`Fetching data with page size ${pageSize}...`));\n\n const {\n templates,\n attributes,\n actions,\n identifiers,\n 'data-subjects': dataSubjects,\n 'business-entities': businessEntities,\n enrichers,\n cookies,\n 'consent-manager': consentManager,\n 'data-silos': dataSilos,\n 'data-flows': dataFlows,\n vendors,\n 'data-categories': dataCategories,\n 'processing-activities': processingActivities,\n 'processing-purposes': processingPurposes,\n 'action-items': actionItems,\n 'action-item-collections': actionItemCollections,\n teams,\n 'privacy-center': privacyCenter,\n messages,\n policies,\n partitions,\n 'preference-workflow-configs': preferenceWorkflowConfigs,\n 'workflow-configs': workflowConfigs,\n } = input;\n\n const preferenceOptions = input['preference-options'];\n const purposes = input.purposes;\n\n const preferenceSlugErrors = validatePreferenceManagementSlugs(input);\n if (preferenceSlugErrors.length > 0) {\n for (const message of preferenceSlugErrors) {\n recordError('preference-slugs', message);\n }\n return {\n success: false,\n errors,\n warnings: warnings.length > 0 ? warnings : undefined,\n };\n }\n\n const [identifierByName, dataSubjectsByName, apiKeyTitleMap] = await Promise.all([\n // Ensure all identifiers are created and create a map from name -> identifier.id\n enrichers \|\| identifiers\n ? fetchIdentifiersAndCreateMissing(client, {\n input,\n skipPublish: !publishToPrivacyCenter,\n logger,\n })\n : ({} as { [k in string]: Identifier }),\n // Grab all data subjects in the organization\n dataSilos \|\| dataSubjects \|\| enrichers \|\| processingActivities\n ? ensureAllDataSubjectsExist(input, client)\n : {},\n // Grab API keys\n dataSilos &&\n dataSilos\n .map((dataSilo) => dataSilo['api-key-title'] \|\| [])\n .reduce((acc, lst) => acc + lst.length, 0) > 0\n ? fetchApiKeys(client, { apiKeyInputs: input, logger })\n : {},\n ]);\n\n if (preferenceWorkflowConfigs?.length) {\n const preferenceWorkflowConfigsSuccess = await syncConsentWorkflowTriggers(\n client,\n preferenceWorkflowConfigs,\n {\n logger: activeLogger,\n pageSize,\n },\n );\n if (!preferenceWorkflowConfigsSuccess) {\n recordError('preference-workflow-configs', 'Failed to sync preference workflow configs');\n }\n }\n\n if (workflowConfigs?.length) {\n const workflowConfigsSuccess = await syncWorkflowConfigs(client, workflowConfigs, {\n logger: activeLogger,\n });\n if (!workflowConfigsSuccess) {\n recordError('workflow-configs', 'Failed to sync workflow configs');\n }\n }\n\n if (preferenceOptions?.length) {\n const preferenceOptionsSuccess = await syncPreferenceOptionValues(client, preferenceOptions, {\n logger: activeLogger,\n });\n if (!preferenceOptionsSuccess) {\n recordError('preference-options', 'Failed to sync preference option values');\n }\n }\n\n // Sync consent purposes and their nested preference topics.\n // Order matters: option values (above) exist first, then purposes, then topics\n // (which link to their purpose by ID and to option values by slug).\n if (purposes?.length) {\n const { success: purposesSuccess, purposeIdByTrackingType } = await syncPurposes(\n client,\n purposes,\n { logger: activeLogger },\n );\n if (!purposesSuccess) {\n recordError('purposes', 'Failed to sync one or more purposes');\n }\n\n const topics = purposes.flatMap((purpose) =>\n (purpose['preference-topics'] ?? []).map((topic) => ({\n ...topic,\n 'tracking-type': purpose.trackingType,\n })),\n );\n if (topics.length > 0) {\n const topicsSuccess = await syncPreferenceTopics(client, topics, {\n logger: activeLogger,\n purposeIdByTrackingType,\n });\n if (!topicsSuccess) {\n recordError('preference-topics', 'Failed to sync one or more preference topics');\n }\n }\n }\n\n // Sync consent manager\n if (consentManager) {\n activeLogger.info(colors.magenta('Syncing consent manager...'));\n try {\n await syncConsentManager(client, consentManager, { logger: activeLogger });\n activeLogger.info(colors.green('Successfully synced consent manager!'));\n } catch (err) {\n recordError('consent-manager', (err as Error).message);\n activeLogger.error(colors.red(`Failed to sync consent manager! - ${(err as Error).message}`));\n }\n }\n\n if (teams) {\n const teamsSuccess = await syncTeams(client, teams, { logger });\n encounteredError = encounteredError \|\| !teamsSuccess;\n }\n\n // Sync email templates\n if (templates) {\n logger.info(colors.magenta(`Syncing \"${templates.length}\" email templates...`));\n await map(\n templates,\n async (template) => {\n logger.info(colors.magenta(`Syncing template \"${template.title}\"...`));\n try {\n await syncTemplate(client, template, { logger });\n logger.info(colors.green(`Successfully synced template \"${template.title}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync template \"${template.title}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${templates.length}\" email templates!`));\n }\n\n // Sync business entities\n if (businessEntities) {\n const businessEntitySuccess = await syncBusinessEntities(client, businessEntities, { logger });\n encounteredError = encounteredError \|\| !businessEntitySuccess;\n }\n\n // Sync vendors\n if (vendors) {\n const vendorsSuccess = await syncVendors(client, vendors, { logger });\n encounteredError = encounteredError \|\| !vendorsSuccess;\n }\n\n // Sync data categories\n if (dataCategories) {\n const dataCategoriesSuccess = await syncDataCategories(client, dataCategories, { logger });\n encounteredError = encounteredError \|\| !dataCategoriesSuccess;\n }\n\n // Sync processing purposes\n if (processingPurposes) {\n const processingPurposesSuccess = await syncProcessingPurposes(client, processingPurposes, {\n logger,\n });\n encounteredError = encounteredError \|\| !processingPurposesSuccess;\n }\n\n // Sync partitions\n if (partitions) {\n const partitionsSuccess = await syncPartitions(client, partitions, { logger });\n encounteredError = encounteredError \|\| !partitionsSuccess;\n }\n\n // Sync cookies\n if (cookies) {\n const cookiesSuccess = await syncCookies(client, cookies, { logger });\n encounteredError = encounteredError \|\| !cookiesSuccess;\n }\n\n // Sync action item collections\n if (actionItemCollections) {\n const actionItemCollectionsSuccess = await syncActionItemCollections(\n client,\n actionItemCollections,\n { logger },\n );\n encounteredError = encounteredError \|\| !actionItemCollectionsSuccess;\n }\n\n // Sync attributes\n if (attributes) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${attributes.length}\" attributes...`));\n const existingAttributes = await fetchAllAttributes(client, { logger });\n await map(\n attributes,\n async (attribute) => {\n const existing = existingAttributes.find((attr) => attr.name === attribute.name);\n\n logger.info(colors.magenta(`Syncing attribute \"${attribute.name}\"...`));\n try {\n await syncAttribute(client, attribute, {\n existingAttribute: existing,\n deleteExtraAttributeValues,\n logger,\n });\n logger.info(colors.green(`Successfully synced attribute \"${attribute.name}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(\n colors.red(`Failed to sync attribute \"${attribute.name}\"! - ${err.message}`),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${attributes.length}\" attributes!`));\n }\n\n // Sync action items\n if (actionItems) {\n const actionItemsSuccess = await syncActionItems(client, actionItems, { logger });\n encounteredError = encounteredError \|\| !actionItemsSuccess;\n }\n\n // Sync enrichers\n if (enrichers) {\n logger.info(colors.magenta(`Syncing \"${enrichers.length}\" enrichers...`));\n await map(\n enrichers,\n async (enricher) => {\n logger.info(colors.magenta(`Syncing enricher \"${enricher.title}\"...`));\n try {\n await syncEnricher(client, {\n input: enricher,\n identifierByName,\n dataSubjectsByName,\n logger,\n });\n logger.info(colors.green(`Successfully synced enricher \"${enricher.title}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync enricher \"${enricher.title}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${enrichers.length}\" enrichers!`));\n }\n\n // Sync identifiers\n if (identifiers) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${identifiers.length}\" identifiers...`));\n await map(\n identifiers,\n async (identifier) => {\n const existing = identifierByName[identifier.name];\n if (!existing) {\n throw new Error(\n `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing identifier \"${identifier.type}\"...`));\n try {\n await syncIdentifier(client, {\n input: identifier,\n dataSubjectsByName,\n identifierId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n logger,\n });\n logger.info(colors.green(`Successfully synced identifier \"${identifier.type}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.info(\n colors.red(`Failed to sync identifier \"${identifier.type}\"! - ${err.message}`),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${identifiers.length}\" identifiers!`));\n }\n\n // Sync actions\n if (actions) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${actions.length}\" actions...`));\n const existingActions = await fetchAllActions(client, { logger });\n await map(\n actions,\n async (action) => {\n const existing = existingActions.find((act) => act.type === action.type);\n if (!existing) {\n throw new Error(\n `Failed to find action with type: ${action.type}. Should have already existing in the organization.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing action \"${action.type}\"...`));\n try {\n await syncAction(\n client,\n {\n action,\n actionId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n },\n { logger },\n );\n logger.info(colors.green(`Successfully synced action \"${action.type}\"!`));\n } catch (err) {\n encounteredError = true;\n logger.error(colors.red(`Failed to sync action \"${action.type}\"! - ${err.message}`));\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${actions.length}\" actions!`));\n }\n\n // Sync data subjects\n if (dataSubjects) {\n // Fetch existing\n logger.info(colors.magenta(`Syncing \"${dataSubjects.length}\" data subjects...`));\n const existingDataSubjects = await fetchAllDataSubjects(client, { logger });\n await map(\n dataSubjects,\n async (dataSubject) => {\n const existing = existingDataSubjects.find((subj) => subj.type === dataSubject.type);\n if (!existing) {\n throw new Error(\n `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`,\n );\n }\n\n logger.info(colors.magenta(`Syncing data subject \"${dataSubject.type}\"...`));\n try {\n await syncDataSubject(client, {\n input: dataSubject,\n dataSubjectId: existing.id,\n skipPublish: !publishToPrivacyCenter,\n logger,\n });\n logger.info(colors.green(`Successfully synced data subject \"${dataSubject.type}\"!`));\n } catch (err) {\n encounteredError = true;\n // supportsAuthorizedAgent is gated on the Authorized Agents feature being\n // enabled for the organization; surface a clearer hint than the raw API error\n const hint =\n dataSubject.supportsAuthorizedAgent !== undefined &&\n /authorized.agent/i.test(err.message)\n ? ' (enable the Authorized Agents feature for this organization before setting \"supportsAuthorizedAgent\")'\n : '';\n logger.info(\n colors.red(\n `Failed to sync data subject \"${dataSubject.type}\"! - ${err.message}${hint}`,\n ),\n );\n }\n },\n {\n concurrency: CONCURRENCY,\n },\n );\n logger.info(colors.green(`Synced \"${dataSubjects.length}\" data subjects!`));\n }\n\n // Sync data flows\n if (dataFlows) {\n const syncedDataFlows = await syncDataFlows(client, dataFlows, { classifyService, logger });\n encounteredError = encounteredError \|\| !syncedDataFlows;\n }\n\n // Sync privacy center\n if (privacyCenter) {\n try {\n const privacyCenterSuccess = await syncPrivacyCenter(client, privacyCenter, {\n logger,\n skipPublish: !publishToPrivacyCenter,\n });\n if (!privacyCenterSuccess) {\n recordError('privacy-center', 'Failed to sync privacy center');\n }\n } catch (err) {\n recordError('privacy-center', (err as Error).message);\n }\n }\n\n // Sync messages\n if (messages) {\n const messagesSuccess = await syncIntlMessages(client, messages, { logger });\n encounteredError = encounteredError \|\| !messagesSuccess;\n }\n\n // Sync policies\n if (policies) {\n const policiesSuccess = await syncPolicies(client, policies, { logger });\n encounteredError = encounteredError \|\| !policiesSuccess;\n }\n\n // Store dependency updates\n const dependencyUpdates: [string, DependedOnDataSiloInput[]][] = [];\n // Sync data silos\n if (dataSilos) {\n const { success, dataSiloTitleToId } = await syncDataSilos(dataSilos, client, {\n dataSubjectsByName,\n apiKeysByTitle: apiKeyTitleMap,\n pageSize,\n });\n dataSilos?.forEach((dataSilo) => {\n // Queue up dependency update\n const dependencies = dataSilo['deletion-dependencies'];\n if (dependencies) {\n try {\n dependencyUpdates.push([\n dataSiloTitleToId[dataSilo.title],\n normalizeDeletionDependencies(dependencies, dataSilo.title),\n ]);\n } catch (err) {\n recordError('data-silos', (err as Error).message, dataSilo.title);\n activeLogger.error(colors.red((err as Error).message));\n }\n }\n });\n encounteredError = encounteredError \|\| !success;\n }\n\n // Dependencies updated at the end after all data silos are created\n if (dependencyUpdates.length > 0) {\n const dependenciesSuccess = await syncDataSiloDependencies(client, {\n input: dependencyUpdates,\n logger,\n });\n encounteredError = encounteredError \|\| !dependenciesSuccess;\n }\n\n // Update processing activities\n if (processingActivities) {\n const processingActivitySuccess = await syncProcessingActivities(client, processingActivities, {\n logger,\n });\n encounteredError = encounteredError \|\| !processingActivitySuccess;\n }\n\n if (publishToPrivacyCenter) {\n // TODO: https://transcend.height.app/T-23779\n }\n\n return {\n success: !encounteredError,\n errors,\n warnings: warnings.length > 0 ? warnings : undefined,\n };\n}\n/ eslint-enable max-lines /\n","import { getEntries } from '@transcend-io/type-utils';\n\nimport { TranscendInput } from '../codecs.js';\n\n/\n Combine a set of TranscendInput yaml files into a single yaml\n \n @param base - Base input\n * @param inputs - The list of inputs\n * @returns Merged input\n */\nexport function mergeTranscendInputs(\n base: TranscendInput,\n ...inputs: TranscendInput[]\n): TranscendInput {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const cloned: any = JSON.parse(JSON.stringify(base));\n inputs.forEach((input) => {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n getEntries(input).forEach(([key, value]: [any, any]) => {\n if (cloned[key] === undefined) {\n cloned[key] = value;\n } else if (Array.isArray(value)) {\n cloned[key] = [...cloned[key], ...value];\n } else {\n cloned[key] = value;\n }\n });\n });\n return cloned;\n}\n"],"mappings":"yvCAGA,MAAa,EAAwB,cAQrC,SAAgB,GAAkC,EAAiC,CACjF,IAAM,EAAmB,EAAE,CAErB,GAAgB,EAAc,IAA0B,CACvD,EAAsB,KAAK,EAAK,EACnC,EAAO,KACL,GAAG,EAAQ,UAAU,EAAK,sNAG3B,EAIL,IAAK,IAAM,KAAU,EAAM,uBAAyB,EAAE,CACpD,EAAa,EAAO,KAAM,qBAAqB,CAGjD,IAAK,IAAM,KAAW,EAAM,UAAY,EAAE,CACxC,IAAK,IAAM,KAAS,EAAQ,sBAAwB,EAAE,CAAE,CACtD,EAAa,EAAM,KAAM,oBAAoB,EAAM,MAAM,IAAI,CAC7D,IAAK,IAAM,KAAU,EAAM,SAAW,EAAE,CACtC,EAAa,EAAO,KAAM,oBAAoB,EAAM,MAAM,YAAY,EAAO,MAAM,IAAI,CAK7F,OAAO,ECrBT,eAAsB,EACpB,CACE,aAAc,EAAY,EAAE,CAC5B,gBAAiB,EAAe,EAAE,CAClC,wBAAyB,EAAuB,EAAE,CAClD,YAAY,EAAE,EAEhB,EACA,EAAW,GACiC,CAC5C,IAAM,EAAuB,EAAK,CAChC,GAAG,EAAQ,EAAU,IAAK,GAAS,EAAK,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CACtE,GAAG,EAAQ,EAAqB,KAAK,CAAE,sBAAuB,GAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC5F,GAAG,EAAQ,EAAU,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC9E,GAAG,EAAa,IAAK,GAAY,EAAQ,KAAK,CAC/C,CAAC,CACF,GAAI,EAAqB,SAAW,GAAK,CAAC,EACxC,MAAO,EAAE,CAGX,IAAM,EAAmB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CACjE,EAAoB,EAAM,EAAkB,OAAO,CAEnD,EAAsB,EAC1B,EACA,EAAiB,KAAK,CAAE,UAAW,EAAK,CACzC,CAED,GAAI,EAAoB,OAAS,EAAG,CAClC,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAoB,OAAO,uBAAuB,CAAC,CAC1F,IAAK,IAAM,KAAmB,EAAqB,CACjD,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAgB,KAAK,CAAC,CAC1E,IAAM,EAAU,MAAM,EAAkB,EAAQ,CAAE,MAAO,EAAiB,SAAQ,CAAC,CACnF,EAAO,KAAK,EAAO,MAAM,wBAAwB,EAAgB,GAAG,CAAC,CACrE,EAAkB,GAAmB,GAIzC,OAAO,ECrCT,SAAgB,GACd,EACA,EAC2B,CAC3B,GAAI,EAAa,SAAW,EAC1B,MAAO,EAAE,CAIX,GAAI,OAAO,EAAa,IAAO,SAC7B,MAAO,CAAC,CAAE,OAAQ,EAAK,EAAyB,CAAE,CAAC,CAGrD,IAAI,EACE,EAA+C,EAAE,CACjD,EAAgB,IAAI,IASpB,GACJ,EACA,IACS,CACT,GAAI,EAAc,IAAI,EAAS,CAC7B,MAAU,MACR,cAAc,EAAc,6DACtB,EAAS,sCAChB,CAEH,EAAc,IAAI,EAAS,CAC3B,EAAkB,KAAK,CAAE,2BAA4B,EAAU,GAAG,EAAU,CAAC,EAkC/E,OA/BC,EAA2C,QAAS,GAAe,CAGlE,GAAI,oBAAqB,GAAc,WAAY,EACjD,MAAU,MACR,cAAc,EAAc,oDACtB,EAAW,SAAS,mMAG3B,CAGH,GAAI,oBAAqB,EAAY,CACnC,EAAoB,EAAW,SAAU,CAAE,cAAe,GAAM,CAAC,CACjE,OAGF,GAAI,EAAW,SAAU,CACvB,EAAoB,EAAW,SAAU,CAAE,OAAQ,EAAW,OAAQ,CAAC,CACvE,OAGF,GAAI,IAAiB,IAAA,GACnB,MAAU,MACR,cAAc,EAAc,8GAE7B,CAEH,EAAe,EAAW,QAC1B,CAEK,CACL,GAAI,IAAiB,IAAA,GAA+C,EAAE,CAArC,CAAC,CAAE,OAAQ,EAAK,EAAa,CAAE,CAAC,CACjE,GAAG,EACJ,CCzDH,eAAsB,EACpB,EACA,EACA,CACE,WACA,qBACA,kBAcD,CACD,IAAI,EAAmB,GAGjB,EAAK,IAAI,MAAM,CAAC,SAAS,CAC/B,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,iBAAiB,CAAC,CAU1E,IAAM,EAA0B,EAAsC,MAPtC,EAAkB,EAAQ,CACxD,OAAQ,EAAU,KAAK,CAAE,WAAY,EAAM,CAC3C,WACA,SACD,CAAC,CAGuF,QAAQ,CAG3F,EAAoB,EAAU,QAAQ,CAAE,WAAY,CAAC,EAAwB,GAAO,CACtF,EAAkB,OAAS,IAC7B,EAAO,KACL,EAAO,QAAQ,aAAa,EAAkB,OAAO,oCAAoC,CAC1F,CAID,MAAM,EADU,EAAM,EAAmB,GAClB,CAAE,KAAO,IAA0B,CACxD,GAAM,CACJ,gBAAiB,CAAE,cACjB,MAAM,EAMP,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,EAAsB,IAAK,IAAW,CAC3C,KAAM,EAAM,eAAiB,EAAM,gBACnC,MAAO,EAAM,MACb,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,SAAU,EAAM,aACjB,EAAE,CACJ,CACD,SACD,CAAC,CAGF,EAAU,QAAS,GAAS,CAC1B,EAAwB,EAAK,OAAS,GACtC,EACF,CAEF,EAAO,KAAK,EAAO,MAAM,yBAAyB,EAAkB,OAAO,eAAe,CAAC,EAI7F,IAAM,EAAiB,EAAM,EAAW,GAAkB,CAC1D,MAAM,EAAU,EAAgB,MAAO,EAAqB,IAAQ,CAClE,EAAO,KACL,EAAO,QACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,aACzC,EAAoB,OACrB,cACF,CACF,CACD,MAAM,EAMH,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,CACL,UAAW,EAAoB,IAAK,IAAW,CAC7C,GAAI,EAAwB,EAAM,OAAO,GACzC,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,IAAK,EAAM,IACX,QAAS,EAAM,QACf,YAAa,EAAM,YACnB,YAAa,EAAM,iBACnB,OAAQ,CAAC,EAAM,SACf,YAAa,EAAM,OACnB,UAAW,EAAM,MAGjB,oBAAqB,EAAM,yBAA2B,IAAA,GAAY,CAAC,CAAE,OAAQ,EAAE,CAAE,CAAC,CAClF,SAAU,EAAM,iBACZ,EAAe,EAAM,kBAAkB,GACvC,IAAA,GACJ,wBAAyB,EAAM,iBAC3B,EAA8B,EAAM,iBAAkB,EAAmB,CACzE,IAAA,GACJ,WAAY,EAAM,WAClB,qBAAsB,EAAM,qBAC5B,SAAU,EAAM,aAEhB,mBAAoB,EAAM,oBAAoB,wBAC9C,gCAAiC,EAAM,oBAAoB,kBAC3D,2BAA4B,EAAM,oBAAoB,aACtD,+CACE,EAAM,oBAAoB,kCAC5B,qCAAsC,EAAM,oBAAoB,wBAChE,yBAA0B,EAAM,oBAAoB,+BACrD,EAAE,CACJ,CACF,CACD,SACD,CAAC,CACF,EAAO,KACL,EAAO,MACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,YACzC,EAAoB,OACrB,eACF,CACF,EACD,CAKF,IAAM,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAA0B,EAAU,QAAQ,CAAE,aAAa,EAAE,IAAO,EAAW,OAAS,EAAE,CAC1F,EAAkB,EACrB,KAAK,CAAE,aAAa,EAAE,IAAO,EAAW,OAAO,CAC/C,QAAQ,EAAK,IAAU,EAAM,EAAO,EAAE,CACzC,EAAO,KACL,EAAO,QACL,YAAY,EAAgB,qBAAqB,EAAwB,OAAO,iBACjF,CACF,CACD,EAAY,MAAM,EAAiB,EAAE,CACrC,IAAI,EAAQ,EAEZ,MAAM,EACJ,EACA,MAAO,CAAE,aAAY,WAAY,CAC3B,GACF,MAAM,EAAU,EAAY,KAAO,IAAc,CAC/C,IAAM,EAAS,EAAU,OACrB,EAAU,OAAO,KACd,CAAE,MAAK,cAAa,aAAY,WAAU,aAAY,GAAG,MAEvD,CACC,KAAM,EACN,cACA,WAAa,EAET,EAAW,IAAK,IAAc,CAC5B,GAAG,EACH,KAAM,EAAS,MAAQ,QACxB,EAAE,CAJH,IAAA,GAKJ,SAAW,EAEP,EAAS,IAAK,IAAa,CACzB,GAAG,EACH,KAAM,EAAQ,MAAQ,QACvB,EAAE,CAJH,IAAA,GAKJ,aACA,+BAAgC,EAAK,qCACrC,+BAAgC,EAAK,qCACtC,EACJ,CACD,IAAA,GAEE,EAAU,CACd,WAAY,EAAwB,GAAO,GAC3C,KAAM,EAAU,KAChB,KAAM,EAAU,IAChB,MAAO,EAAU,MACjB,YAAa,EAAU,YACvB,GAAI,EAAU,OACV,CACE,YAAa,EAAU,OACxB,CACD,EAAE,CACN,GAAI,EAAU,MACV,CACE,UAAW,EAAU,MACtB,CACD,EAAE,CACN,GAAI,EAAU,uBACV,CAAE,kBAAmB,EAAU,uBAAwB,CACvD,EAAE,CACN,iBAAmB,EAAU,0BAEzB,OAAO,QAAQ,EAAU,0BAA0B,CAAC,KAAK,CAAC,EAAK,MAAY,CACzE,YAAa,EACb,eAAgB,EACjB,EAAE,CAJH,IAAA,GAKJ,eAAgB,EAAU,oBAAsB,EAAE,CAClD,cAAe,EAChB,CAGK,GAAyB,EAAQ,eAAiB,EAAE,EAAE,KAAK,CAAE,UAAW,EAAK,CAC7E,EAAsB,EAAsB,QAC/C,EAAM,IAAU,EAAsB,QAAQ,EAAK,GAAK,EAC1D,CACD,GAAI,EAAoB,OAAS,EAC/B,EAAO,KACL,EAAO,IACL,8BACE,EAAU,IACX,6DAA6D,EAAoB,KAChF;EACD,GACF,CACF,CACD,EAAmB,QAEnB,GAAI,CACF,MAAM,EAAmB,EAAQ,EAA6B,CAC5D,UAAW,EACX,SACD,CAAC,OACK,EAAK,CACZ,EAAO,KACL,EAAO,IACL,iCAAiC,EAAU,IAAI,mBAAmB,EAAM,SAAS,EAAI,UACtF,CACF,CACD,EAAmB,GAGvB,GAAS,EACT,EAAY,OAAO,EAAM,EACzB,EAGN,CACE,YAAa,GACd,CACF,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EASvB,OAPA,EAAO,KACL,EAAO,MACL,WAAW,EAAU,OAAO,oBAAoB,EAAgB,mBAC9D,EAAY,IACb,YACF,CACF,CACM,CACL,QAAS,CAAC,EACV,kBAAmB,EAAM,GAA0B,CAAE,QAAS,EAAG,CAClE,CCtPH,eAAsB,EACpB,EACA,EACA,CACE,WAAW,GAEX,yBAAyB,GACzB,kBAAkB,GAClB,6BAA6B,GAC7B,OAAQ,EACR,WAAW,EAAE,EAeM,CACrB,IAAM,EAAe,GAAc,EAC7B,EAAsB,EAAE,CAC1B,EAAmB,GAEjB,GAAe,EAAkB,EAAiB,IAAwB,CAC9E,EAAO,KAAK,CAAE,WAAU,OAAM,UAAS,CAAC,CACxC,EAAmB,IAGrB,EAAa,KAAK,EAAO,QAAQ,gCAAgC,EAAS,KAAK,CAAC,CAEhF,GAAM,CACJ,YACA,aACA,UACA,cACA,gBAAiB,EACjB,oBAAqB,EACrB,YACA,UACA,kBAAmB,EACnB,aAAc,EACd,aAAc,EACd,UACA,kBAAmB,EACnB,wBAAyB,EACzB,sBAAuB,EACvB,eAAgB,EAChB,0BAA2B,EAC3B,QACA,iBAAkB,EAClB,WACA,WACA,aACA,8BAA+B,GAC/B,mBAAoB,GAClB,EAEE,GAAoB,EAAM,sBAC1B,EAAW,EAAM,SAEjB,GAAuB,GAAkC,EAAM,CACrE,GAAI,GAAqB,OAAS,EAAG,CACnC,IAAK,IAAM,KAAW,GACpB,EAAY,mBAAoB,EAAQ,CAE1C,MAAO,CACL,QAAS,GACT,SACA,SAAU,EAAS,OAAS,EAAI,EAAW,IAAA,GAC5C,CAGH,GAAM,CAAC,EAAkB,EAAoB,IAAkB,MAAM,QAAQ,IAAI,CAE/E,GAAa,EACT,EAAiC,EAAQ,CACvC,QACA,YAAa,CAAC,EACd,SACD,CAAC,CACD,EAAE,CAEP,GAAa,GAAgB,GAAa,EACtC,EAA2B,EAAO,EAAO,CACzC,EAAE,CAEN,GACA,EACG,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,CAClD,QAAQ,EAAK,IAAQ,EAAM,EAAI,OAAQ,EAAE,CAAG,EAC3C,EAAa,EAAQ,CAAE,aAAc,EAAO,SAAQ,CAAC,CACrD,EAAE,CACP,CAAC,CAqCF,GAnCI,IAA2B,SASxB,MAR0C,GAC7C,EACA,GACA,CACE,OAAQ,EACR,WACD,CACF,EAEC,EAAY,8BAA+B,6CAA6C,EAIxF,GAAiB,SAId,MAHgC,GAAoB,EAAQ,EAAiB,CAChF,OAAQ,EACT,CAAC,EAEA,EAAY,mBAAoB,kCAAkC,EAIlE,IAAmB,SAIhB,MAHkC,GAA2B,EAAQ,GAAmB,CAC3F,OAAQ,EACT,CAAC,EAEA,EAAY,qBAAsB,0CAA0C,EAO5E,GAAU,OAAQ,CACpB,GAAM,CAAE,QAAS,EAAiB,2BAA4B,MAAM,GAClE,EACA,EACA,CAAE,OAAQ,EAAc,CACzB,CACI,GACH,EAAY,WAAY,sCAAsC,CAGhE,IAAM,EAAS,EAAS,QAAS,IAC9B,EAAQ,sBAAwB,EAAE,EAAE,IAAK,IAAW,CACnD,GAAG,EACH,gBAAiB,EAAQ,aAC1B,EAAE,CACJ,CACG,EAAO,OAAS,IAKb,MAJuB,GAAqB,EAAQ,EAAQ,CAC/D,OAAQ,EACR,0BACD,CAAC,EAEA,EAAY,oBAAqB,+CAA+C,EAMtF,GAAI,EAAgB,CAClB,EAAa,KAAK,EAAO,QAAQ,6BAA6B,CAAC,CAC/D,GAAI,CACF,MAAM,GAAmB,EAAQ,EAAgB,CAAE,OAAQ,EAAc,CAAC,CAC1E,EAAa,KAAK,EAAO,MAAM,uCAAuC,CAAC,OAChE,EAAK,CACZ,EAAY,kBAAoB,EAAc,QAAQ,CACtD,EAAa,MAAM,EAAO,IAAI,qCAAsC,EAAc,UAAU,CAAC,EAIjG,GAAI,EAAO,CACT,IAAM,EAAe,MAAM,GAAU,EAAQ,EAAO,CAAE,SAAQ,CAAC,CAC/D,IAAuC,CAAC,EA0B1C,GAtBI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,sBAAsB,CAAC,CAC/E,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAChD,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,oBAAoB,CAAC,EAIxE,EAAkB,CACpB,IAAM,EAAwB,MAAM,EAAqB,EAAQ,EAAkB,CAAE,SAAQ,CAAC,CAC9F,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAoB,CACtB,IAAM,EAA4B,MAAM,GAAuB,EAAQ,EAAoB,CACzF,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,GAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAuB,CACzB,IAAM,EAA+B,MAAM,EACzC,EACA,EACA,CAAE,SAAQ,CACX,CACD,IAAuC,CAAC,EAI1C,GAAI,EAAY,CAEd,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAW,OAAO,iBAAiB,CAAC,CAC3E,IAAM,EAAqB,MAAM,EAAmB,EAAQ,CAAE,SAAQ,CAAC,CACvE,MAAM,EACJ,EACA,KAAO,IAAc,CACnB,IAAM,EAAW,EAAmB,KAAM,GAAS,EAAK,OAAS,EAAU,KAAK,CAEhF,EAAO,KAAK,EAAO,QAAQ,sBAAsB,EAAU,KAAK,MAAM,CAAC,CACvE,GAAI,CACF,MAAM,EAAc,EAAQ,EAAW,CACrC,kBAAmB,EACnB,6BACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,kCAAkC,EAAU,KAAK,IAAI,CAAC,OACxE,EAAK,CACZ,EAAmB,GACnB,EAAO,MACL,EAAO,IAAI,6BAA6B,EAAU,KAAK,OAAO,EAAI,UAAU,CAC7E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAW,OAAO,eAAe,CAAC,CAIxE,GAAI,EAAa,CACf,IAAM,EAAqB,MAAM,EAAgB,EAAQ,EAAa,CAAE,SAAQ,CAAC,CACjF,IAAuC,CAAC,EAqE1C,GAjEI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,gBAAgB,CAAC,CACzE,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,CACzB,MAAO,EACP,mBACA,qBACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,cAAc,CAAC,EAIlE,IAEF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAY,OAAO,kBAAkB,CAAC,CAC7E,MAAM,EACJ,EACA,KAAO,IAAe,CACpB,IAAM,EAAW,EAAiB,EAAW,MAC7C,GAAI,CAAC,EACH,MAAU,MACR,wCAAwC,EAAW,KAAK,yCACzD,CAGH,EAAO,KAAK,EAAO,QAAQ,uBAAuB,EAAW,KAAK,MAAM,CAAC,CACzE,GAAI,CACF,MAAM,GAAe,EAAQ,CAC3B,MAAO,EACP,qBACA,aAAc,EAAS,GACvB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,mCAAmC,EAAW,KAAK,IAAI,CAAC,OAC1E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,8BAA8B,EAAW,KAAK,OAAO,EAAI,UAAU,CAC/E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAY,OAAO,gBAAgB,CAAC,EAItE,EAAS,CAEX,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAQ,OAAO,cAAc,CAAC,CACrE,IAAM,EAAkB,MAAM,EAAgB,EAAQ,CAAE,SAAQ,CAAC,CACjE,MAAM,EACJ,EACA,KAAO,IAAW,CAChB,IAAM,EAAW,EAAgB,KAAM,GAAQ,EAAI,OAAS,EAAO,KAAK,CACxE,GAAI,CAAC,EACH,MAAU,MACR,oCAAoC,EAAO,KAAK,qDACjD,CAGH,EAAO,KAAK,EAAO,QAAQ,mBAAmB,EAAO,KAAK,MAAM,CAAC,CACjE,GAAI,CACF,MAAM,EACJ,EACA,CACE,SACA,SAAU,EAAS,GACnB,YAAa,CAAC,EACf,CACD,CAAE,SAAQ,CACX,CACD,EAAO,KAAK,EAAO,MAAM,+BAA+B,EAAO,KAAK,IAAI,CAAC,OAClE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,0BAA0B,EAAO,KAAK,OAAO,EAAI,UAAU,CAAC,GAGxF,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAQ,OAAO,YAAY,CAAC,CAIlE,GAAI,EAAc,CAEhB,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAa,OAAO,oBAAoB,CAAC,CAChF,IAAM,EAAuB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CAC3E,MAAM,EACJ,EACA,KAAO,IAAgB,CACrB,IAAM,EAAW,EAAqB,KAAM,GAAS,EAAK,OAAS,EAAY,KAAK,CACpF,GAAI,CAAC,EACH,MAAU,MACR,0CAA0C,EAAY,KAAK,qDAC5D,CAGH,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAY,KAAK,MAAM,CAAC,CAC5E,GAAI,CACF,MAAM,GAAgB,EAAQ,CAC5B,MAAO,EACP,cAAe,EAAS,GACxB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,qCAAqC,EAAY,KAAK,IAAI,CAAC,OAC7E,EAAK,CACZ,EAAmB,GAGnB,IAAM,EACJ,EAAY,0BAA4B,IAAA,IACxC,oBAAoB,KAAK,EAAI,QAAQ,CACjC,yGACA,GACN,EAAO,KACL,EAAO,IACL,gCAAgC,EAAY,KAAK,OAAO,EAAI,UAAU,IACvE,CACF,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAa,OAAO,kBAAkB,CAAC,CAI7E,GAAI,EAAW,CACb,IAAM,EAAkB,MAAM,GAAc,EAAQ,EAAW,CAAE,kBAAiB,SAAQ,CAAC,CAC3F,IAAuC,CAAC,EAI1C,GAAI,EACF,GAAI,CAKG,MAJ8B,GAAkB,EAAQ,EAAe,CAC1E,SACA,YAAa,CAAC,EACf,CAAC,EAEA,EAAY,iBAAkB,gCAAgC,OAEzD,EAAK,CACZ,EAAY,iBAAmB,EAAc,QAAQ,CAKzD,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAiB,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAC5E,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CACxE,IAAuC,CAAC,EAI1C,IAAM,EAA2D,EAAE,CAEnE,GAAI,EAAW,CACb,GAAM,CAAE,UAAS,qBAAsB,MAAM,EAAc,EAAW,EAAQ,CAC5E,qBACA,eAAgB,GAChB,WACD,CAAC,CACF,GAAW,QAAS,GAAa,CAE/B,IAAM,EAAe,EAAS,yBAC9B,GAAI,EACF,GAAI,CACF,EAAkB,KAAK,CACrB,EAAkB,EAAS,OAC3B,GAA8B,EAAc,EAAS,MAAM,CAC5D,CAAC,OACK,EAAK,CACZ,EAAY,aAAe,EAAc,QAAS,EAAS,MAAM,CACjE,EAAa,MAAM,EAAO,IAAK,EAAc,QAAQ,CAAC,GAG1D,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAkB,OAAS,EAAG,CAChC,IAAM,EAAsB,MAAM,GAAyB,EAAQ,CACjE,MAAO,EACP,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAsB,CACxB,IAAM,EAA4B,MAAM,GAAyB,EAAQ,EAAsB,CAC7F,SACD,CAAC,CACF,IAAuC,CAAC,EAO1C,MAAO,CACL,QAAS,CAAC,EACV,SACA,SAAU,EAAS,OAAS,EAAI,EAAW,IAAA,GAC5C,CC3jBH,SAAgB,EACd,EACA,GAAG,EACa,CAEhB,IAAM,EAAc,KAAK,MAAM,KAAK,UAAU,EAAK,CAAC,CAapD,OAZA,EAAO,QAAS,GAAU,CAExB,EAAW,EAAM,CAAC,SAAS,CAAC,EAAK,KAAuB,CAClD,EAAO,KAAS,IAAA,GAClB,EAAO,GAAO,EACL,MAAM,QAAQ,EAAM,CAC7B,EAAO,GAAO,CAAC,GAAG,EAAO,GAAM,GAAG,EAAM,CAExC,EAAO,GAAO,GAEhB,EACF,CACK"}

package/dist/{notifyPrivacyRequestsAdditionalTime-vHikIxVO.mjs → notifyPrivacyRequestsAdditionalTime-it1FA62W.mjs} RENAMED Viewed

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-DKKXqoSr.mjs";import{t}from"./logger-Ckep4QXn.mjs";import{r as n}from"./request-BIplTb3L.mjs";import{r}from"./fetchAllRequests-DP24WqrP.mjs";import{RequestAction as i}from"@transcend-io/privacy-types";import a from"colors";import{buildTranscendGraphQLClient as o,fetchAllTemplates as s,makeGraphQLRequest as c}from"@transcend-io/sdk";import{map as l}from"@transcend-io/utils";import u from"cli-progress";async function d({requestActions:d=Object.values(i),auth:f,requestIds:p,createdAtBefore:m,days:h=45,daysLeft:g=10,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,emailTemplate:b=`Additional Time Needed`,concurrency:x=100,transcendUrl:S=e}){let C=o(S,f),w=new Date().getTime(),T=new u.SingleBar({},u.Presets.shades_classic),E=(await s(C,{logger:t,filterBy:{title:b}})).find(e=>e.title===b);if(!E)throw Error(`Failed to find a template with title: "${b}"`);let D=await r(C,{actions:d,createdAtBefore:m,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,isSilent:!1,isClosed:!1,requestIds:p});D=D.filter(e=>typeof e.daysRemaining==`number`&&e.daysRemaining<g),t.info(a.magenta(`Notifying "${D.length}" that more time is needed.`));let O=0;T.start(D.length,0),await l(D,async e=>{await c(C,n,{variables:{input:{requestId:e.id,template:E.template.defaultMessage,subject:E.subject.defaultMessage,additionalTime:h}},logger:t}),O+=1,T.update(O)},{concurrency:x}),T.stop();let k=new Date().getTime()-w;return t.info(a.green(`Successfully marked ${O} requests as silent mode in "${k/1e3}" seconds!`)),D.length}export{d as t};
-//# sourceMappingURL=notifyPrivacyRequestsAdditionalTime-vHikIxVO.mjs.map
+import{a as e}from"./constants-CdgK1myP.mjs";import{t}from"./logger-CB_h_zbj.mjs";import{r as n}from"./request-CCCU2SdV.mjs";import{r}from"./fetchAllRequests-DUQGTJjx.mjs";import{RequestAction as i}from"@transcend-io/privacy-types";import a from"colors";import{buildTranscendGraphQLClient as o,fetchAllTemplates as s,makeGraphQLRequest as c}from"@transcend-io/sdk";import{map as l}from"@transcend-io/utils";import u from"cli-progress";async function d({requestActions:d=Object.values(i),auth:f,requestIds:p,createdAtBefore:m,days:h=45,daysLeft:g=10,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,emailTemplate:b=`Additional Time Needed`,concurrency:x=100,transcendUrl:S=e}){let C=o(S,f),w=new Date().getTime(),T=new u.SingleBar({},u.Presets.shades_classic),E=(await s(C,{logger:t,filterBy:{title:b}})).find(e=>e.title===b);if(!E)throw Error(`Failed to find a template with title: "${b}"`);let D=await r(C,{actions:d,createdAtBefore:m,createdAtAfter:_,updatedAtBefore:v,updatedAtAfter:y,isSilent:!1,isClosed:!1,requestIds:p});D=D.filter(e=>typeof e.daysRemaining==`number`&&e.daysRemaining<g),t.info(a.magenta(`Notifying "${D.length}" that more time is needed.`));let O=0;T.start(D.length,0),await l(D,async e=>{await c(C,n,{variables:{input:{requestId:e.id,template:E.template.defaultMessage,subject:E.subject.defaultMessage,additionalTime:h}},logger:t}),O+=1,T.update(O)},{concurrency:x}),T.stop();let k=new Date().getTime()-w;return t.info(a.green(`Successfully marked ${O} requests as silent mode in "${k/1e3}" seconds!`)),D.length}export{d as t};
+//# sourceMappingURL=notifyPrivacyRequestsAdditionalTime-it1FA62W.mjs.map

package/dist/{notifyPrivacyRequestsAdditionalTime-vHikIxVO.mjs.map → notifyPrivacyRequestsAdditionalTime-it1FA62W.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"notifyPrivacyRequestsAdditionalTime-~~vHikIxVO~~.mjs","names":[],"sources":["../src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts"],"sourcesContent":["import { RequestAction } from '@transcend-io/privacy-types';\nimport {\n buildTranscendGraphQLClient,\n fetchAllTemplates,\n makeGraphQLRequest,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { NOTIFY_ADDITIONAL_TIME, fetchAllRequests } from '../graphql/index.js';\n\n/*\n Mark a set of privacy requests to be in silent mode.\n * Note requests in silent mode are ignored\n \n @param options - Options\n * @returns The number of requests marked silent\n /\nexport async function notifyPrivacyRequestsAdditionalTime({\n requestActions = Object.values(RequestAction),\n auth,\n requestIds,\n createdAtBefore,\n days = 45,\n daysLeft = 10,\n createdAtAfter,\n updatedAtBefore,\n updatedAtAfter,\n emailTemplate = 'Additional Time Needed',\n concurrency = 100,\n transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n /* The request actions that should be restarted /\n requestActions?: RequestAction[];\n /* Filter for requests created before this date /\n createdAtBefore: Date;\n /* Filter for requests created after this date /\n createdAtAfter?: Date;\n /* Filter for requests updated before this date /\n updatedAtBefore?: Date;\n /* Filter for requests updated after this date /\n updatedAtAfter?: Date;\n /* Email template /\n emailTemplate?: string;\n /* Transcend API key authentication /\n auth: string;\n /* Number of days to extend request by /\n days?: number;\n /\n Only notify requests that have less than this number of days until they are considered expired.\n * This allows for re-running the command without notifying the same users multiple times\n /\n daysLeft?: number;\n /* Concurrency limit for approving /\n concurrency?: number;\n /* The set of privacy requests to notify /\n requestIds?: string[];\n /* API URL for Transcend backend */\n transcendUrl?: string;\n}): Promise<number> {\n // Find all requests made before createdAt that are in a removing data state\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n // Grab the template with that title\n const matchingTemplates = await fetchAllTemplates(client, {\n logger,\n filterBy: { title: emailTemplate },\n });\n const exactTemplateMatch = matchingTemplates.find((template) => template.title === emailTemplate);\n if (!exactTemplateMatch) {\n throw new Error(`Failed to find a template with title: \"${emailTemplate}\"`);\n }\n\n // Pull in the requests\n let allRequests = await fetchAllRequests(client, {\n actions: requestActions,\n createdAtBefore,\n createdAtAfter,\n updatedAtBefore,\n updatedAtAfter,\n isSilent: false,\n isClosed: false,\n requestIds,\n });\n\n // Filter requests by daysLeft\n allRequests = allRequests.filter(\n (request) => typeof request.daysRemaining === 'number' && request.daysRemaining < daysLeft,\n );\n\n // Notify Transcend\n logger.info(colors.magenta(`Notifying \"${allRequests.length}\" that more time is needed.`));\n\n let total = 0;\n progressBar.start(allRequests.length, 0);\n await map(\n allRequests,\n async (requestToNotify) => {\n await makeGraphQLRequest(client, NOTIFY_ADDITIONAL_TIME, {\n variables: {\n input: {\n requestId: requestToNotify.id,\n template: exactTemplateMatch.template.defaultMessage,\n subject: exactTemplateMatch.subject.defaultMessage,\n additionalTime: days,\n },\n },\n logger,\n });\n\n total += 1;\n progressBar.update(total);\n },\n { concurrency },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully marked ${total} requests as silent mode in \"${totalTime / 1000}\" seconds!`,\n ),\n );\n return allRequests.length;\n}\n"],"mappings":"mbAqBA,eAAsB,EAAoC,CACxD,iBAAiB,OAAO,OAAO,EAAc,CAC7C,OACA,aACA,kBACA,OAAO,GACP,WAAW,GACX,iBACA,kBACA,iBACA,gBAAgB,yBAChB,cAAc,IACd,eAAe,GA6BG,CAElB,IAAM,EAAS,EAA4B,EAAc,EAAK,CAGxD,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAO/E,GAAqB,MAJK,EAAkB,EAAQ,CACxD,SACA,SAAU,CAAE,MAAO,EAAe,CACnC,CAAC,EAC2C,KAAM,GAAa,EAAS,QAAU,EAAc,CACjG,GAAI,CAAC,EACH,MAAU,MAAM,0CAA0C,EAAc,GAAG,CAI7E,IAAI,EAAc,MAAM,EAAiB,EAAQ,CAC/C,QAAS,EACT,kBACA,iBACA,kBACA,iBACA,SAAU,GACV,SAAU,GACV,aACD,CAAC,CAGF,EAAc,EAAY,OACvB,GAAY,OAAO,EAAQ,eAAkB,UAAY,EAAQ,cAAgB,EACnF,CAGD,EAAO,KAAK,EAAO,QAAQ,cAAc,EAAY,OAAO,6BAA6B,CAAC,CAE1F,IAAI,EAAQ,EACZ,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,MAAM,EACJ,EACA,KAAO,IAAoB,CACzB,MAAM,EAAmB,EAAQ,EAAwB,CACvD,UAAW,CACT,MAAO,CACL,UAAW,EAAgB,GAC3B,SAAU,EAAmB,SAAS,eACtC,QAAS,EAAmB,QAAQ,eACpC,eAAgB,EACjB,CACF,CACD,SACD,CAAC,CAEF,GAAS,EACT,EAAY,OAAO,EAAM,EAE3B,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAM,+BAA+B,EAAY,IAAK,YAC9E,CACF,CACM,EAAY"}
1	+ {"version":3,"file":"notifyPrivacyRequestsAdditionalTime-it1FA62W.mjs","names":[],"sources":["../src/lib/requests/notifyPrivacyRequestsAdditionalTime.ts"],"sourcesContent":["import { RequestAction } from '@transcend-io/privacy-types';\nimport {\n buildTranscendGraphQLClient,\n fetchAllTemplates,\n makeGraphQLRequest,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { NOTIFY_ADDITIONAL_TIME, fetchAllRequests } from '../graphql/index.js';\n\n/*\n Mark a set of privacy requests to be in silent mode.\n * Note requests in silent mode are ignored\n \n @param options - Options\n * @returns The number of requests marked silent\n /\nexport async function notifyPrivacyRequestsAdditionalTime({\n requestActions = Object.values(RequestAction),\n auth,\n requestIds,\n createdAtBefore,\n days = 45,\n daysLeft = 10,\n createdAtAfter,\n updatedAtBefore,\n updatedAtAfter,\n emailTemplate = 'Additional Time Needed',\n concurrency = 100,\n transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n /* The request actions that should be restarted /\n requestActions?: RequestAction[];\n /* Filter for requests created before this date /\n createdAtBefore: Date;\n /* Filter for requests created after this date /\n createdAtAfter?: Date;\n /* Filter for requests updated before this date /\n updatedAtBefore?: Date;\n /* Filter for requests updated after this date /\n updatedAtAfter?: Date;\n /* Email template /\n emailTemplate?: string;\n /* Transcend API key authentication /\n auth: string;\n /* Number of days to extend request by /\n days?: number;\n /\n Only notify requests that have less than this number of days until they are considered expired.\n * This allows for re-running the command without notifying the same users multiple times\n /\n daysLeft?: number;\n /* Concurrency limit for approving /\n concurrency?: number;\n /* The set of privacy requests to notify /\n requestIds?: string[];\n /* API URL for Transcend backend */\n transcendUrl?: string;\n}): Promise<number> {\n // Find all requests made before createdAt that are in a removing data state\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n // Grab the template with that title\n const matchingTemplates = await fetchAllTemplates(client, {\n logger,\n filterBy: { title: emailTemplate },\n });\n const exactTemplateMatch = matchingTemplates.find((template) => template.title === emailTemplate);\n if (!exactTemplateMatch) {\n throw new Error(`Failed to find a template with title: \"${emailTemplate}\"`);\n }\n\n // Pull in the requests\n let allRequests = await fetchAllRequests(client, {\n actions: requestActions,\n createdAtBefore,\n createdAtAfter,\n updatedAtBefore,\n updatedAtAfter,\n isSilent: false,\n isClosed: false,\n requestIds,\n });\n\n // Filter requests by daysLeft\n allRequests = allRequests.filter(\n (request) => typeof request.daysRemaining === 'number' && request.daysRemaining < daysLeft,\n );\n\n // Notify Transcend\n logger.info(colors.magenta(`Notifying \"${allRequests.length}\" that more time is needed.`));\n\n let total = 0;\n progressBar.start(allRequests.length, 0);\n await map(\n allRequests,\n async (requestToNotify) => {\n await makeGraphQLRequest(client, NOTIFY_ADDITIONAL_TIME, {\n variables: {\n input: {\n requestId: requestToNotify.id,\n template: exactTemplateMatch.template.defaultMessage,\n subject: exactTemplateMatch.subject.defaultMessage,\n additionalTime: days,\n },\n },\n logger,\n });\n\n total += 1;\n progressBar.update(total);\n },\n { concurrency },\n );\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully marked ${total} requests as silent mode in \"${totalTime / 1000}\" seconds!`,\n ),\n );\n return allRequests.length;\n}\n"],"mappings":"mbAqBA,eAAsB,EAAoC,CACxD,iBAAiB,OAAO,OAAO,EAAc,CAC7C,OACA,aACA,kBACA,OAAO,GACP,WAAW,GACX,iBACA,kBACA,iBACA,gBAAgB,yBAChB,cAAc,IACd,eAAe,GA6BG,CAElB,IAAM,EAAS,EAA4B,EAAc,EAAK,CAGxD,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAO/E,GAAqB,MAJK,EAAkB,EAAQ,CACxD,SACA,SAAU,CAAE,MAAO,EAAe,CACnC,CAAC,EAC2C,KAAM,GAAa,EAAS,QAAU,EAAc,CACjG,GAAI,CAAC,EACH,MAAU,MAAM,0CAA0C,EAAc,GAAG,CAI7E,IAAI,EAAc,MAAM,EAAiB,EAAQ,CAC/C,QAAS,EACT,kBACA,iBACA,kBACA,iBACA,SAAU,GACV,SAAU,GACV,aACD,CAAC,CAGF,EAAc,EAAY,OACvB,GAAY,OAAO,EAAQ,eAAkB,UAAY,EAAQ,cAAgB,EACnF,CAGD,EAAO,KAAK,EAAO,QAAQ,cAAc,EAAY,OAAO,6BAA6B,CAAC,CAE1F,IAAI,EAAQ,EACZ,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,MAAM,EACJ,EACA,KAAO,IAAoB,CACzB,MAAM,EAAmB,EAAQ,EAAwB,CACvD,UAAW,CACT,MAAO,CACL,UAAW,EAAgB,GAC3B,SAAU,EAAmB,SAAS,eACtC,QAAS,EAAmB,QAAQ,eACpC,eAAgB,EACjB,CACF,CACD,SACD,CAAC,CAEF,GAAS,EACT,EAAY,OAAO,EAAM,EAE3B,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAM,+BAA+B,EAAY,IAAK,YAC9E,CACF,CACM,EAAY"}

package/dist/{parquetToCsvOneFile-0O4iLz0t.mjs → parquetToCsvOneFile-Dtz90qfJ.mjs} RENAMED Viewed

@@ -1,6 +1,6 @@
-import{t as e}from"./logger-Ckep4QXn.mjs";import{existsSync as t,mkdirSync as n,rmSync as r}from"node:fs";import{dirname as i,join as a,parse as o}from"node:path";import s from"colors";async function c(c,f){let{filePath:p,outputDir:m,clearOutputDir:h,onProgress:g}=c,_=m||i(p),{name:v}=o(p),y=a(_,`${v}.csv`);if(n(_,{recursive:!0}),h&&t(y))try{r(y,{force:!0})}catch(t){e.warn(s.yellow(`Could not remove existing output file ${y}: ${t.message}`))}let b=await f.create(`:memory:`),x=await b.connect();try{await u(x,`PRAGMA temp_directory='';`);let t=e=>`'${e.replace(/'/g,`''`)}'`;await l(x,`
+import{t as e}from"./logger-CB_h_zbj.mjs";import{existsSync as t,mkdirSync as n,rmSync as r}from"node:fs";import{dirname as i,join as a,parse as o}from"node:path";import s from"colors";async function c(c,f){let{filePath:p,outputDir:m,clearOutputDir:h,onProgress:g}=c,_=m||i(p),{name:v}=o(p),y=a(_,`${v}.csv`);if(n(_,{recursive:!0}),h&&t(y))try{r(y,{force:!0})}catch(t){e.warn(s.yellow(`Could not remove existing output file ${y}: ${t.message}`))}let b=await f.create(`:memory:`),x=await b.connect();try{await u(x,`PRAGMA temp_directory='';`);let t=e=>`'${e.replace(/'/g,`''`)}'`;await l(x,`
       COPY (SELECT * FROM read_parquet(${t(p)}))
       TO ${t(y)}
       (HEADER, DELIMITER ',', QUOTE '"', ESCAPE '"', NULL '');
     `),g?.(0,void 0),e.info(s.green(`Wrote CSV → ${y}`))}finally{await d(x),await d(b)}}async function l(e,t){await d(await e.run(t))}async function u(e,t){try{await l(e,t)}catch{}}async function d(e){if(!(!e||typeof e.dispose!=`function`))try{await e.dispose()}catch{}}export{c as t};
-//# sourceMappingURL=parquetToCsvOneFile-0O4iLz0t.mjs.map
+//# sourceMappingURL=parquetToCsvOneFile-Dtz90qfJ.mjs.map

package/dist/{parquetToCsvOneFile-0O4iLz0t.mjs.map → parquetToCsvOneFile-Dtz90qfJ.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parquetToCsvOneFile-~~0O4iLz0t~~.mjs","names":[],"sources":["../src/lib/helpers/parquetToCsvOneFile.ts"],"sourcesContent":["import { mkdirSync, rmSync, existsSync } from 'node:fs';\nimport { dirname, join, parse } from 'node:path';\n\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport colors from 'colors';\n\nimport { logger } from '../../logger.js';\n\n/** Progress callback used by the parent runner to surface progress to the UI. /\ntype OnProgress = (processed: number, total?: number) => void;\n\n/\n Options for converting a single Parquet file into a single CSV file.\n /\nexport type ParquetToCsvOneFileOptions = {\n /* Absolute or relative path to the input `.parquet` file. /\n filePath: string;\n /\n Directory where the output CSV will be written.\n * If omitted, the CSV is written next to the input file.\n /\n outputDir?: string;\n /\n When true, removes a pre-existing output file with the same name before writing.\n * Useful for re-runs; ignored if the file does not exist.\n /\n clearOutputDir: boolean;\n /\n Optional progress hook. Called with the number of processed records.\n * `total` is not computed here; it will be `undefined`.\n /\n onProgress?: OnProgress;\n};\n\n/\n Convert a single Parquet file to a single CSV file (1:1) using DuckDB.\n \n Output naming: `${basename}.csv` in `outputDir ?? dirname(filePath)`.\n \n Errors:\n * - Throws on I/O failures or DuckDB execution errors.\n \n Why DuckDB?\n * - Robust reader for many Parquet dialects (e.g., Spark output, nested types, timestamps).\n * - Streaming COPY handles large files without loading everything into JS memory.\n \n What this does:\n * - Opens an in-memory DuckDB database (no `.db` file created).\n * - Optionally disables temp spilling to disk (so only your CSV is written).\n * - Executes a single `COPY (SELECT * FROM read_parquet(...)) TO ...` statement.\n * - Produces exactly one CSV per input Parquet (no chunking or rotation).\n \n Notes & defaults:\n * - DuckDBInstance: `:memory:` (ephemeral). No persistent DB file is created.\n * - Temp files: disabled via `PRAGMA temp_directory=''` (best-effort; ignored if unsupported).\n * - CSV format: header row, comma delimiter, double-quote quoting, empty string for NULL.\n * - Progress: DuckDB COPY doesn't expose row-level progress via the JS API; we emit a\n * best-effort final callback.\n \n Requirements:\n * - `@duckdb/node-api` npm package installed and available at runtime.\n * - Supported platform binary (mac arm64/x64, linux x64, windows x64).\n \n @param opts - Conversion options\n * @param DuckDb - DuckDB instance to use\n * @returns Promise<void> when the CSV has been written\n /\nexport async function parquetToCsvOneFile(\n opts: ParquetToCsvOneFileOptions,\n DuckDb: typeof DuckDBInstance,\n): Promise<void> {\n const { filePath, outputDir, clearOutputDir, onProgress } = opts;\n\n const baseDir = outputDir \|\| dirname(filePath);\n const { name: baseName } = parse(filePath);\n const outPath = join(baseDir, `${baseName}.csv`);\n\n // Ensure output directory exists\n mkdirSync(baseDir, { recursive: true });\n\n // Remove any pre-existing output file if requested\n if (clearOutputDir && existsSync(outPath)) {\n try {\n rmSync(outPath, { force: true });\n } catch (err) {\n logger.warn(\n colors.yellow(\n `Could not remove existing output file ${outPath}: ${(err as Error).message}`,\n ),\n );\n }\n }\n\n // In-memory DB: no .db file created on disk\n const db = await DuckDb.create(':memory:');\n const conn = await db.connect();\n\n try {\n // Optional: prevent DuckDB from creating temp files on disk (best-effort).\n // Some versions may ignore or error; we ignore such errors safely.\n await runIgnoreError(conn, \"PRAGMA temp_directory='';\");\n\n // Optionally: cap memory to encourage in-memory execution or fail-fast\n // (commented out by default; uncomment to enforce a limit)\n // await runIgnoreError(conn, \"PRAGMA memory_limit='4GB';\");\n\n // Ensure stable CSV settings: header, comma delimiter, double quotes, empty string for NULLs.\n // Escape single quotes for SQL string literals\n const q = (p: string): string => `'${p.replace(/'/g, \"''\")}'`;\n\n // Use COPY with a subquery so DuckDB streams Parquet -> CSV efficiently.\n const sql = `\n COPY (SELECT FROM read_parquet(${q(filePath)}))\n TO ${q(outPath)}\n (HEADER, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '');\n `;\n\n await run(conn, sql);\n\n // Best-effort progress notification (DuckDB JS API doesn't expose progress for COPY)\n onProgress?.(0, undefined);\n\n logger.info(colors.green(`Wrote CSV → ${outPath}`));\n } finally {\n // Close connection + db handles gracefully\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(conn as any);\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(db as any);\n }\n}\n\n/* =============================================================================\n * DuckDB helpers\n * =============================================================================\n /\n\n/\n Execute a SQL statement on a DuckDB connection and dispose the result.\n \n @param conn - DuckDB connection\n * @param sql - SQL string to run\n * @returns Promise<void>\n /\nasync function run(conn: DuckDBConnection, sql: string): Promise<void> {\n const result = await conn.run(sql);\n // The high-level API returns a Result; ensure we dispose it to free buffers.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(result as any);\n}\n\n/\n Execute a SQL statement but ignore any error that occurs.\n * Useful for best-effort PRAGMAs that may not be supported across versions.\n \n @param conn - DuckDB connection\n * @param sql - SQL string to run\n * @returns Promise<void>\n /\nasync function runIgnoreError(conn: DuckDBConnection, sql: string): Promise<void> {\n try {\n await run(conn, sql);\n } catch {\n // ignore\n }\n}\n\n/\n Dispose a DuckDB resource (connection or instance) if present.\n \n @param handle - Object exposing an async `dispose()` method\n * @returns Promise<void>\n /\nasync function disposeSafe(\n handle:\n \| {\n /* Dispose handler */\n dispose: () => Promise<void>;\n }\n \| null\n \| undefined,\n): Promise<void> {\n if (!handle \|\| typeof handle.dispose !== 'function') return;\n try {\n await handle.dispose();\n } catch {\n // ignore\n }\n}\n"],"mappings":"yLAmEA,eAAsB,EACpB,EACA,EACe,CACf,GAAM,CAAE,WAAU,YAAW,iBAAgB,cAAe,EAEtD,EAAU,GAAa,EAAQ,EAAS,CACxC,CAAE,KAAM,GAAa,EAAM,EAAS,CACpC,EAAU,EAAK,EAAS,GAAG,EAAS,MAAM,CAMhD,GAHA,EAAU,EAAS,CAAE,UAAW,GAAM,CAAC,CAGnC,GAAkB,EAAW,EAAQ,CACvC,GAAI,CACF,EAAO,EAAS,CAAE,MAAO,GAAM,CAAC,OACzB,EAAK,CACZ,EAAO,KACL,EAAO,OACL,yCAAyC,EAAQ,IAAK,EAAc,UACrE,CACF,CAKL,IAAM,EAAK,MAAM,EAAO,OAAO,WAAW,CACpC,EAAO,MAAM,EAAG,SAAS,CAE/B,GAAI,CAGF,MAAM,EAAe,EAAM,4BAA4B,CAQvD,IAAM,EAAK,GAAsB,IAAI,EAAE,QAAQ,KAAM,KAAK,CAAC,GAS3D,MAAM,EAAI,EAAM;yCALqB,EAAE,EAAS,CAAC;WAC1C,EAAE,EAAQ,CAAC;;MAIE,CAGpB,IAAa,EAAG,IAAA,GAAU,CAE1B,EAAO,KAAK,EAAO,MAAM,eAAe,IAAU,CAAC,QAC3C,CAGR,MAAM,EAAY,EAAY,CAE9B,MAAM,EAAY,EAAU,EAgBhC,eAAe,EAAI,EAAwB,EAA4B,CAIrE,MAAM,EAAY,MAHG,EAAK,IAAI,EAAI,CAGF,CAWlC,eAAe,EAAe,EAAwB,EAA4B,CAChF,GAAI,CACF,MAAM,EAAI,EAAM,EAAI,MACd,GAWV,eAAe,EACb,EAOe,CACX,MAAC,GAAU,OAAO,EAAO,SAAY,YACzC,GAAI,CACF,MAAM,EAAO,SAAS,MAChB"}
1	+ {"version":3,"file":"parquetToCsvOneFile-Dtz90qfJ.mjs","names":[],"sources":["../src/lib/helpers/parquetToCsvOneFile.ts"],"sourcesContent":["import { mkdirSync, rmSync, existsSync } from 'node:fs';\nimport { dirname, join, parse } from 'node:path';\n\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport colors from 'colors';\n\nimport { logger } from '../../logger.js';\n\n/** Progress callback used by the parent runner to surface progress to the UI. /\ntype OnProgress = (processed: number, total?: number) => void;\n\n/\n Options for converting a single Parquet file into a single CSV file.\n /\nexport type ParquetToCsvOneFileOptions = {\n /* Absolute or relative path to the input `.parquet` file. /\n filePath: string;\n /\n Directory where the output CSV will be written.\n * If omitted, the CSV is written next to the input file.\n /\n outputDir?: string;\n /\n When true, removes a pre-existing output file with the same name before writing.\n * Useful for re-runs; ignored if the file does not exist.\n /\n clearOutputDir: boolean;\n /\n Optional progress hook. Called with the number of processed records.\n * `total` is not computed here; it will be `undefined`.\n /\n onProgress?: OnProgress;\n};\n\n/\n Convert a single Parquet file to a single CSV file (1:1) using DuckDB.\n \n Output naming: `${basename}.csv` in `outputDir ?? dirname(filePath)`.\n \n Errors:\n * - Throws on I/O failures or DuckDB execution errors.\n \n Why DuckDB?\n * - Robust reader for many Parquet dialects (e.g., Spark output, nested types, timestamps).\n * - Streaming COPY handles large files without loading everything into JS memory.\n \n What this does:\n * - Opens an in-memory DuckDB database (no `.db` file created).\n * - Optionally disables temp spilling to disk (so only your CSV is written).\n * - Executes a single `COPY (SELECT * FROM read_parquet(...)) TO ...` statement.\n * - Produces exactly one CSV per input Parquet (no chunking or rotation).\n \n Notes & defaults:\n * - DuckDBInstance: `:memory:` (ephemeral). No persistent DB file is created.\n * - Temp files: disabled via `PRAGMA temp_directory=''` (best-effort; ignored if unsupported).\n * - CSV format: header row, comma delimiter, double-quote quoting, empty string for NULL.\n * - Progress: DuckDB COPY doesn't expose row-level progress via the JS API; we emit a\n * best-effort final callback.\n \n Requirements:\n * - `@duckdb/node-api` npm package installed and available at runtime.\n * - Supported platform binary (mac arm64/x64, linux x64, windows x64).\n \n @param opts - Conversion options\n * @param DuckDb - DuckDB instance to use\n * @returns Promise<void> when the CSV has been written\n /\nexport async function parquetToCsvOneFile(\n opts: ParquetToCsvOneFileOptions,\n DuckDb: typeof DuckDBInstance,\n): Promise<void> {\n const { filePath, outputDir, clearOutputDir, onProgress } = opts;\n\n const baseDir = outputDir \|\| dirname(filePath);\n const { name: baseName } = parse(filePath);\n const outPath = join(baseDir, `${baseName}.csv`);\n\n // Ensure output directory exists\n mkdirSync(baseDir, { recursive: true });\n\n // Remove any pre-existing output file if requested\n if (clearOutputDir && existsSync(outPath)) {\n try {\n rmSync(outPath, { force: true });\n } catch (err) {\n logger.warn(\n colors.yellow(\n `Could not remove existing output file ${outPath}: ${(err as Error).message}`,\n ),\n );\n }\n }\n\n // In-memory DB: no .db file created on disk\n const db = await DuckDb.create(':memory:');\n const conn = await db.connect();\n\n try {\n // Optional: prevent DuckDB from creating temp files on disk (best-effort).\n // Some versions may ignore or error; we ignore such errors safely.\n await runIgnoreError(conn, \"PRAGMA temp_directory='';\");\n\n // Optionally: cap memory to encourage in-memory execution or fail-fast\n // (commented out by default; uncomment to enforce a limit)\n // await runIgnoreError(conn, \"PRAGMA memory_limit='4GB';\");\n\n // Ensure stable CSV settings: header, comma delimiter, double quotes, empty string for NULLs.\n // Escape single quotes for SQL string literals\n const q = (p: string): string => `'${p.replace(/'/g, \"''\")}'`;\n\n // Use COPY with a subquery so DuckDB streams Parquet -> CSV efficiently.\n const sql = `\n COPY (SELECT FROM read_parquet(${q(filePath)}))\n TO ${q(outPath)}\n (HEADER, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '');\n `;\n\n await run(conn, sql);\n\n // Best-effort progress notification (DuckDB JS API doesn't expose progress for COPY)\n onProgress?.(0, undefined);\n\n logger.info(colors.green(`Wrote CSV → ${outPath}`));\n } finally {\n // Close connection + db handles gracefully\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(conn as any);\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(db as any);\n }\n}\n\n/* =============================================================================\n * DuckDB helpers\n * =============================================================================\n /\n\n/\n Execute a SQL statement on a DuckDB connection and dispose the result.\n \n @param conn - DuckDB connection\n * @param sql - SQL string to run\n * @returns Promise<void>\n /\nasync function run(conn: DuckDBConnection, sql: string): Promise<void> {\n const result = await conn.run(sql);\n // The high-level API returns a Result; ensure we dispose it to free buffers.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n await disposeSafe(result as any);\n}\n\n/\n Execute a SQL statement but ignore any error that occurs.\n * Useful for best-effort PRAGMAs that may not be supported across versions.\n \n @param conn - DuckDB connection\n * @param sql - SQL string to run\n * @returns Promise<void>\n /\nasync function runIgnoreError(conn: DuckDBConnection, sql: string): Promise<void> {\n try {\n await run(conn, sql);\n } catch {\n // ignore\n }\n}\n\n/\n Dispose a DuckDB resource (connection or instance) if present.\n \n @param handle - Object exposing an async `dispose()` method\n * @returns Promise<void>\n /\nasync function disposeSafe(\n handle:\n \| {\n /* Dispose handler */\n dispose: () => Promise<void>;\n }\n \| null\n \| undefined,\n): Promise<void> {\n if (!handle \|\| typeof handle.dispose !== 'function') return;\n try {\n await handle.dispose();\n } catch {\n // ignore\n }\n}\n"],"mappings":"yLAmEA,eAAsB,EACpB,EACA,EACe,CACf,GAAM,CAAE,WAAU,YAAW,iBAAgB,cAAe,EAEtD,EAAU,GAAa,EAAQ,EAAS,CACxC,CAAE,KAAM,GAAa,EAAM,EAAS,CACpC,EAAU,EAAK,EAAS,GAAG,EAAS,MAAM,CAMhD,GAHA,EAAU,EAAS,CAAE,UAAW,GAAM,CAAC,CAGnC,GAAkB,EAAW,EAAQ,CACvC,GAAI,CACF,EAAO,EAAS,CAAE,MAAO,GAAM,CAAC,OACzB,EAAK,CACZ,EAAO,KACL,EAAO,OACL,yCAAyC,EAAQ,IAAK,EAAc,UACrE,CACF,CAKL,IAAM,EAAK,MAAM,EAAO,OAAO,WAAW,CACpC,EAAO,MAAM,EAAG,SAAS,CAE/B,GAAI,CAGF,MAAM,EAAe,EAAM,4BAA4B,CAQvD,IAAM,EAAK,GAAsB,IAAI,EAAE,QAAQ,KAAM,KAAK,CAAC,GAS3D,MAAM,EAAI,EAAM;yCALqB,EAAE,EAAS,CAAC;WAC1C,EAAE,EAAQ,CAAC;;MAIE,CAGpB,IAAa,EAAG,IAAA,GAAU,CAE1B,EAAO,KAAK,EAAO,MAAM,eAAe,IAAU,CAAC,QAC3C,CAGR,MAAM,EAAY,EAAY,CAE9B,MAAM,EAAY,EAAU,EAgBhC,eAAe,EAAI,EAAwB,EAA4B,CAIrE,MAAM,EAAY,MAHG,EAAK,IAAI,EAAI,CAGF,CAWlC,eAAe,EAAe,EAAwB,EAA4B,CAChF,GAAI,CACF,MAAM,EAAI,EAAM,EAAI,MACd,GAWV,eAAe,EACb,EAOe,CACX,MAAC,GAAU,OAAO,EAAO,SAAY,YACzC,GAAI,CACF,MAAM,EAAO,SAAS,MAChB"}

package/dist/{parseAttributesFromString-CYcxsS3n.mjs → parseAttributesFromString-BxJcnKmn.mjs} RENAMED Viewed

@@ -1,2 +1,2 @@
-import{t as e}from"./logger-Ckep4QXn.mjs";import t from"colors";import*as n from"io-ts";const r=n.type({key:n.string,values:n.array(n.string)});function i(n){let r=n.map(e=>{let[t,n]=e.trim().split(`:`);if(!n)throw Error(`Expected attributes in key:value1;value2,key2:value3;value4`);return{key:t,values:n.split(`;`)}});return e.info(t.magenta(`Attributes to apply to all requests:`)),e.info(t.magenta(JSON.stringify(r,null,2))),r}export{i as n,r as t};
-//# sourceMappingURL=parseAttributesFromString-CYcxsS3n.mjs.map
+import{t as e}from"./logger-CB_h_zbj.mjs";import t from"colors";import*as n from"io-ts";const r=n.type({key:n.string,values:n.array(n.string)});function i(n){let r=n.map(e=>{let[t,n]=e.trim().split(`:`);if(!n)throw Error(`Expected attributes in key:value1;value2,key2:value3;value4`);return{key:t,values:n.split(`;`)}});return e.info(t.magenta(`Attributes to apply to all requests:`)),e.info(t.magenta(JSON.stringify(r,null,2))),r}export{i as n,r as t};
+//# sourceMappingURL=parseAttributesFromString-BxJcnKmn.mjs.map

package/dist/{parseAttributesFromString-CYcxsS3n.mjs.map → parseAttributesFromString-BxJcnKmn.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parseAttributesFromString-~~CYcxsS3n~~.mjs","names":[],"sources":["../src/lib/requests/parseAttributesFromString.ts"],"sourcesContent":["import colors from 'colors';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\nexport const ParsedAttributeInput = t.type({\n /** Attribute key /\n key: t.string,\n /* Attribute values /\n values: t.array(t.string),\n});\n\n/* Type override /\nexport type ParsedAttributeInput = t.TypeOf<typeof ParsedAttributeInput>;\n\n/\n Parse out the extra attributes to apply to all requests uploaded\n \n @param attributes - input as string, e.g. ['key:value1;value2','key2:value3;value4']\n * @returns The parsed attributes\n */\nexport function parseAttributesFromString(attributes: string[]): ParsedAttributeInput[] {\n // Parse out the extra attributes to apply to all requests uploaded\n const parsedAttributes = attributes.map((attribute) => {\n const [attributeKey, attributeValuesRaw] = attribute.trim().split(':');\n if (!attributeValuesRaw) {\n throw new Error('Expected attributes in key:value1;value2,key2:value3;value4');\n }\n const attributeValues = attributeValuesRaw.split(';');\n return {\n key: attributeKey,\n values: attributeValues,\n };\n });\n logger.info(colors.magenta('Attributes to apply to all requests:'));\n logger.info(colors.magenta(JSON.stringify(parsedAttributes, null, 2)));\n return parsedAttributes;\n}\n"],"mappings":"wFAKA,MAAa,EAAuB,EAAE,KAAK,CAEzC,IAAK,EAAE,OAEP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CAWF,SAAgB,EAA0B,EAA8C,CAEtF,IAAM,EAAmB,EAAW,IAAK,GAAc,CACrD,GAAM,CAAC,EAAc,GAAsB,EAAU,MAAM,CAAC,MAAM,IAAI,CACtE,GAAI,CAAC,EACH,MAAU,MAAM,8DAA8D,CAGhF,MAAO,CACL,IAAK,EACL,OAHsB,EAAmB,MAAM,IAGxB,CACxB,EACD,CAGF,OAFA,EAAO,KAAK,EAAO,QAAQ,uCAAuC,CAAC,CACnE,EAAO,KAAK,EAAO,QAAQ,KAAK,UAAU,EAAkB,KAAM,EAAE,CAAC,CAAC,CAC/D"}
1	+ {"version":3,"file":"parseAttributesFromString-BxJcnKmn.mjs","names":[],"sources":["../src/lib/requests/parseAttributesFromString.ts"],"sourcesContent":["import colors from 'colors';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\nexport const ParsedAttributeInput = t.type({\n /** Attribute key /\n key: t.string,\n /* Attribute values /\n values: t.array(t.string),\n});\n\n/* Type override /\nexport type ParsedAttributeInput = t.TypeOf<typeof ParsedAttributeInput>;\n\n/\n Parse out the extra attributes to apply to all requests uploaded\n \n @param attributes - input as string, e.g. ['key:value1;value2','key2:value3;value4']\n * @returns The parsed attributes\n */\nexport function parseAttributesFromString(attributes: string[]): ParsedAttributeInput[] {\n // Parse out the extra attributes to apply to all requests uploaded\n const parsedAttributes = attributes.map((attribute) => {\n const [attributeKey, attributeValuesRaw] = attribute.trim().split(':');\n if (!attributeValuesRaw) {\n throw new Error('Expected attributes in key:value1;value2,key2:value3;value4');\n }\n const attributeValues = attributeValuesRaw.split(';');\n return {\n key: attributeKey,\n values: attributeValues,\n };\n });\n logger.info(colors.magenta('Attributes to apply to all requests:'));\n logger.info(colors.magenta(JSON.stringify(parsedAttributes, null, 2)));\n return parsedAttributes;\n}\n"],"mappings":"wFAKA,MAAa,EAAuB,EAAE,KAAK,CAEzC,IAAK,EAAE,OAEP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CAWF,SAAgB,EAA0B,EAA8C,CAEtF,IAAM,EAAmB,EAAW,IAAK,GAAc,CACrD,GAAM,CAAC,EAAc,GAAsB,EAAU,MAAM,CAAC,MAAM,IAAI,CACtE,GAAI,CAAC,EACH,MAAU,MAAM,8DAA8D,CAGhF,MAAO,CACL,IAAK,EACL,OAHsB,EAAmB,MAAM,IAGxB,CACxB,EACD,CAGF,OAFA,EAAO,KAAK,EAAO,QAAQ,uCAAuC,CAAC,CACnE,EAAO,KAAK,EAAO,QAAQ,KAAK,UAAU,EAAkB,KAAM,EAAE,CAAC,CAAC,CAC/D"}

package/dist/{parseVariablesFromString-B4TUkJRF.mjs → parseVariablesFromString-cu5gPC3M.mjs} RENAMED Viewed

@@ -1,2 +1,2 @@
 function e(e){let t=e.split(`,`).filter(e=>!!e),n={};return t.forEach(e=>{let[t,r]=e.split(`:`);if(!t||!r)throw Error(`Invalid variable: ${e}. Expected format: key:value`);n[t]=r}),n}export{e as t};
-//# sourceMappingURL=parseVariablesFromString-B4TUkJRF.mjs.map
+//# sourceMappingURL=parseVariablesFromString-cu5gPC3M.mjs.map

package/dist/{parseVariablesFromString-B4TUkJRF.mjs.map → parseVariablesFromString-cu5gPC3M.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parseVariablesFromString-~~B4TUkJRF~~.mjs","names":[],"sources":["../src/lib/helpers/parseVariablesFromString.ts"],"sourcesContent":["/*\n Parse variables from string\n \n @param variables - Variables as string\n * @returns Variables as object\n */\nexport function parseVariablesFromString(variables: string): {\n [k in string]: string;\n} {\n // Parse out the variables\n const splitVars = variables.split(',').filter((x) => !!x);\n const vars: { [k in string]: string } = {};\n splitVars.forEach((variable) => {\n const [k, v] = variable.split(':');\n if (!k \|\| !v) {\n throw new Error(`Invalid variable: ${variable}. Expected format: key:value`);\n }\n vars[k] = v;\n });\n return vars;\n}\n"],"mappings":"AAMA,SAAgB,EAAyB,EAEvC,CAEA,IAAM,EAAY,EAAU,MAAM,IAAI,CAAC,OAAQ,GAAM,CAAC,CAAC,EAAE,CACnD,EAAkC,EAAE,CAQ1C,OAPA,EAAU,QAAS,GAAa,CAC9B,GAAM,CAAC,EAAG,GAAK,EAAS,MAAM,IAAI,CAClC,GAAI,CAAC,GAAK,CAAC,EACT,MAAU,MAAM,qBAAqB,EAAS,8BAA8B,CAE9E,EAAK,GAAK,GACV,CACK"}
1	+ {"version":3,"file":"parseVariablesFromString-cu5gPC3M.mjs","names":[],"sources":["../src/lib/helpers/parseVariablesFromString.ts"],"sourcesContent":["/*\n Parse variables from string\n \n @param variables - Variables as string\n * @returns Variables as object\n */\nexport function parseVariablesFromString(variables: string): {\n [k in string]: string;\n} {\n // Parse out the variables\n const splitVars = variables.split(',').filter((x) => !!x);\n const vars: { [k in string]: string } = {};\n splitVars.forEach((variable) => {\n const [k, v] = variable.split(':');\n if (!k \|\| !v) {\n throw new Error(`Invalid variable: ${variable}. Expected format: key:value`);\n }\n vars[k] = v;\n });\n return vars;\n}\n"],"mappings":"AAMA,SAAgB,EAAyB,EAEvC,CAEA,IAAM,EAAY,EAAU,MAAM,IAAI,CAAC,OAAQ,GAAM,CAAC,CAAC,EAAE,CACnD,EAAkC,EAAE,CAQ1C,OAPA,EAAU,QAAS,GAAa,CAC9B,GAAM,CAAC,EAAG,GAAK,EAAS,MAAM,IAAI,CAClC,GAAI,CAAC,GAAK,CAAC,EACT,MAAU,MAAM,qBAAqB,EAAS,8BAA8B,CAE9E,EAAK,GAAK,GACV,CACK"}

package/dist/{pullAllDatapoints-DJno8LkB.mjs → pullAllDatapoints-XqreMBT5.mjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-import{t as e}from"./logger-Ckep4QXn.mjs";import{SubDataPointDataSubCategoryGuessStatus as t}from"@transcend-io/privacy-types";import{chunk as n,keyBy as r,sortBy as i,uniq as a}from"lodash-es";import o from"colors";import{DATAPOINT_EXPORT as s,DATA_SILO_EXPORT as c,SUB_DATA_POINTS_COUNT as l,makeGraphQLRequest as u}from"@transcend-io/sdk";import{mapSeries as d}from"@transcend-io/utils";import f from"cli-progress";import{gql as p}from"graphql-request";async function m(n,{dataSiloIds:r=[],includeGuessedCategories:a,includeAttributes:s,parentCategories:c=[],subCategories:d=[],pageSize:m=1e3}={}){let h=[],g=new Date().getTime(),_=new f.SingleBar({},f.Presets.shades_classic),v={...c.length>0?{category:c}:{},...d.length>0?{subCategoryIds:d}:{},...c.length+d.length>0&&!a?{status:t.Approved}:{},...r.length>0?{dataSilos:r}:{}},{subDataPoints:{totalCount:y}}=await u(n,l,{variables:{filterBy:v},logger:e});e.info(o.magenta(`[Step 1/3] Pulling in all subdatapoints`)),_.start(y,0);let b=0,x=!1,S,C=0;do try{let{subDataPoints:{nodes:t}}=await u(n,p`
+import{t as e}from"./logger-CB_h_zbj.mjs";import{SubDataPointDataSubCategoryGuessStatus as t}from"@transcend-io/privacy-types";import{chunk as n,keyBy as r,sortBy as i,uniq as a}from"lodash-es";import o from"colors";import{DATAPOINT_EXPORT as s,DATA_SILO_EXPORT as c,SUB_DATA_POINTS_COUNT as l,makeGraphQLRequest as u}from"@transcend-io/sdk";import{mapSeries as d}from"@transcend-io/utils";import f from"cli-progress";import{gql as p}from"graphql-request";async function m(n,{dataSiloIds:r=[],includeGuessedCategories:a,includeAttributes:s,parentCategories:c=[],subCategories:d=[],pageSize:m=1e3}={}){let h=[],g=new Date().getTime(),_=new f.SingleBar({},f.Presets.shades_classic),v={...c.length>0?{category:c}:{},...d.length>0?{subCategoryIds:d}:{},...c.length+d.length>0&&!a?{status:t.Approved}:{},...r.length>0?{dataSilos:r}:{}},{subDataPoints:{totalCount:y}}=await u(n,l,{variables:{filterBy:v},logger:e});e.info(o.magenta(`[Step 1/3] Pulling in all subdatapoints`)),_.start(y,0);let b=0,x=!1,S,C=0;do try{let{subDataPoints:{nodes:t}}=await u(n,p`
           query TranscendCliSubDataPointCsvExport(
             $filterBy: SubDataPointFiltersInput
             $first: Int!
@@ -42,4 +42,4 @@ import{t as e}from"./logger-Ckep4QXn.mjs";import{SubDataPointDataSubCategoryGues
             }
           }
         `,{variables:{first:m,offset:C,filterBy:{...v}},logger:e});S=t[t.length-1]?.id,h.push(...t),x=t.length===m,b+=t.length,C+=t.length,_.update(b)}catch(t){throw e.error(o.red(`An error fetching subdatapoints for cursor ${S} and offset ${C}`)),t}while(x);_.stop();let w=new Date().getTime()-g,T=i(h,`name`);return e.info(o.green(`Successfully pulled in ${T.length} subdatapoints in ${w/1e3} seconds!`)),T}async function h(t,{dataPointIds:r=[],pageSize:i=100}){let a=[],c=new Date().getTime(),l=new f.SingleBar({},f.Presets.shades_classic);e.info(o.magenta(`[Step 2/3] Fetching metadata for ${r.length} datapoints`));let p=n(r,i);l.start(r.length,0);let m=0;await d(p,async n=>{try{let{dataPoints:{nodes:r}}=await u(t,s,{variables:{first:i,filterBy:{ids:n}},logger:e});a.push(...r),m+=n.length,l.update(m)}catch(t){throw e.error(o.red(`An error fetching subdatapoints for IDs ${n.join(`, `)}`)),t}}),l.stop();let h=new Date().getTime()-c;return e.info(o.green(`Successfully pulled in ${a.length} dataPoints in ${h/1e3} seconds!`)),a}async function g(t,{dataSiloIds:r=[],pageSize:i=100}){let a=[],s=new Date().getTime(),l=new f.SingleBar({},f.Presets.shades_classic);e.info(o.magenta(`[Step 3/3] Fetching metadata for ${r.length} data silos`));let p=n(r,i);l.start(r.length,0);let m=0;await d(p,async n=>{try{let{dataSilos:{nodes:r}}=await u(t,c,{variables:{first:i,filterBy:{ids:n}},logger:e});a.push(...r),m+=n.length,l.update(m)}catch(t){throw e.error(o.red(`An error fetching data silos for IDs ${n.join(`, `)}`)),t}}),l.stop();let h=new Date().getTime()-s;return e.info(o.green(`Successfully pulled in ${a.length} data silos in ${h/1e3} seconds!`)),a}async function _(e,{dataSiloIds:t=[],includeGuessedCategories:n,includeAttributes:i,parentCategories:o=[],subCategories:s=[],pageSize:c=1e3}={}){let l=await m(e,{dataSiloIds:t,includeGuessedCategories:n,includeAttributes:i,parentCategories:o,subCategories:s,pageSize:c}),u=r(await h(e,{dataPointIds:a(l.map(e=>e.dataPointId))}),`id`),d=r(await g(e,{dataSiloIds:a(l.map(e=>e.dataSiloId))}),`id`);return l.map(e=>({...e,dataPoint:u[e.dataPointId],dataSilo:d[e.dataSiloId]}))}export{_ as t};
-//# sourceMappingURL=pullAllDatapoints-DJno8LkB.mjs.map
+//# sourceMappingURL=pullAllDatapoints-XqreMBT5.mjs.map

package/dist/{pullAllDatapoints-DJno8LkB.mjs.map → pullAllDatapoints-XqreMBT5.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"pullAllDatapoints-DJno8LkB.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport {\n makeGraphQLRequest,\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n SUB_DATA_POINTS_COUNT,\n type DataSiloAttributeValue,\n} from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n variables: { filterBy },\n logger,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n variables: {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n logger,\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n variables: {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n },\n logger,\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n variables: {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n },\n logger,\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines */\n"],"mappings":"wcA0FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,UAAW,CAAE,WAAU,CACvB,SACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,UAAW,CACT,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACD,SACD,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAAM,MAHH,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAE3D,CACb,CAAC,CACsC,KAAK,CAOvC,EAAe,EAAM,MAHH,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAE7C,CAC5B,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}
1	+ {"version":3,"file":"pullAllDatapoints-XqreMBT5.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n type DataCategoryType,\n SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport {\n makeGraphQLRequest,\n DATAPOINT_EXPORT,\n DATA_SILO_EXPORT,\n SUB_DATA_POINTS_COUNT,\n type DataSiloAttributeValue,\n} from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines /\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nexport interface DataSiloCsvPreview {\n /* ID of dataSilo /\n id: string;\n /* Name of dataSilo /\n title: string;\n}\n\nexport interface DataPointCsvPreview {\n /* ID of dataPoint /\n id: string;\n /* The path to this data point /\n path: string[];\n /* Description /\n description: {\n /* Default message /\n defaultMessage: string;\n };\n /* Name /\n name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n /* ID of subDatapoint /\n id: string;\n /* Name (or key) of the subdatapoint /\n name: string;\n /* The description /\n description?: string;\n /* Personal data category /\n categories: DataCategoryInput[];\n /* Data point ID /\n dataPointId: string;\n /* The data silo ID /\n dataSiloId: string;\n /* The processing purpose for this sub datapoint /\n purposes: ProcessingPurposeInput[];\n /* Attribute attached to subdatapoint /\n attributeValues?: DataSiloAttributeValue[];\n /* Data category guesses that are output by the classifier /\n pendingCategoryGuesses?: {\n /* Data category being guessed /\n category: DataCategoryInput;\n /* Status of guess /\n status: SubDataPointDataSubCategoryGuessStatus;\n /* classifier version that produced the guess /\n classifierVersion: number;\n }[];\n}\n\nexport interface DatapointFilterOptions {\n /* IDs of data silos to filter down /\n dataSiloIds?: string[];\n /* Whether to include guessed categories, defaults to only approved categories /\n includeGuessedCategories?: boolean;\n /* Whether or not to include attributes /\n includeAttributes?: boolean;\n /* Parent categories to filter down for /\n parentCategories?: DataCategoryType[];\n /* Sub categories to filter down for /\n subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/\n Pull subdatapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n /\nasync function pullSubDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<SubDataPointCsvPreview[]> {\n const subDataPoints: SubDataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n // Filters\n const filterBy = {\n ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n // if parentCategories or subCategories and not includeGuessedCategories\n ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n ? // then only show data points with approved data categories\n { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n : {}),\n ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n };\n\n // Build a GraphQL client\n const {\n subDataPoints: { totalCount },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* Count /\n totalCount: number;\n };\n }>(client, SUB_DATA_POINTS_COUNT, {\n variables: { filterBy },\n logger,\n });\n\n logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n progressBar.start(totalCount, 0);\n let total = 0;\n let shouldContinue = false;\n let cursor: string \| undefined;\n let offset = 0;\n do {\n try {\n const {\n subDataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n subDataPoints: {\n /* List of matches /\n nodes: SubDataPointCsvPreview[];\n };\n }>(\n client,\n gql`\n query TranscendCliSubDataPointCsvExport(\n $filterBy: SubDataPointFiltersInput\n $first: Int!\n $offset: Int!\n ) {\n subDataPoints(\n filterBy: $filterBy\n first: $first\n offset: $offset\n useMaster: false\n ) {\n nodes {\n id\n name\n description\n dataPointId\n dataSiloId\n purposes {\n name\n purpose\n }\n categories {\n name\n category\n }\n ${\n includeGuessedCategories\n ? `pendingCategoryGuesses {\n category {\n name\n category\n }\n status\n classifierVersion\n }`\n : ''\n }\n ${\n includeAttributes\n ? `attributeValues {\n attributeKey {\n name\n }\n name\n }`\n : ''\n }\n }\n }\n }\n `,\n {\n variables: {\n first: pageSize,\n offset,\n filterBy: {\n ...filterBy,\n // TODO: https://transcend.height.app/T-40484 - add cursor support\n // ...(cursor ? { cursor: { id: cursor } } : {}),\n },\n },\n logger,\n },\n );\n\n cursor = nodes[nodes.length - 1]?.id as string;\n subDataPoints.push(...nodes);\n shouldContinue = nodes.length === pageSize;\n total += nodes.length;\n offset += nodes.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n );\n throw err;\n }\n } while (shouldContinue);\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n const sorted = sortBy(subDataPoints, 'name');\n\n logger.info(\n colors.green(\n `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n ),\n );\n return sorted;\n}\n\n/\n Pull datapoint information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n /\nasync function pullDatapoints(\n client: GraphQLClient,\n {\n dataPointIds = [],\n pageSize = 100,\n }: {\n /* IDs of data points to filter down /\n dataPointIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataPointCsvPreview[]> {\n const dataPoints: DataPointCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n // Group by 100\n const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n progressBar.start(dataPointIds.length, 0);\n let total = 0;\n await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n try {\n const {\n dataPoints: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataPoints: {\n /* List of matches /\n nodes: DataPointCsvPreview[];\n };\n }>(client, DATAPOINT_EXPORT, {\n variables: {\n first: pageSize,\n filterBy: {\n ids: dataPointIdsGroup,\n },\n },\n logger,\n });\n\n dataPoints.push(...nodes);\n total += dataPointIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n ),\n );\n return dataPoints;\n}\n\n/\n Pull data silo information\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n /\nasync function pullDataSilos(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n pageSize = 100,\n }: {\n /* IDs of data silos to filter down /\n dataSiloIds: string[];\n /* Page size to pull in /\n pageSize?: number;\n },\n): Promise<DataSiloCsvPreview[]> {\n const dataSilos: DataSiloCsvPreview[] = [];\n\n // Time duration\n const t0 = new Date().getTime();\n\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n // Group by 100\n const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n progressBar.start(dataSiloIds.length, 0);\n let total = 0;\n await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n try {\n const {\n dataSilos: { nodes },\n } = await makeGraphQLRequest<{\n /* Query response /\n dataSilos: {\n /* List of matches /\n nodes: DataSiloCsvPreview[];\n };\n }>(client, DATA_SILO_EXPORT, {\n variables: {\n first: pageSize,\n filterBy: {\n ids: dataSiloIdsGroup,\n },\n },\n logger,\n });\n\n dataSilos.push(...nodes);\n total += dataSiloIdsGroup.length;\n progressBar.update(total);\n } catch (err) {\n logger.error(\n colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n );\n throw err;\n }\n });\n\n progressBar.stop();\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n ),\n );\n return dataSilos;\n}\n\n/\n Pull all datapoints from the data inventory.\n \n @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n /\nexport async function pullAllDatapoints(\n client: GraphQLClient,\n {\n dataSiloIds = [],\n includeGuessedCategories,\n includeAttributes,\n parentCategories = [],\n subCategories = [],\n pageSize = 1000,\n }: DatapointFilterOptions & {\n /* Page size to pull in /\n pageSize?: number;\n } = {},\n): Promise<\n (SubDataPointCsvPreview & {\n /* Data point information /\n dataPoint: DataPointCsvPreview;\n /* Data silo information /\n dataSilo: DataSiloCsvPreview;\n })[]\n> {\n // Subdatapoint information\n const subDatapoints = await pullSubDatapoints(client, {\n dataSiloIds,\n includeGuessedCategories,\n includeAttributes,\n parentCategories,\n subCategories,\n pageSize,\n });\n\n // The datapoint ids to grab\n const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n const dataPoints = await pullDatapoints(client, {\n dataPointIds,\n });\n const dataPointById = keyBy(dataPoints, 'id');\n\n // The data silo IDs to grab\n const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n const dataSilos = await pullDataSilos(client, {\n dataSiloIds: allDataSiloIds,\n });\n const dataSiloById = keyBy(dataSilos, 'id');\n\n return subDatapoints.map((subDataPoint) => ({\n ...subDataPoint,\n dataPoint: dataPointById[subDataPoint.dataPointId],\n dataSilo: dataSiloById[subDataPoint.dataSiloId],\n }));\n}\n/ eslint-enable max-lines */\n"],"mappings":"wcA0FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,UAAW,CAAE,WAAU,CACvB,SACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,UAAW,CACT,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACD,SACD,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAAM,MAHH,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAE3D,CACb,CAAC,CACsC,KAAK,CAOvC,EAAe,EAAM,MAHH,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAE7C,CAC5B,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-DE7vqkyt.mjs → pullChunkedCustomSiloOutstandingIdentifiers-DuPDOm6f.mjs} RENAMED Viewed

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-DKKXqoSr.mjs";import{t}from"./logger-Ckep4QXn.mjs";import n from"colors";import{decodeCodec as r}from"@transcend-io/type-utils";import*as i from"io-ts";import{REDUCED_REQUESTS_FOR_DATA_SILO_COUNT as a,buildTranscendGraphQLClient as o,createSombraGotInstance as s,makeGraphQLRequest as c,withTransientRetry as l}from"@transcend-io/sdk";import{mapSeries as u}from"@transcend-io/utils";import d from"cli-progress";async function f(e,{dataSiloId:n}){let{listReducedRequestsForDataSilo:{totalCount:r}}=await c(e,a,{variables:{input:{dataSiloId:n,isResolved:!1}},logger:t});return r}const p=i.type({id:i.string,name:i.string,partition:i.string}),m=i.intersection([i.type({identifier:i.string,type:i.string,coreIdentifier:i.string,dataSiloId:i.string,requestId:i.string,nonce:i.string,requestCreatedAt:i.string,daysUntilOverdue:i.number,attributes:i.array(i.type({key:i.string,values:i.array(i.string)}))}),i.partial({partition:i.union([p,i.null])})]);async function h(e,{dataSiloId:n,limit:a=100,offset:o=0,requestType:s}){try{let c=await l(`pullCronPageOfIdentifiers`,()=>e.get(`v1/data-silo/${n}/pending-requests/${s}`,{searchParams:{offset:o,limit:a}}).json(),{logger:t,maxAttempts:6,baseDelayMs:500}),{items:u}=r(i.type({items:i.array(m)}),c);return u}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function g({dataSiloId:r,auth:i,sombraAuth:a,actions:c,apiPageSize:l=100,savePageSize:p=1e3,onSave:m,transcendUrl:g=e,skipRequestCount:_=!1}){if(p%l!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${p}, apiPageSize: ${l}`);let v=await s(g,i,{logger:t,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),y=o(g,i),b=0;_||(b=await f(y,{dataSiloId:r})),t.info(n.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${r}" for requests of types "${c.join(`", "`)}"`));let x=new Date().getTime(),S=new d.SingleBar({},d.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await u(c,async e=>{let i=0,a=!0;for(;a;){let o=await h(v,{dataSiloId:r,limit:l,offset:i,requestType:e}),s=o.map(t=>(C.add(t.requestId),{...t,action:e})),c=s.map(({attributes:e,partition:t,...n})=>({...n,...t?{partitionId:t.id,partitionName:t.name,partitionKey:t.partition}:{},...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...s),T.push(...c),T.length>=p&&(await m(T),T=[]),a=o.length===l,i+=l,_?t.info(n.magenta(`Pulled ${o.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await m(T),_||S.stop();let E=new Date().getTime()-x;return t.info(n.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{f as a,h as i,m as n,p as r,g as t};
-//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-DE7vqkyt.mjs.map
+import{a as e}from"./constants-CdgK1myP.mjs";import{t}from"./logger-CB_h_zbj.mjs";import n from"colors";import{decodeCodec as r}from"@transcend-io/type-utils";import*as i from"io-ts";import{REDUCED_REQUESTS_FOR_DATA_SILO_COUNT as a,buildTranscendGraphQLClient as o,createSombraGotInstance as s,makeGraphQLRequest as c,withTransientRetry as l}from"@transcend-io/sdk";import{mapSeries as u}from"@transcend-io/utils";import d from"cli-progress";async function f(e,{dataSiloId:n}){let{listReducedRequestsForDataSilo:{totalCount:r}}=await c(e,a,{variables:{input:{dataSiloId:n,isResolved:!1}},logger:t});return r}const p=i.type({id:i.string,name:i.string,partition:i.string}),m=i.intersection([i.type({identifier:i.string,type:i.string,coreIdentifier:i.string,dataSiloId:i.string,requestId:i.string,nonce:i.string,requestCreatedAt:i.string,daysUntilOverdue:i.number,attributes:i.array(i.type({key:i.string,values:i.array(i.string)}))}),i.partial({partition:i.union([p,i.null])})]);async function h(e,{dataSiloId:n,limit:a=100,offset:o=0,requestType:s}){try{let c=await l(`pullCronPageOfIdentifiers`,()=>e.get(`v1/data-silo/${n}/pending-requests/${s}`,{searchParams:{offset:o,limit:a}}).json(),{logger:t,maxAttempts:6,baseDelayMs:500}),{items:u}=r(i.type({items:i.array(m)}),c);return u}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function g({dataSiloId:r,auth:i,sombraAuth:a,actions:c,apiPageSize:l=100,savePageSize:p=1e3,onSave:m,transcendUrl:g=e,skipRequestCount:_=!1}){if(p%l!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${p}, apiPageSize: ${l}`);let v=await s(g,i,{logger:t,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),y=o(g,i),b=0;_||(b=await f(y,{dataSiloId:r})),t.info(n.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${r}" for requests of types "${c.join(`", "`)}"`));let x=new Date().getTime(),S=new d.SingleBar({},d.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await u(c,async e=>{let i=0,a=!0;for(;a;){let o=await h(v,{dataSiloId:r,limit:l,offset:i,requestType:e}),s=o.map(t=>(C.add(t.requestId),{...t,action:e})),c=s.map(({attributes:e,partition:t,...n})=>({...n,...t?{partitionId:t.id,partitionName:t.name,partitionKey:t.partition}:{},...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...s),T.push(...c),T.length>=p&&(await m(T),T=[]),a=o.length===l,i+=l,_?t.info(n.magenta(`Pulled ${o.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await m(T),_||S.stop();let E=new Date().getTime()-x;return t.info(n.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{f as a,h as i,m as n,p as r,g as t};
+//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-DuPDOm6f.mjs.map

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-DE7vqkyt.mjs.map → pullChunkedCustomSiloOutstandingIdentifiers-DuPDOm6f.mjs.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-DE7vqkyt.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { makeGraphQLRequest, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from '@transcend-io/sdk';\nimport { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/*\n Get number of open requests for a data silo\n \n @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n /\nexport async function fetchRequestDataSiloActiveCount(\n client: GraphQLClient,\n {\n dataSiloId,\n }: {\n /* Data silo ID /\n dataSiloId: string;\n },\n): Promise<number> {\n const {\n listReducedRequestsForDataSilo: { totalCount },\n } = await makeGraphQLRequest<{\n /* Requests /\n listReducedRequestsForDataSilo: {\n /* Total count /\n totalCount: number;\n };\n }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n variables: {\n input: {\n dataSiloId,\n isResolved: false,\n },\n },\n logger,\n });\n\n return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { withTransientRetry } from '@transcend-io/sdk';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\n/** Consent partition metadata when a DSR is scoped to a partition /\nexport const CronPartition = t.type({\n /* ID of the partition /\n id: t.string,\n /* The human readable name of the partition /\n name: t.string,\n /* The unique identifying partition value /\n partition: t.string,\n});\n\n/* Type override /\nexport type CronPartition = t.TypeOf<typeof CronPartition>;\n\nexport const CronIdentifier = t.intersection([\n t.type({\n /* The identifier value /\n identifier: t.string,\n /* The type of identifier /\n type: t.string,\n /* The core identifier of the request /\n coreIdentifier: t.string,\n /* The ID of the underlying data silo /\n dataSiloId: t.string,\n /* The ID of the underlying request /\n requestId: t.string,\n /* The request nonce /\n nonce: t.string,\n /* The time the request was created /\n requestCreatedAt: t.string,\n /* The number of days until the request is overdue /\n daysUntilOverdue: t.number,\n /* Request attributes /\n attributes: t.array(\n t.type({\n key: t.string,\n values: t.array(t.string),\n }),\n ),\n }),\n t.partial({\n /* The consent partition that scopes this request, when applicable /\n partition: t.union([CronPartition, t.null]),\n }),\n]);\n\n/* Type override /\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/\n Pull a offset of identifiers for a cron job\n \n @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n /\nexport async function pullCronPageOfIdentifiers(\n sombra: Got,\n {\n dataSiloId,\n limit = 100,\n offset = 0,\n requestType,\n }: {\n /* Data Silo ID /\n dataSiloId: string;\n /* Type of request /\n requestType: RequestAction;\n /* Number of identifiers to pull in /\n limit?: number;\n /* Page to pull in /\n offset?: number;\n },\n): Promise<CronIdentifier[]> {\n try {\n // `GET pending-requests` is a read and therefore safe to retry on transient\n // gateway / network errors. Customers running `transcend cron\n // pull-identifiers` against large silos would otherwise see the command\n // abort on a single 502 from the Sombra reverse tunnel.\n const response = await withTransientRetry(\n 'pullCronPageOfIdentifiers',\n () =>\n sombra\n .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n searchParams: {\n offset,\n limit,\n },\n })\n .json(),\n { logger, maxAttempts: 6, baseDelayMs: 500 },\n );\n\n const { items } = decodeCodec(\n t.type({\n items: t.array(CronIdentifier),\n }),\n response,\n );\n return items;\n } catch (err) {\n throw new Error(`Received an error from server: ${err?.response?.body \|\| err?.message}`);\n }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { buildTranscendGraphQLClient, createSombraGotInstance } from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchRequestDataSiloActiveCount } from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/\n A CSV formatted identifier\n /\nexport type CsvFormattedIdentifier = {\n [k in string]: string \| null \| boolean \| number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n /* The request action that the identifier relates to /\n action: RequestAction;\n}\n\n/\n Pull the set of identifiers outstanding for a cron or AVC integration\n \n This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n \n @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n /\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n dataSiloId,\n auth,\n sombraAuth,\n actions,\n apiPageSize = 100,\n savePageSize = 1000,\n onSave,\n transcendUrl = DEFAULT_TRANSCEND_API,\n skipRequestCount = false,\n}: {\n /* Transcend API key authentication /\n auth: string;\n /* Data Silo ID to pull down jobs for /\n dataSiloId: string;\n /* The request actions to fetch /\n actions: RequestAction[];\n /* How many identifiers to pull in a single call to the backend /\n apiPageSize: number;\n /* How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) /\n savePageSize: number;\n /* Callback function called when a chunk of identifiers is ready to be saved /\n onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n /* API URL for Transcend backend /\n transcendUrl?: string;\n /* Sombra API key authentication /\n sombraAuth?: string;\n /* Skip request count /\n skipRequestCount?: boolean;\n}): Promise<{\n /* Raw Identifiers */\n identifiers: CronIdentifierWithAction[];\n}> {\n // Validate savePageSize\n if (savePageSize % apiPageSize !== 0) {\n throw new Error(\n `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n );\n }\n\n // Create sombra instance to communicate with\n const sombra = await createSombraGotInstance(transcendUrl, auth, {\n logger,\n sombraApiKey: sombraAuth,\n sombraUrl: process.env.SOMBRA_URL,\n });\n\n // Create GraphQL client to connect to Transcend backend\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n let totalRequestCount = 0;\n if (!skipRequestCount) {\n totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n dataSiloId,\n });\n }\n\n logger.info(\n colors.magenta(\n `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n const foundRequestIds = new Set<string>();\n\n // identifiers found in total\n const identifiers: CronIdentifierWithAction[] = [];\n // current chunk of identifiers to be saved\n let currentChunk: CsvFormattedIdentifier[] = [];\n\n // map over each action\n if (!skipRequestCount) {\n progressBar.start(totalRequestCount, 0);\n }\n await mapSeries(actions, async (action) => {\n let offset = 0;\n let shouldContinue = true;\n\n // Fetch a page of identifiers\n while (shouldContinue) {\n const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n dataSiloId,\n limit: apiPageSize,\n offset,\n requestType: action,\n });\n\n const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n (identifier) => {\n foundRequestIds.add(identifier.requestId);\n return {\n ...identifier,\n action,\n };\n },\n );\n\n const csvFormattedIdentifiers = identifiersWithAction.map(\n ({ attributes, partition, ...identifier }) => ({\n ...identifier,\n ...(partition\n ? {\n partitionId: partition.id,\n partitionName: partition.name,\n partitionKey: partition.partition,\n }\n : {}),\n ...attributes.reduce(\n (acc, val) =>\n Object.assign(acc, {\n [val.key]: val.values.join(','),\n }),\n {},\n ),\n }),\n );\n\n identifiers.push(...identifiersWithAction);\n currentChunk.push(...csvFormattedIdentifiers);\n\n // Check if we've reached the savePageSize and call the onSave callback\n if (currentChunk.length >= savePageSize) {\n await onSave(currentChunk);\n currentChunk = [];\n }\n\n shouldContinue = pageIdentifiers.length === apiPageSize;\n offset += apiPageSize;\n if (!skipRequestCount) {\n progressBar.update(foundRequestIds.size);\n } else {\n logger.info(\n colors.magenta(\n `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n ),\n );\n }\n }\n });\n\n // Save any remaining identifiers in the current chunk\n if (currentChunk.length > 0) {\n await onSave(currentChunk);\n }\n\n if (!skipRequestCount) {\n progressBar.stop();\n }\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n foundRequestIds.size\n } requests in \"${totalTime / 1000}\" seconds!`,\n ),\n );\n\n return { identifiers };\n}\n"],"mappings":"0bAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,UAAW,CACT,MAAO,CACL,aACA,WAAY,GACb,CACF,CACD,SACD,CAAC,CAEF,OAAO,EC9BT,MAAa,EAAgB,EAAE,KAAK,CAElC,GAAI,EAAE,OAEN,KAAM,EAAE,OAER,UAAW,EAAE,OACd,CAAC,CAKW,EAAiB,EAAE,aAAa,CAC3C,EAAE,KAAK,CAEL,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CACF,EAAE,QAAQ,CAER,UAAW,EAAE,MAAM,CAAC,EAAe,EAAE,KAAK,CAAC,CAC5C,CAAC,CACH,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAKF,IAAM,EAAW,MAAM,EACrB,gCAEE,EACG,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CACX,CAAE,SAAQ,YAAa,EAAG,YAAa,IAAK,CAC7C,CAEK,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC7E5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGI,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,YAAW,GAAG,MAAkB,CAC7C,GAAG,EACH,GAAI,EACA,CACE,YAAa,EAAU,GACvB,cAAe,EAAU,KACzB,aAAc,EAAU,UACzB,CACD,EAAE,CACN,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}
1	+ {"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-DuPDOm6f.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { makeGraphQLRequest, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from '@transcend-io/sdk';\nimport { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/*\n Get number of open requests for a data silo\n \n @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n /\nexport async function fetchRequestDataSiloActiveCount(\n client: GraphQLClient,\n {\n dataSiloId,\n }: {\n /* Data silo ID /\n dataSiloId: string;\n },\n): Promise<number> {\n const {\n listReducedRequestsForDataSilo: { totalCount },\n } = await makeGraphQLRequest<{\n /* Requests /\n listReducedRequestsForDataSilo: {\n /* Total count /\n totalCount: number;\n };\n }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n variables: {\n input: {\n dataSiloId,\n isResolved: false,\n },\n },\n logger,\n });\n\n return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { withTransientRetry } from '@transcend-io/sdk';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\n/** Consent partition metadata when a DSR is scoped to a partition /\nexport const CronPartition = t.type({\n /* ID of the partition /\n id: t.string,\n /* The human readable name of the partition /\n name: t.string,\n /* The unique identifying partition value /\n partition: t.string,\n});\n\n/* Type override /\nexport type CronPartition = t.TypeOf<typeof CronPartition>;\n\nexport const CronIdentifier = t.intersection([\n t.type({\n /* The identifier value /\n identifier: t.string,\n /* The type of identifier /\n type: t.string,\n /* The core identifier of the request /\n coreIdentifier: t.string,\n /* The ID of the underlying data silo /\n dataSiloId: t.string,\n /* The ID of the underlying request /\n requestId: t.string,\n /* The request nonce /\n nonce: t.string,\n /* The time the request was created /\n requestCreatedAt: t.string,\n /* The number of days until the request is overdue /\n daysUntilOverdue: t.number,\n /* Request attributes /\n attributes: t.array(\n t.type({\n key: t.string,\n values: t.array(t.string),\n }),\n ),\n }),\n t.partial({\n /* The consent partition that scopes this request, when applicable /\n partition: t.union([CronPartition, t.null]),\n }),\n]);\n\n/* Type override /\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/\n Pull a offset of identifiers for a cron job\n \n @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n /\nexport async function pullCronPageOfIdentifiers(\n sombra: Got,\n {\n dataSiloId,\n limit = 100,\n offset = 0,\n requestType,\n }: {\n /* Data Silo ID /\n dataSiloId: string;\n /* Type of request /\n requestType: RequestAction;\n /* Number of identifiers to pull in /\n limit?: number;\n /* Page to pull in /\n offset?: number;\n },\n): Promise<CronIdentifier[]> {\n try {\n // `GET pending-requests` is a read and therefore safe to retry on transient\n // gateway / network errors. Customers running `transcend cron\n // pull-identifiers` against large silos would otherwise see the command\n // abort on a single 502 from the Sombra reverse tunnel.\n const response = await withTransientRetry(\n 'pullCronPageOfIdentifiers',\n () =>\n sombra\n .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n searchParams: {\n offset,\n limit,\n },\n })\n .json(),\n { logger, maxAttempts: 6, baseDelayMs: 500 },\n );\n\n const { items } = decodeCodec(\n t.type({\n items: t.array(CronIdentifier),\n }),\n response,\n );\n return items;\n } catch (err) {\n throw new Error(`Received an error from server: ${err?.response?.body \|\| err?.message}`);\n }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { buildTranscendGraphQLClient, createSombraGotInstance } from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchRequestDataSiloActiveCount } from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/\n A CSV formatted identifier\n /\nexport type CsvFormattedIdentifier = {\n [k in string]: string \| null \| boolean \| number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n /* The request action that the identifier relates to /\n action: RequestAction;\n}\n\n/\n Pull the set of identifiers outstanding for a cron or AVC integration\n \n This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n \n @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n /\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n dataSiloId,\n auth,\n sombraAuth,\n actions,\n apiPageSize = 100,\n savePageSize = 1000,\n onSave,\n transcendUrl = DEFAULT_TRANSCEND_API,\n skipRequestCount = false,\n}: {\n /* Transcend API key authentication /\n auth: string;\n /* Data Silo ID to pull down jobs for /\n dataSiloId: string;\n /* The request actions to fetch /\n actions: RequestAction[];\n /* How many identifiers to pull in a single call to the backend /\n apiPageSize: number;\n /* How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) /\n savePageSize: number;\n /* Callback function called when a chunk of identifiers is ready to be saved /\n onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n /* API URL for Transcend backend /\n transcendUrl?: string;\n /* Sombra API key authentication /\n sombraAuth?: string;\n /* Skip request count /\n skipRequestCount?: boolean;\n}): Promise<{\n /* Raw Identifiers */\n identifiers: CronIdentifierWithAction[];\n}> {\n // Validate savePageSize\n if (savePageSize % apiPageSize !== 0) {\n throw new Error(\n `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n );\n }\n\n // Create sombra instance to communicate with\n const sombra = await createSombraGotInstance(transcendUrl, auth, {\n logger,\n sombraApiKey: sombraAuth,\n sombraUrl: process.env.SOMBRA_URL,\n });\n\n // Create GraphQL client to connect to Transcend backend\n const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n let totalRequestCount = 0;\n if (!skipRequestCount) {\n totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n dataSiloId,\n });\n }\n\n logger.info(\n colors.magenta(\n `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n ),\n );\n\n // Time duration\n const t0 = new Date().getTime();\n // create a new progress bar instance and use shades_classic theme\n const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n const foundRequestIds = new Set<string>();\n\n // identifiers found in total\n const identifiers: CronIdentifierWithAction[] = [];\n // current chunk of identifiers to be saved\n let currentChunk: CsvFormattedIdentifier[] = [];\n\n // map over each action\n if (!skipRequestCount) {\n progressBar.start(totalRequestCount, 0);\n }\n await mapSeries(actions, async (action) => {\n let offset = 0;\n let shouldContinue = true;\n\n // Fetch a page of identifiers\n while (shouldContinue) {\n const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n dataSiloId,\n limit: apiPageSize,\n offset,\n requestType: action,\n });\n\n const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n (identifier) => {\n foundRequestIds.add(identifier.requestId);\n return {\n ...identifier,\n action,\n };\n },\n );\n\n const csvFormattedIdentifiers = identifiersWithAction.map(\n ({ attributes, partition, ...identifier }) => ({\n ...identifier,\n ...(partition\n ? {\n partitionId: partition.id,\n partitionName: partition.name,\n partitionKey: partition.partition,\n }\n : {}),\n ...attributes.reduce(\n (acc, val) =>\n Object.assign(acc, {\n [val.key]: val.values.join(','),\n }),\n {},\n ),\n }),\n );\n\n identifiers.push(...identifiersWithAction);\n currentChunk.push(...csvFormattedIdentifiers);\n\n // Check if we've reached the savePageSize and call the onSave callback\n if (currentChunk.length >= savePageSize) {\n await onSave(currentChunk);\n currentChunk = [];\n }\n\n shouldContinue = pageIdentifiers.length === apiPageSize;\n offset += apiPageSize;\n if (!skipRequestCount) {\n progressBar.update(foundRequestIds.size);\n } else {\n logger.info(\n colors.magenta(\n `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n ),\n );\n }\n }\n });\n\n // Save any remaining identifiers in the current chunk\n if (currentChunk.length > 0) {\n await onSave(currentChunk);\n }\n\n if (!skipRequestCount) {\n progressBar.stop();\n }\n const t1 = new Date().getTime();\n const totalTime = t1 - t0;\n\n logger.info(\n colors.green(\n `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n foundRequestIds.size\n } requests in \"${totalTime / 1000}\" seconds!`,\n ),\n );\n\n return { identifiers };\n}\n"],"mappings":"0bAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,UAAW,CACT,MAAO,CACL,aACA,WAAY,GACb,CACF,CACD,SACD,CAAC,CAEF,OAAO,EC9BT,MAAa,EAAgB,EAAE,KAAK,CAElC,GAAI,EAAE,OAEN,KAAM,EAAE,OAER,UAAW,EAAE,OACd,CAAC,CAKW,EAAiB,EAAE,aAAa,CAC3C,EAAE,KAAK,CAEL,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CACF,EAAE,QAAQ,CAER,UAAW,EAAE,MAAM,CAAC,EAAe,EAAE,KAAK,CAAC,CAC5C,CAAC,CACH,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAKF,IAAM,EAAW,MAAM,EACrB,gCAEE,EACG,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CACX,CAAE,SAAQ,YAAa,EAAG,YAAa,IAAK,CAC7C,CAEK,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC7E5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGI,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,YAAW,GAAG,MAAkB,CAC7C,GAAG,EACH,GAAI,EACA,CACE,YAAa,EAAU,GACvB,cAAe,EAAU,KACzB,aAAc,EAAU,UACzB,CACD,EAAE,CACN,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}