@transcend-io/cli 10.5.4 → 10.5.5

package/dist/parquetToCsvOneFile-B84XXInh.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"parquetToCsvOneFile-B84XXInh.mjs","names":[],"sources":["../src/lib/helpers/parquetToCsvOneFile.ts"],"sourcesContent":["import { mkdirSync, rmSync, existsSync } from 'node:fs';\nimport { dirname, join, parse } from 'node:path';\n\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport colors from 'colors';\n\nimport { logger } from '../../logger.js';\n\n/** Progress callback used by the parent runner to surface progress to the UI. */\ntype OnProgress = (processed: number, total?: number) => void;\n\n/**\n * Options for converting a single Parquet file into a single CSV file.\n */\nexport type ParquetToCsvOneFileOptions = {\n  /** Absolute or relative path to the input `.parquet` file. */\n  filePath: string;\n  /**\n   * Directory where the output CSV will be written.\n   * If omitted, the CSV is written next to the input file.\n   */\n  outputDir?: string;\n  /**\n   * When true, removes a pre-existing output file with the same name before writing.\n   * Useful for re-runs; ignored if the file does not exist.\n   */\n  clearOutputDir: boolean;\n  /**\n   * Optional progress hook. Called with the number of processed records.\n   * `total` is not computed here; it will be `undefined`.\n   */\n  onProgress?: OnProgress;\n};\n\n/**\n * Convert a single Parquet file to a single CSV file (1:1) using DuckDB.\n *\n * Output naming: `${basename}.csv` in `outputDir ?? dirname(filePath)`.\n *\n * Errors:\n *  - Throws on I/O failures or DuckDB execution errors.\n *\n *  Why DuckDB?\n * - Robust reader for many Parquet dialects (e.g., Spark output, nested types, timestamps).\n * - Streaming COPY handles large files without loading everything into JS memory.\n *\n * What this does:\n *  - Opens an in-memory DuckDB database (no `.db` file created).\n *  - Optionally disables temp spilling to disk (so only your CSV is written).\n *  - Executes a single `COPY (SELECT * FROM read_parquet(...)) TO ...` statement.\n *  - Produces exactly one CSV per input Parquet (no chunking or rotation).\n *\n * Notes & defaults:\n *  - DuckDBInstance: `:memory:` (ephemeral). No persistent DB file is created.\n *  - Temp files: disabled via `PRAGMA temp_directory=''` (best-effort; ignored if unsupported).\n *  - CSV format: header row, comma delimiter, double-quote quoting, empty string for NULL.\n *  - Progress: DuckDB COPY doesn't expose row-level progress via the JS API; we emit a\n *    best-effort final callback.\n *\n * Requirements:\n *  - `@duckdb/node-api` npm package installed and available at runtime.\n *  - Supported platform binary (mac arm64/x64, linux x64, windows x64).\n *\n * @param opts - Conversion options\n * @param DuckDb - DuckDB instance to use\n * @returns Promise<void> when the CSV has been written\n */\nexport async function parquetToCsvOneFile(\n  opts: ParquetToCsvOneFileOptions,\n  DuckDb: typeof DuckDBInstance,\n): Promise<void> {\n  const { filePath, outputDir, clearOutputDir, onProgress } = opts;\n\n  const baseDir = outputDir || dirname(filePath);\n  const { name: baseName } = parse(filePath);\n  const outPath = join(baseDir, `${baseName}.csv`);\n\n  // Ensure output directory exists\n  mkdirSync(baseDir, { recursive: true });\n\n  // Remove any pre-existing output file if requested\n  if (clearOutputDir && existsSync(outPath)) {\n    try {\n      rmSync(outPath, { force: true });\n    } catch (err) {\n      logger.warn(\n        colors.yellow(\n          `Could not remove existing output file ${outPath}: ${(err as Error).message}`,\n        ),\n      );\n    }\n  }\n\n  // In-memory DB: no .db file created on disk\n  const db = await DuckDb.create(':memory:');\n  const conn = await db.connect();\n\n  try {\n    // Optional: prevent DuckDB from creating temp files on disk (best-effort).\n    // Some versions may ignore or error; we ignore such errors safely.\n    await runIgnoreError(conn, \"PRAGMA temp_directory='';\");\n\n    // Optionally: cap memory to encourage in-memory execution or fail-fast\n    // (commented out by default; uncomment to enforce a limit)\n    // await runIgnoreError(conn, \"PRAGMA memory_limit='4GB';\");\n\n    // Ensure stable CSV settings: header, comma delimiter, double quotes, empty string for NULLs.\n    // Escape single quotes for SQL string literals\n    const q = (p: string): string => `'${p.replace(/'/g, \"''\")}'`;\n\n    // Use COPY with a subquery so DuckDB streams Parquet -> CSV efficiently.\n    const sql = `\n      COPY (SELECT * FROM read_parquet(${q(filePath)}))\n      TO ${q(outPath)}\n      (HEADER, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '');\n    `;\n\n    await run(conn, sql);\n\n    // Best-effort progress notification (DuckDB JS API doesn't expose progress for COPY)\n    onProgress?.(0, undefined);\n\n    logger.info(colors.green(`Wrote CSV → ${outPath}`));\n  } finally {\n    // Close connection + db handles gracefully\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(conn as any);\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(db as any);\n  }\n}\n\n/* =============================================================================\n * DuckDB helpers\n * =============================================================================\n */\n\n/**\n * Execute a SQL statement on a DuckDB connection and dispose the result.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function run(conn: DuckDBConnection, sql: string): Promise<void> {\n  const result = await conn.run(sql);\n  // The high-level API returns a Result; ensure we dispose it to free buffers.\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  await disposeSafe(result as any);\n}\n\n/**\n * Execute a SQL statement but ignore any error that occurs.\n * Useful for best-effort PRAGMAs that may not be supported across versions.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function runIgnoreError(conn: DuckDBConnection, sql: string): Promise<void> {\n  try {\n    await run(conn, sql);\n  } catch {\n    // ignore\n  }\n}\n\n/**\n * Dispose a DuckDB resource (connection or instance) if present.\n *\n * @param handle - Object exposing an async `dispose()` method\n * @returns Promise<void>\n */\nasync function disposeSafe(\n  handle:\n    | {\n        /** Dispose handler */\n        dispose: () => Promise<void>;\n      }\n    | null\n    | undefined,\n): Promise<void> {\n  if (!handle || typeof handle.dispose !== 'function') return;\n  try {\n    await handle.dispose();\n  } catch {\n    // ignore\n  }\n}\n"],"mappings":"yLAmEA,eAAsB,EACpB,EACA,EACe,CACf,GAAM,CAAE,WAAU,YAAW,iBAAgB,cAAe,EAEtD,EAAU,GAAa,EAAQ,EAAS,CACxC,CAAE,KAAM,GAAa,EAAM,EAAS,CACpC,EAAU,EAAK,EAAS,GAAG,EAAS,MAAM,CAMhD,GAHA,EAAU,EAAS,CAAE,UAAW,GAAM,CAAC,CAGnC,GAAkB,EAAW,EAAQ,CACvC,GAAI,CACF,EAAO,EAAS,CAAE,MAAO,GAAM,CAAC,OACzB,EAAK,CACZ,EAAO,KACL,EAAO,OACL,yCAAyC,EAAQ,IAAK,EAAc,UACrE,CACF,CAKL,IAAM,EAAK,MAAM,EAAO,OAAO,WAAW,CACpC,EAAO,MAAM,EAAG,SAAS,CAE/B,GAAI,CAGF,MAAM,EAAe,EAAM,4BAA4B,CAQvD,IAAM,EAAK,GAAsB,IAAI,EAAE,QAAQ,KAAM,KAAK,CAAC,GAS3D,MAAM,EAAI,EANE;yCACyB,EAAE,EAAS,CAAC;WAC1C,EAAE,EAAQ,CAAC;;MAIE,CAGpB,IAAa,EAAG,IAAA,GAAU,CAE1B,EAAO,KAAK,EAAO,MAAM,eAAe,IAAU,CAAC,QAC3C,CAGR,MAAM,EAAY,EAAY,CAE9B,MAAM,EAAY,EAAU,EAgBhC,eAAe,EAAI,EAAwB,EAA4B,CAIrE,MAAM,EAHS,MAAM,EAAK,IAAI,EAAI,CAGF,CAWlC,eAAe,EAAe,EAAwB,EAA4B,CAChF,GAAI,CACF,MAAM,EAAI,EAAM,EAAI,MACd,GAWV,eAAe,EACb,EAOe,CACX,MAAC,GAAU,OAAO,EAAO,SAAY,YACzC,GAAI,CACF,MAAM,EAAO,SAAS,MAChB"}
+{"version":3,"file":"parquetToCsvOneFile-B84XXInh.mjs","names":[],"sources":["../src/lib/helpers/parquetToCsvOneFile.ts"],"sourcesContent":["import { mkdirSync, rmSync, existsSync } from 'node:fs';\nimport { dirname, join, parse } from 'node:path';\n\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport colors from 'colors';\n\nimport { logger } from '../../logger.js';\n\n/** Progress callback used by the parent runner to surface progress to the UI. */\ntype OnProgress = (processed: number, total?: number) => void;\n\n/**\n * Options for converting a single Parquet file into a single CSV file.\n */\nexport type ParquetToCsvOneFileOptions = {\n  /** Absolute or relative path to the input `.parquet` file. */\n  filePath: string;\n  /**\n   * Directory where the output CSV will be written.\n   * If omitted, the CSV is written next to the input file.\n   */\n  outputDir?: string;\n  /**\n   * When true, removes a pre-existing output file with the same name before writing.\n   * Useful for re-runs; ignored if the file does not exist.\n   */\n  clearOutputDir: boolean;\n  /**\n   * Optional progress hook. Called with the number of processed records.\n   * `total` is not computed here; it will be `undefined`.\n   */\n  onProgress?: OnProgress;\n};\n\n/**\n * Convert a single Parquet file to a single CSV file (1:1) using DuckDB.\n *\n * Output naming: `${basename}.csv` in `outputDir ?? dirname(filePath)`.\n *\n * Errors:\n *  - Throws on I/O failures or DuckDB execution errors.\n *\n *  Why DuckDB?\n * - Robust reader for many Parquet dialects (e.g., Spark output, nested types, timestamps).\n * - Streaming COPY handles large files without loading everything into JS memory.\n *\n * What this does:\n *  - Opens an in-memory DuckDB database (no `.db` file created).\n *  - Optionally disables temp spilling to disk (so only your CSV is written).\n *  - Executes a single `COPY (SELECT * FROM read_parquet(...)) TO ...` statement.\n *  - Produces exactly one CSV per input Parquet (no chunking or rotation).\n *\n * Notes & defaults:\n *  - DuckDBInstance: `:memory:` (ephemeral). No persistent DB file is created.\n *  - Temp files: disabled via `PRAGMA temp_directory=''` (best-effort; ignored if unsupported).\n *  - CSV format: header row, comma delimiter, double-quote quoting, empty string for NULL.\n *  - Progress: DuckDB COPY doesn't expose row-level progress via the JS API; we emit a\n *    best-effort final callback.\n *\n * Requirements:\n *  - `@duckdb/node-api` npm package installed and available at runtime.\n *  - Supported platform binary (mac arm64/x64, linux x64, windows x64).\n *\n * @param opts - Conversion options\n * @param DuckDb - DuckDB instance to use\n * @returns Promise<void> when the CSV has been written\n */\nexport async function parquetToCsvOneFile(\n  opts: ParquetToCsvOneFileOptions,\n  DuckDb: typeof DuckDBInstance,\n): Promise<void> {\n  const { filePath, outputDir, clearOutputDir, onProgress } = opts;\n\n  const baseDir = outputDir || dirname(filePath);\n  const { name: baseName } = parse(filePath);\n  const outPath = join(baseDir, `${baseName}.csv`);\n\n  // Ensure output directory exists\n  mkdirSync(baseDir, { recursive: true });\n\n  // Remove any pre-existing output file if requested\n  if (clearOutputDir && existsSync(outPath)) {\n    try {\n      rmSync(outPath, { force: true });\n    } catch (err) {\n      logger.warn(\n        colors.yellow(\n          `Could not remove existing output file ${outPath}: ${(err as Error).message}`,\n        ),\n      );\n    }\n  }\n\n  // In-memory DB: no .db file created on disk\n  const db = await DuckDb.create(':memory:');\n  const conn = await db.connect();\n\n  try {\n    // Optional: prevent DuckDB from creating temp files on disk (best-effort).\n    // Some versions may ignore or error; we ignore such errors safely.\n    await runIgnoreError(conn, \"PRAGMA temp_directory='';\");\n\n    // Optionally: cap memory to encourage in-memory execution or fail-fast\n    // (commented out by default; uncomment to enforce a limit)\n    // await runIgnoreError(conn, \"PRAGMA memory_limit='4GB';\");\n\n    // Ensure stable CSV settings: header, comma delimiter, double quotes, empty string for NULLs.\n    // Escape single quotes for SQL string literals\n    const q = (p: string): string => `'${p.replace(/'/g, \"''\")}'`;\n\n    // Use COPY with a subquery so DuckDB streams Parquet -> CSV efficiently.\n    const sql = `\n      COPY (SELECT * FROM read_parquet(${q(filePath)}))\n      TO ${q(outPath)}\n      (HEADER, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '');\n    `;\n\n    await run(conn, sql);\n\n    // Best-effort progress notification (DuckDB JS API doesn't expose progress for COPY)\n    onProgress?.(0, undefined);\n\n    logger.info(colors.green(`Wrote CSV → ${outPath}`));\n  } finally {\n    // Close connection + db handles gracefully\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(conn as any);\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(db as any);\n  }\n}\n\n/* =============================================================================\n * DuckDB helpers\n * =============================================================================\n */\n\n/**\n * Execute a SQL statement on a DuckDB connection and dispose the result.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function run(conn: DuckDBConnection, sql: string): Promise<void> {\n  const result = await conn.run(sql);\n  // The high-level API returns a Result; ensure we dispose it to free buffers.\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  await disposeSafe(result as any);\n}\n\n/**\n * Execute a SQL statement but ignore any error that occurs.\n * Useful for best-effort PRAGMAs that may not be supported across versions.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function runIgnoreError(conn: DuckDBConnection, sql: string): Promise<void> {\n  try {\n    await run(conn, sql);\n  } catch {\n    // ignore\n  }\n}\n\n/**\n * Dispose a DuckDB resource (connection or instance) if present.\n *\n * @param handle - Object exposing an async `dispose()` method\n * @returns Promise<void>\n */\nasync function disposeSafe(\n  handle:\n    | {\n        /** Dispose handler */\n        dispose: () => Promise<void>;\n      }\n    | null\n    | undefined,\n): Promise<void> {\n  if (!handle || typeof handle.dispose !== 'function') return;\n  try {\n    await handle.dispose();\n  } catch {\n    // ignore\n  }\n}\n"],"mappings":"yLAmEA,eAAsB,EACpB,EACA,EACe,CACf,GAAM,CAAE,WAAU,YAAW,iBAAgB,cAAe,EAEtD,EAAU,GAAa,EAAQ,EAAS,CACxC,CAAE,KAAM,GAAa,EAAM,EAAS,CACpC,EAAU,EAAK,EAAS,GAAG,EAAS,MAAM,CAMhD,GAHA,EAAU,EAAS,CAAE,UAAW,GAAM,CAAC,CAGnC,GAAkB,EAAW,EAAQ,CACvC,GAAI,CACF,EAAO,EAAS,CAAE,MAAO,GAAM,CAAC,OACzB,EAAK,CACZ,EAAO,KACL,EAAO,OACL,yCAAyC,EAAQ,IAAK,EAAc,UACrE,CACF,CAKL,IAAM,EAAK,MAAM,EAAO,OAAO,WAAW,CACpC,EAAO,MAAM,EAAG,SAAS,CAE/B,GAAI,CAGF,MAAM,EAAe,EAAM,4BAA4B,CAQvD,IAAM,EAAK,GAAsB,IAAI,EAAE,QAAQ,KAAM,KAAK,CAAC,GAS3D,MAAM,EAAI,EAAM;yCALqB,EAAE,EAAS,CAAC;WAC1C,EAAE,EAAQ,CAAC;;MAIE,CAGpB,IAAa,EAAG,IAAA,GAAU,CAE1B,EAAO,KAAK,EAAO,MAAM,eAAe,IAAU,CAAC,QAC3C,CAGR,MAAM,EAAY,EAAY,CAE9B,MAAM,EAAY,EAAU,EAgBhC,eAAe,EAAI,EAAwB,EAA4B,CAIrE,MAAM,EAAY,MAHG,EAAK,IAAI,EAAI,CAGF,CAWlC,eAAe,EAAe,EAAwB,EAA4B,CAChF,GAAI,CACF,MAAM,EAAI,EAAM,EAAI,MACd,GAWV,eAAe,EACb,EAOe,CACX,MAAC,GAAU,OAAO,EAAO,SAAY,YACzC,GAAI,CACF,MAAM,EAAO,SAAS,MAChB"}

package/dist/{parseAttributesFromString-D1Yl0xwT.mjs → parseAttributesFromString-q4-59nEN.mjs}

@@ -1,2 +1,2 @@
-import{t as e}from"./logger-Bj782ZYD.mjs";import{l as t,n}from"./constants-ClkQQhJs.mjs";import r from"colors";import*as i from"io-ts";import a from"fuzzysearch";import o from"inquirer";function s(e,t){return a(e.toLowerCase(),t.toLowerCase())||a(t.toLowerCase(),e.toLowerCase())}function c(e,r,i,a){let c=e.filter(e=>s(r.toLowerCase(),e.toLowerCase()));return[...c,new o.Separator,...i?[]:[t],...a?[n]:[],...e.filter(e=>!c.includes(e))]}const l=i.type({key:i.string,values:i.array(i.string)});function u(t){let n=t.map(e=>{let[t,n]=e.trim().split(`:`);if(!n)throw Error(`Expected attributes in key:value1;value2,key2:value3;value4`);return{key:t,values:n.split(`;`)}});return e.info(r.magenta(`Attributes to apply to all requests:`)),e.info(r.magenta(JSON.stringify(n,null,2))),n}export{s as i,u as n,c as r,l as t};
-//# sourceMappingURL=parseAttributesFromString-D1Yl0xwT.mjs.map
+import{t as e}from"./logger-Bj782ZYD.mjs";import{l as t,n}from"./constants-CLxPXlJM.mjs";import r from"colors";import*as i from"io-ts";import a from"fuzzysearch";import o from"inquirer";function s(e,t){return a(e.toLowerCase(),t.toLowerCase())||a(t.toLowerCase(),e.toLowerCase())}function c(e,r,i,a){let c=e.filter(e=>s(r.toLowerCase(),e.toLowerCase()));return[...c,new o.Separator,...i?[]:[t],...a?[n]:[],...e.filter(e=>!c.includes(e))]}const l=i.type({key:i.string,values:i.array(i.string)});function u(t){let n=t.map(e=>{let[t,n]=e.trim().split(`:`);if(!n)throw Error(`Expected attributes in key:value1;value2,key2:value3;value4`);return{key:t,values:n.split(`;`)}});return e.info(r.magenta(`Attributes to apply to all requests:`)),e.info(r.magenta(JSON.stringify(n,null,2))),n}export{s as i,u as n,c as r,l as t};
+//# sourceMappingURL=parseAttributesFromString-q4-59nEN.mjs.map

package/dist/{parseAttributesFromString-D1Yl0xwT.mjs.map → parseAttributesFromString-q4-59nEN.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"parseAttributesFromString-D1Yl0xwT.mjs","names":[],"sources":["../src/lib/requests/fuzzyMatchColumns.ts","../src/lib/requests/parseAttributesFromString.ts"],"sourcesContent":["import fuzzysearch from 'fuzzysearch';\nimport inquirer from 'inquirer';\n\nimport { NONE, BULK_APPLY } from './constants.js';\n\n/**\n * Check if word1 and word2 are a fuzzy match of each other.\n * Returns true if word1 is fuzzy match of word2 or vice versa.\n *\n * @param word1 - First word\n * @param word2 - Second word\n * @returns True if words are fuzzy match\n */\nexport function fuzzySearch(word1: string, word2: string): boolean {\n  return (\n    fuzzysearch(word1.toLowerCase(), word2.toLowerCase()) ||\n    fuzzysearch(word2.toLowerCase(), word1.toLowerCase())\n  );\n}\n\n/**\n * Fuzzy match column names for a particular field\n *\n * @param allColumnNames - List of all column names\n * @param fuzzyMapName - The name of field being mapped to\n * @param isRequired - When true, don't include \"NONE\" as an option\n * @param canApplyAll - When true, include an option to specify the value in bulk\n * @returns The list of suggestions for inquirer\n */\nexport function fuzzyMatchColumns(\n  allColumnNames: string[],\n  fuzzyMapName: string,\n  isRequired: boolean,\n  canApplyAll?: boolean,\n): (string | InstanceType<typeof inquirer.Separator>)[] {\n  const matchingColumnNames = allColumnNames.filter((x) =>\n    fuzzySearch(fuzzyMapName.toLowerCase(), x.toLowerCase()),\n  );\n  return [\n    ...matchingColumnNames,\n    new inquirer.Separator(),\n    ...(isRequired ? [] : [NONE]),\n    ...(canApplyAll ? [BULK_APPLY] : []),\n    ...allColumnNames.filter((x) => !matchingColumnNames.includes(x)),\n  ];\n}\n","import colors from 'colors';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\nexport const ParsedAttributeInput = t.type({\n  /** Attribute key */\n  key: t.string,\n  /** Attribute values */\n  values: t.array(t.string),\n});\n\n/** Type override */\nexport type ParsedAttributeInput = t.TypeOf<typeof ParsedAttributeInput>;\n\n/**\n * Parse out the extra attributes to apply to all requests uploaded\n *\n * @param attributes - input as string, e.g. ['key:value1;value2','key2:value3;value4']\n * @returns The parsed attributes\n */\nexport function parseAttributesFromString(attributes: string[]): ParsedAttributeInput[] {\n  // Parse out the extra attributes to apply to all requests uploaded\n  const parsedAttributes = attributes.map((attribute) => {\n    const [attributeKey, attributeValuesRaw] = attribute.trim().split(':');\n    if (!attributeValuesRaw) {\n      throw new Error('Expected attributes in key:value1;value2,key2:value3;value4');\n    }\n    const attributeValues = attributeValuesRaw.split(';');\n    return {\n      key: attributeKey,\n      values: attributeValues,\n    };\n  });\n  logger.info(colors.magenta('Attributes to apply to all requests:'));\n  logger.info(colors.magenta(JSON.stringify(parsedAttributes, null, 2)));\n  return parsedAttributes;\n}\n"],"mappings":"0LAaA,SAAgB,EAAY,EAAe,EAAwB,CACjE,OACE,EAAY,EAAM,aAAa,CAAE,EAAM,aAAa,CAAC,EACrD,EAAY,EAAM,aAAa,CAAE,EAAM,aAAa,CAAC,CAazD,SAAgB,EACd,EACA,EACA,EACA,EACsD,CACtD,IAAM,EAAsB,EAAe,OAAQ,GACjD,EAAY,EAAa,aAAa,CAAE,EAAE,aAAa,CAAC,CACzD,CACD,MAAO,CACL,GAAG,EACH,IAAI,EAAS,UACb,GAAI,EAAa,EAAE,CAAG,CAAC,EAAK,CAC5B,GAAI,EAAc,CAAC,EAAW,CAAG,EAAE,CACnC,GAAG,EAAe,OAAQ,GAAM,CAAC,EAAoB,SAAS,EAAE,CAAC,CAClE,CCvCH,MAAa,EAAuB,EAAE,KAAK,CAEzC,IAAK,EAAE,OAEP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CAWF,SAAgB,EAA0B,EAA8C,CAEtF,IAAM,EAAmB,EAAW,IAAK,GAAc,CACrD,GAAM,CAAC,EAAc,GAAsB,EAAU,MAAM,CAAC,MAAM,IAAI,CACtE,GAAI,CAAC,EACH,MAAU,MAAM,8DAA8D,CAGhF,MAAO,CACL,IAAK,EACL,OAHsB,EAAmB,MAAM,IAAI,CAIpD,EACD,CAGF,OAFA,EAAO,KAAK,EAAO,QAAQ,uCAAuC,CAAC,CACnE,EAAO,KAAK,EAAO,QAAQ,KAAK,UAAU,EAAkB,KAAM,EAAE,CAAC,CAAC,CAC/D"}
+{"version":3,"file":"parseAttributesFromString-q4-59nEN.mjs","names":[],"sources":["../src/lib/requests/fuzzyMatchColumns.ts","../src/lib/requests/parseAttributesFromString.ts"],"sourcesContent":["import fuzzysearch from 'fuzzysearch';\nimport inquirer from 'inquirer';\n\nimport { NONE, BULK_APPLY } from './constants.js';\n\n/**\n * Check if word1 and word2 are a fuzzy match of each other.\n * Returns true if word1 is fuzzy match of word2 or vice versa.\n *\n * @param word1 - First word\n * @param word2 - Second word\n * @returns True if words are fuzzy match\n */\nexport function fuzzySearch(word1: string, word2: string): boolean {\n  return (\n    fuzzysearch(word1.toLowerCase(), word2.toLowerCase()) ||\n    fuzzysearch(word2.toLowerCase(), word1.toLowerCase())\n  );\n}\n\n/**\n * Fuzzy match column names for a particular field\n *\n * @param allColumnNames - List of all column names\n * @param fuzzyMapName - The name of field being mapped to\n * @param isRequired - When true, don't include \"NONE\" as an option\n * @param canApplyAll - When true, include an option to specify the value in bulk\n * @returns The list of suggestions for inquirer\n */\nexport function fuzzyMatchColumns(\n  allColumnNames: string[],\n  fuzzyMapName: string,\n  isRequired: boolean,\n  canApplyAll?: boolean,\n): (string | InstanceType<typeof inquirer.Separator>)[] {\n  const matchingColumnNames = allColumnNames.filter((x) =>\n    fuzzySearch(fuzzyMapName.toLowerCase(), x.toLowerCase()),\n  );\n  return [\n    ...matchingColumnNames,\n    new inquirer.Separator(),\n    ...(isRequired ? [] : [NONE]),\n    ...(canApplyAll ? [BULK_APPLY] : []),\n    ...allColumnNames.filter((x) => !matchingColumnNames.includes(x)),\n  ];\n}\n","import colors from 'colors';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\nexport const ParsedAttributeInput = t.type({\n  /** Attribute key */\n  key: t.string,\n  /** Attribute values */\n  values: t.array(t.string),\n});\n\n/** Type override */\nexport type ParsedAttributeInput = t.TypeOf<typeof ParsedAttributeInput>;\n\n/**\n * Parse out the extra attributes to apply to all requests uploaded\n *\n * @param attributes - input as string, e.g. ['key:value1;value2','key2:value3;value4']\n * @returns The parsed attributes\n */\nexport function parseAttributesFromString(attributes: string[]): ParsedAttributeInput[] {\n  // Parse out the extra attributes to apply to all requests uploaded\n  const parsedAttributes = attributes.map((attribute) => {\n    const [attributeKey, attributeValuesRaw] = attribute.trim().split(':');\n    if (!attributeValuesRaw) {\n      throw new Error('Expected attributes in key:value1;value2,key2:value3;value4');\n    }\n    const attributeValues = attributeValuesRaw.split(';');\n    return {\n      key: attributeKey,\n      values: attributeValues,\n    };\n  });\n  logger.info(colors.magenta('Attributes to apply to all requests:'));\n  logger.info(colors.magenta(JSON.stringify(parsedAttributes, null, 2)));\n  return parsedAttributes;\n}\n"],"mappings":"0LAaA,SAAgB,EAAY,EAAe,EAAwB,CACjE,OACE,EAAY,EAAM,aAAa,CAAE,EAAM,aAAa,CAAC,EACrD,EAAY,EAAM,aAAa,CAAE,EAAM,aAAa,CAAC,CAazD,SAAgB,EACd,EACA,EACA,EACA,EACsD,CACtD,IAAM,EAAsB,EAAe,OAAQ,GACjD,EAAY,EAAa,aAAa,CAAE,EAAE,aAAa,CAAC,CACzD,CACD,MAAO,CACL,GAAG,EACH,IAAI,EAAS,UACb,GAAI,EAAa,EAAE,CAAG,CAAC,EAAK,CAC5B,GAAI,EAAc,CAAC,EAAW,CAAG,EAAE,CACnC,GAAG,EAAe,OAAQ,GAAM,CAAC,EAAoB,SAAS,EAAE,CAAC,CAClE,CCvCH,MAAa,EAAuB,EAAE,KAAK,CAEzC,IAAK,EAAE,OAEP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CAWF,SAAgB,EAA0B,EAA8C,CAEtF,IAAM,EAAmB,EAAW,IAAK,GAAc,CACrD,GAAM,CAAC,EAAc,GAAsB,EAAU,MAAM,CAAC,MAAM,IAAI,CACtE,GAAI,CAAC,EACH,MAAU,MAAM,8DAA8D,CAGhF,MAAO,CACL,IAAK,EACL,OAHsB,EAAmB,MAAM,IAGxB,CACxB,EACD,CAGF,OAFA,EAAO,KAAK,EAAO,QAAQ,uCAAuC,CAAC,CACnE,EAAO,KAAK,EAAO,QAAQ,KAAK,UAAU,EAAkB,KAAM,EAAE,CAAC,CAAC,CAC/D"}

package/dist/parseVariablesFromString-BeKOGw5n.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"parseVariablesFromString-BeKOGw5n.mjs","names":[],"sources":["../src/lib/graphql/ensureAllDataSubjectsExist.ts","../src/lib/graphql/syncDataSilos.ts","../src/lib/graphql/syncConfigurationToTranscend.ts","../src/lib/mergeTranscendInputs.ts","../src/lib/helpers/parseVariablesFromString.ts"],"sourcesContent":["import { createDataSubject, fetchAllDataSubjects, type DataSubject } from '@transcend-io/sdk';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { keyBy, flatten, uniq, difference } from 'lodash-es';\n\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\n/**\n * Fetch all of the data subjects in the organization\n *\n * @param input - Input to fetch\n * @param client - GraphQL client\n * @param fetchAll - When true, always fetch all subjects\n * @returns The list of data subjects\n */\nexport async function ensureAllDataSubjectsExist(\n  {\n    'data-silos': dataSilos = [],\n    'data-subjects': dataSubjects = [],\n    'processing-activities': processingActivities = [],\n    enrichers = [],\n  }: TranscendInput,\n  client: GraphQLClient,\n  fetchAll = false,\n): Promise<{ [type in string]: DataSubject }> {\n  const expectedDataSubjects = uniq([\n    ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []),\n    ...flatten(processingActivities.map(({ dataSubjectTypes }) => dataSubjectTypes ?? []) ?? []),\n    ...flatten(enrichers.map((enricher) => enricher['data-subjects'] || []) || []),\n    ...dataSubjects.map((subject) => subject.type),\n  ]);\n  if (expectedDataSubjects.length === 0 && !fetchAll) {\n    return {};\n  }\n\n  const internalSubjects = await fetchAllDataSubjects(client, { logger });\n  const dataSubjectByName = keyBy(internalSubjects, 'type');\n\n  const missingDataSubjects = difference(\n    expectedDataSubjects,\n    internalSubjects.map(({ type }) => type),\n  );\n\n  if (missingDataSubjects.length > 0) {\n    logger.info(colors.magenta(`Creating ${missingDataSubjects.length} new data subjects...`));\n    for (const dataSubjectType of missingDataSubjects) {\n      logger.info(colors.magenta(`Creating data subject ${dataSubjectType}...`));\n      const created = await createDataSubject(client, { input: dataSubjectType, logger });\n      logger.info(colors.green(`Created data subject ${dataSubjectType}!`));\n      dataSubjectByName[dataSubjectType] = created;\n    }\n  }\n\n  return dataSubjectByName;\n}\n","import {\n  makeGraphQLRequest,\n  ApiKey,\n  convertToDataSubjectBlockList,\n  type DataSubject,\n  fetchAllDataSilos,\n  type DataSilo,\n  CREATE_DATA_SILOS,\n  UPDATE_DATA_SILOS,\n  UPDATE_OR_CREATE_DATA_POINT,\n} from '@transcend-io/sdk';\nimport { apply } from '@transcend-io/type-utils';\nimport { mapSeries, map } from '@transcend-io/utils';\n/* eslint-disable max-lines */\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { chunk, keyBy } from 'lodash-es';\n\nimport { DataSiloInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nconst BATCH_SILOS_LIMIT = 20;\n\n/**\n * Sync a data silo configuration\n *\n * @param dataSilos - Data silos to sync\n * @param client - GraphQL client\n * @param options - Options\n * @returns Data silo info\n */\nexport async function syncDataSilos(\n  dataSilos: DataSiloInput[],\n  client: GraphQLClient,\n  {\n    pageSize,\n    dataSubjectsByName,\n    apiKeysByTitle,\n  }: {\n    /** Page size */\n    pageSize: number;\n    /** The data subjects in the organization */\n    dataSubjectsByName: { [type in string]: DataSubject };\n    /** API key title to API key */\n    apiKeysByTitle: { [title in string]: ApiKey };\n  },\n): Promise<{\n  /** Whether successfully updated */\n  success: boolean;\n  /** A mapping between data silo title to data silo ID */\n  dataSiloTitleToId: { [k in string]: string };\n}> {\n  let encounteredError = false;\n\n  // Time duration\n  const t0 = new Date().getTime();\n  logger.info(colors.magenta(`Syncing \"${dataSilos.length}\" data silos...`));\n\n  // Determine the set of data silos that already exist\n  const existingDataSilos = await fetchAllDataSilos(client, {\n    titles: dataSilos.map(({ title }) => title),\n    pageSize,\n    logger,\n  });\n\n  // Create a mapping of title -> existing silo, if it exists\n  const existingDataSiloByTitle = keyBy<Pick<DataSilo, 'id' | 'title'>>(existingDataSilos, 'title');\n\n  // Create new silos that do not exist\n  const newDataSiloInputs = dataSilos.filter(({ title }) => !existingDataSiloByTitle[title]);\n  if (newDataSiloInputs.length > 0) {\n    logger.info(\n      colors.magenta(`Creating \"${newDataSiloInputs.length}\" data silos that did not exist...`),\n    );\n\n    // Batch the creation\n    const chunked = chunk(newDataSiloInputs, BATCH_SILOS_LIMIT);\n    await mapSeries(chunked, async (dependencyUpdateChunk) => {\n      const {\n        createDataSilos: { dataSilos },\n      } = await makeGraphQLRequest<{\n        /** Mutation result */\n        createDataSilos: {\n          /** New data silos */\n          dataSilos: Pick<DataSilo, 'id' | 'title'>[];\n        };\n      }>(client, CREATE_DATA_SILOS, {\n        variables: {\n          input: dependencyUpdateChunk.map((input) => ({\n            name: input['outer-type'] || input.integrationName,\n            title: input.title,\n            country: input.country,\n            countrySubDivision: input.countrySubDivision,\n          })),\n        },\n        logger,\n      });\n\n      // save mapping of title and id\n      dataSilos.forEach((silo) => {\n        existingDataSiloByTitle[silo.title] = silo;\n      });\n    });\n\n    logger.info(colors.green(`Successfully created \"${newDataSiloInputs.length}\" data silos!`));\n  }\n\n  // Batch the updates\n  const chunkedUpdates = chunk(dataSilos, BATCH_SILOS_LIMIT);\n  await mapSeries(chunkedUpdates, async (dataSiloUpdateChunk, ind) => {\n    logger.info(\n      colors.magenta(\n        `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing \"${\n          dataSiloUpdateChunk.length\n        }\" data silos`,\n      ),\n    );\n    await makeGraphQLRequest<{\n      /** Mutation result */\n      updateDataSilos: {\n        /** New data silos */\n        dataSilos: Pick<DataSilo, 'id' | 'title'>[];\n      };\n    }>(client, UPDATE_DATA_SILOS, {\n      variables: {\n        input: {\n          dataSilos: dataSiloUpdateChunk.map((input) => ({\n            id: existingDataSiloByTitle[input.title].id,\n            country: input.country,\n            countrySubDivision: input.countrySubDivision,\n            url: input.url,\n            headers: input.headers,\n            description: input.description,\n            identifiers: input['identity-keys'],\n            isLive: !input.disabled,\n            ownerEmails: input.owners,\n            teamNames: input.teams,\n            // clear out if not specified, otherwise the update needs to be applied after\n            // all data silos are created\n            dependedOnDataSiloTitles: input['deletion-dependencies'] ? undefined : [],\n            apiKeyId: input['api-key-title']\n              ? apiKeysByTitle[input['api-key-title']].id\n              : undefined,\n            dataSubjectBlockListIds: input['data-subjects']\n              ? convertToDataSubjectBlockList(input['data-subjects'], dataSubjectsByName)\n              : undefined,\n            attributes: input.attributes,\n            businessEntityTitles: input.businessEntityTitles,\n            // AVC settings\n            notifyEmailAddress: input['email-settings']?.['notify-email-address'],\n            promptAVendorEmailSendFrequency: input['email-settings']?.['send-frequency'],\n            promptAVendorEmailSendType: input['email-settings']?.['send-type'],\n            promptAVendorEmailIncludeIdentifiersAttachment:\n              input['email-settings']?.['include-identifiers-attachment'],\n            promptAVendorEmailCompletionLinkType: input['email-settings']?.['completion-link-type'],\n            manualWorkRetryFrequency: input['email-settings']?.['manual-work-retry-frequency'],\n          })),\n        },\n      },\n      logger,\n    });\n    logger.info(\n      colors.green(\n        `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced \"${\n          dataSiloUpdateChunk.length\n        }\" data silos!`,\n      ),\n    );\n  });\n\n  // Sync datapoints\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const dataSilosWithDataPoints = dataSilos.filter(({ datapoints = [] }) => datapoints.length > 0);\n  const totalDataPoints = dataSilos\n    .map(({ datapoints = [] }) => datapoints.length)\n    .reduce((acc, count) => acc + count, 0);\n  logger.info(\n    colors.magenta(\n      `Syncing \"${totalDataPoints}\" datapoints from \"${dataSilosWithDataPoints.length}\" data silos...`,\n    ),\n  );\n  progressBar.start(totalDataPoints, 0);\n  let total = 0;\n\n  await map(\n    dataSilosWithDataPoints,\n    async ({ datapoints, title }) => {\n      if (datapoints) {\n        await mapSeries(datapoints, async (datapoint) => {\n          const fields = datapoint.fields\n            ? datapoint.fields.map(\n                ({ key, description, categories, purposes, attributes, ...rest }) =>\n                  // TODO: Support setting title separately from the 'key/name'\n                  ({\n                    name: key,\n                    description,\n                    categories: !categories\n                      ? undefined\n                      : categories.map((category) => ({\n                          ...category,\n                          name: category.name || 'Other',\n                        })),\n                    purposes: !purposes\n                      ? undefined\n                      : purposes.map((purpose) => ({\n                          ...purpose,\n                          name: purpose.name || 'Other',\n                        })),\n                    attributes,\n                    accessRequestVisibilityEnabled: rest['access-request-visibility-enabled'],\n                    erasureRequestRedactionEnabled: rest['erasure-request-redaction-enabled'],\n                  }),\n              )\n            : undefined;\n\n          const payload = {\n            dataSiloId: existingDataSiloByTitle[title].id,\n            path: datapoint.path,\n            name: datapoint.key,\n            title: datapoint.title,\n            description: datapoint.description,\n            ...(datapoint.owners\n              ? {\n                  ownerEmails: datapoint.owners,\n                }\n              : {}),\n            ...(datapoint.teams\n              ? {\n                  teamNames: datapoint.teams,\n                }\n              : {}),\n            ...(datapoint['data-collection-tag']\n              ? { dataCollectionTag: datapoint['data-collection-tag'] }\n              : {}),\n            querySuggestions: !datapoint['privacy-action-queries']\n              ? undefined\n              : Object.entries(datapoint['privacy-action-queries']).map(([key, value]) => ({\n                  requestType: key,\n                  suggestedQuery: value,\n                })),\n            enabledActions: datapoint['privacy-actions'] || [], // clear out when not specified\n            subDataPoints: fields,\n          };\n\n          // Ensure no duplicate sub-datapoints are provided\n          const subDataPointsToUpdate = (payload.subDataPoints || []).map(({ name }) => name);\n          const duplicateDataPoints = subDataPointsToUpdate.filter(\n            (name, index) => subDataPointsToUpdate.indexOf(name) !== index,\n          );\n          if (duplicateDataPoints.length > 0) {\n            logger.info(\n              colors.red(\n                `\\nCannot update datapoint \"${\n                  datapoint.key\n                }\" as it has duplicate sub-datapoints with the same name: \\n${duplicateDataPoints.join(\n                  '\\n',\n                )}`,\n              ),\n            );\n            encounteredError = true;\n          } else {\n            try {\n              await makeGraphQLRequest(client, UPDATE_OR_CREATE_DATA_POINT, {\n                variables: payload,\n                logger,\n              });\n            } catch (err) {\n              logger.info(\n                colors.red(\n                  `\\nFailed to update datapoint \"${datapoint.key}\" for data silo \"${title}\"! - \\n${err.message}`,\n                ),\n              );\n              encounteredError = true;\n            }\n          }\n          total += 1;\n          progressBar.update(total);\n        });\n      }\n    },\n    {\n      concurrency: 10,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Synced \"${dataSilos.length}\" data silos and \"${totalDataPoints}\" datapoints in \"${\n        totalTime / 1000\n      }\" seconds!`,\n    ),\n  );\n  return {\n    success: !encounteredError,\n    dataSiloTitleToId: apply(existingDataSiloByTitle, ({ id }) => id),\n  };\n}\n/* eslint-enable max-lines */\n","import {\n  fetchAllActions,\n  fetchAllAttributes,\n  fetchAllDataSubjects,\n  fetchApiKeys,\n  fetchIdentifiersAndCreateMissing,\n  syncAction,\n  syncActionItemCollections,\n  syncActionItems,\n  syncAgentFiles,\n  syncAgentFunctions,\n  syncAgents,\n  syncAttribute,\n  syncBusinessEntities,\n  syncDataCategories,\n  syncConsentManager,\n  syncCookies,\n  syncDataFlows,\n  syncDataSubject,\n  syncDataSiloDependencies,\n  syncEnricher,\n  syncIdentifier,\n  syncIntlMessages,\n  syncPartitions,\n  syncPolicies,\n  syncPrivacyCenter,\n  syncProcessingActivities,\n  syncProcessingPurposes,\n  syncPromptGroups,\n  syncPromptPartials,\n  syncPrompts,\n  syncTeams,\n  syncTemplate,\n  syncVendors,\n  type Identifier,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\n\n/* eslint-disable max-lines */\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { ensureAllDataSubjectsExist } from './ensureAllDataSubjectsExist.js';\nimport { syncDataSilos } from './syncDataSilos.js';\n\nconst CONCURRENCY = 10;\n\n/**\n * Sync the yaml input back to Transcend using the GraphQL APIs\n *\n * @param input - The yml input\n * @param client - GraphQL client\n * @param pageSize - Page size\n * @returns True if an error was encountered\n */\nexport async function syncConfigurationToTranscend(\n  input: TranscendInput,\n  client: GraphQLClient,\n  {\n    pageSize = 50,\n    // TODO: https://transcend.height.app/T-23779\n    publishToPrivacyCenter = true,\n    classifyService = false,\n    deleteExtraAttributeValues = false,\n  }: {\n    /** Page size */\n    pageSize?: number;\n    /** When true, skip publishing to privacy center */\n    publishToPrivacyCenter?: boolean;\n    /** When true, delete any attributes being synced up */\n    deleteExtraAttributeValues?: boolean;\n    /** classify data flow service if missing */\n    classifyService?: boolean;\n  },\n): Promise<boolean> {\n  let encounteredError = false;\n\n  logger.info(colors.magenta(`Fetching data with page size ${pageSize}...`));\n\n  const {\n    templates,\n    attributes,\n    actions,\n    identifiers,\n    'data-subjects': dataSubjects,\n    'business-entities': businessEntities,\n    enrichers,\n    cookies,\n    'consent-manager': consentManager,\n    'data-silos': dataSilos,\n    'data-flows': dataFlows,\n    prompts,\n    'prompt-groups': promptGroups,\n    'prompt-partials': promptPartials,\n    agents,\n    'agent-functions': agentFunctions,\n    'agent-files': agentFiles,\n    vendors,\n    'data-categories': dataCategories,\n    'processing-activities': processingActivities,\n    'processing-purposes': processingPurposes,\n    'action-items': actionItems,\n    'action-item-collections': actionItemCollections,\n    teams,\n    'privacy-center': privacyCenter,\n    messages,\n    policies,\n    partitions,\n  } = input;\n\n  const [identifierByName, dataSubjectsByName, apiKeyTitleMap] = await Promise.all([\n    // Ensure all identifiers are created and create a map from name -> identifier.id\n    enrichers || identifiers\n      ? fetchIdentifiersAndCreateMissing(client, {\n          input,\n          skipPublish: !publishToPrivacyCenter,\n          logger,\n        })\n      : ({} as { [k in string]: Identifier }),\n    // Grab all data subjects in the organization\n    dataSilos || dataSubjects || enrichers || processingActivities\n      ? ensureAllDataSubjectsExist(input, client)\n      : {},\n    // Grab API keys\n    dataSilos &&\n    dataSilos\n      .map((dataSilo) => dataSilo['api-key-title'] || [])\n      .reduce((acc, lst) => acc + lst.length, 0) > 0\n      ? fetchApiKeys(client, { apiKeyInputs: input, logger })\n      : {},\n  ]);\n\n  // Sync consent manager\n  if (consentManager) {\n    logger.info(colors.magenta('Syncing consent manager...'));\n    try {\n      await syncConsentManager(client, consentManager, { logger });\n      logger.info(colors.green('Successfully synced consent manager!'));\n    } catch (err) {\n      encounteredError = true;\n      logger.error(colors.red(`Failed to sync consent manager! - ${err.message}`));\n    }\n  }\n\n  // Sync prompts\n  if (prompts) {\n    const promptsSuccess = await syncPrompts(client, prompts, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n  if (promptPartials) {\n    const promptsSuccess = await syncPromptPartials(client, promptPartials, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n  if (promptGroups) {\n    const promptsSuccess = await syncPromptGroups(client, promptGroups, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n\n  if (teams) {\n    const teamsSuccess = await syncTeams(client, teams, { logger });\n    encounteredError = encounteredError || !teamsSuccess;\n  }\n\n  // Sync email templates\n  if (templates) {\n    logger.info(colors.magenta(`Syncing \"${templates.length}\" email templates...`));\n    await map(\n      templates,\n      async (template) => {\n        logger.info(colors.magenta(`Syncing template \"${template.title}\"...`));\n        try {\n          await syncTemplate(client, template, { logger });\n          logger.info(colors.green(`Successfully synced template \"${template.title}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync template \"${template.title}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${templates.length}\" email templates!`));\n  }\n\n  // Sync business entities\n  if (businessEntities) {\n    const businessEntitySuccess = await syncBusinessEntities(client, businessEntities, { logger });\n    encounteredError = encounteredError || !businessEntitySuccess;\n  }\n\n  // Sync vendors\n  if (vendors) {\n    const vendorsSuccess = await syncVendors(client, vendors, { logger });\n    encounteredError = encounteredError || !vendorsSuccess;\n  }\n\n  // Sync data categories\n  if (dataCategories) {\n    const dataCategoriesSuccess = await syncDataCategories(client, dataCategories, { logger });\n    encounteredError = encounteredError || !dataCategoriesSuccess;\n  }\n\n  // Sync processing purposes\n  if (processingPurposes) {\n    const processingPurposesSuccess = await syncProcessingPurposes(client, processingPurposes, {\n      logger,\n    });\n    encounteredError = encounteredError || !processingPurposesSuccess;\n  }\n\n  // Sync partitions\n  if (partitions) {\n    const partitionsSuccess = await syncPartitions(client, partitions, { logger });\n    encounteredError = encounteredError || !partitionsSuccess;\n  }\n\n  // Sync agents\n  if (agents) {\n    const agentsSuccess = await syncAgents(client, agents, { logger });\n    encounteredError = encounteredError || !agentsSuccess;\n  }\n\n  // Sync agent functions\n  if (agentFunctions) {\n    const agentFunctionsSuccess = await syncAgentFunctions(client, agentFunctions, { logger });\n    encounteredError = encounteredError || !agentFunctionsSuccess;\n  }\n\n  // Sync agent files\n  if (agentFiles) {\n    const agentFilesSuccess = await syncAgentFiles(client, agentFiles, { logger });\n    encounteredError = encounteredError || !agentFilesSuccess;\n  }\n\n  // Sync cookies\n  if (cookies) {\n    const cookiesSuccess = await syncCookies(client, cookies, { logger });\n    encounteredError = encounteredError || !cookiesSuccess;\n  }\n\n  // Sync action item collections\n  if (actionItemCollections) {\n    const actionItemCollectionsSuccess = await syncActionItemCollections(\n      client,\n      actionItemCollections,\n      { logger },\n    );\n    encounteredError = encounteredError || !actionItemCollectionsSuccess;\n  }\n\n  // Sync attributes\n  if (attributes) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${attributes.length}\" attributes...`));\n    const existingAttributes = await fetchAllAttributes(client, { logger });\n    await map(\n      attributes,\n      async (attribute) => {\n        const existing = existingAttributes.find((attr) => attr.name === attribute.name);\n\n        logger.info(colors.magenta(`Syncing attribute \"${attribute.name}\"...`));\n        try {\n          await syncAttribute(client, attribute, {\n            existingAttribute: existing,\n            deleteExtraAttributeValues,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced attribute \"${attribute.name}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(\n            colors.red(`Failed to sync attribute \"${attribute.name}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${attributes.length}\" attributes!`));\n  }\n\n  // Sync action items\n  if (actionItems) {\n    const actionItemsSuccess = await syncActionItems(client, actionItems, { logger });\n    encounteredError = encounteredError || !actionItemsSuccess;\n  }\n\n  // Sync enrichers\n  if (enrichers) {\n    logger.info(colors.magenta(`Syncing \"${enrichers.length}\" enrichers...`));\n    await map(\n      enrichers,\n      async (enricher) => {\n        logger.info(colors.magenta(`Syncing enricher \"${enricher.title}\"...`));\n        try {\n          await syncEnricher(client, {\n            input: enricher,\n            identifierByName,\n            dataSubjectsByName,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced enricher \"${enricher.title}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync enricher \"${enricher.title}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${enrichers.length}\" enrichers!`));\n  }\n\n  // Sync identifiers\n  if (identifiers) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${identifiers.length}\" identifiers...`));\n    await map(\n      identifiers,\n      async (identifier) => {\n        const existing = identifierByName[identifier.name];\n        if (!existing) {\n          throw new Error(\n            `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing identifier \"${identifier.type}\"...`));\n        try {\n          await syncIdentifier(client, {\n            input: identifier,\n            dataSubjectsByName,\n            identifierId: existing.id,\n            skipPublish: !publishToPrivacyCenter,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced identifier \"${identifier.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.info(\n            colors.red(`Failed to sync identifier \"${identifier.type}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${identifiers.length}\" identifiers!`));\n  }\n\n  // Sync actions\n  if (actions) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${actions.length}\" actions...`));\n    const existingActions = await fetchAllActions(client, { logger });\n    await map(\n      actions,\n      async (action) => {\n        const existing = existingActions.find((act) => act.type === action.type);\n        if (!existing) {\n          throw new Error(\n            `Failed to find action with type: ${action.type}. Should have already existing in the organization.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing action \"${action.type}\"...`));\n        try {\n          await syncAction(\n            client,\n            {\n              action,\n              actionId: existing.id,\n              skipPublish: !publishToPrivacyCenter,\n            },\n            { logger },\n          );\n          logger.info(colors.green(`Successfully synced action \"${action.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync action \"${action.type}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${actions.length}\" actions!`));\n  }\n\n  // Sync data subjects\n  if (dataSubjects) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${dataSubjects.length}\" data subjects...`));\n    const existingDataSubjects = await fetchAllDataSubjects(client, { logger });\n    await map(\n      dataSubjects,\n      async (dataSubject) => {\n        const existing = existingDataSubjects.find((subj) => subj.type === dataSubject.type);\n        if (!existing) {\n          throw new Error(\n            `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing data subject \"${dataSubject.type}\"...`));\n        try {\n          await syncDataSubject(client, {\n            input: dataSubject,\n            dataSubjectId: existing.id,\n            skipPublish: !publishToPrivacyCenter,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced data subject \"${dataSubject.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.info(\n            colors.red(`Failed to sync data subject \"${dataSubject.type}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${dataSubjects.length}\" data subjects!`));\n  }\n\n  // Sync data flows\n  if (dataFlows) {\n    const syncedDataFlows = await syncDataFlows(client, dataFlows, { classifyService, logger });\n    encounteredError = encounteredError || !syncedDataFlows;\n  }\n\n  // Sync privacy center\n  if (privacyCenter) {\n    const privacyCenterSuccess = await syncPrivacyCenter(client, privacyCenter, { logger });\n    encounteredError = encounteredError || !privacyCenterSuccess;\n  }\n\n  // Sync messages\n  if (messages) {\n    const messagesSuccess = await syncIntlMessages(client, messages, { logger });\n    encounteredError = encounteredError || !messagesSuccess;\n  }\n\n  // Sync policies\n  if (policies) {\n    const policiesSuccess = await syncPolicies(client, policies, { logger });\n    encounteredError = encounteredError || !policiesSuccess;\n  }\n\n  // Store dependency updates\n  const dependencyUpdates: [string, string[]][] = [];\n  // Sync data silos\n  if (dataSilos) {\n    const { success, dataSiloTitleToId } = await syncDataSilos(dataSilos, client, {\n      dataSubjectsByName,\n      apiKeysByTitle: apiKeyTitleMap,\n      pageSize,\n    });\n    dataSilos?.forEach((dataSilo) => {\n      // Queue up dependency update\n      if (dataSilo['deletion-dependencies']) {\n        dependencyUpdates.push([\n          dataSiloTitleToId[dataSilo.title],\n          dataSilo['deletion-dependencies'],\n        ]);\n      }\n    });\n    encounteredError = encounteredError || !success;\n  }\n\n  // Dependencies updated at the end after all data silos are created\n  if (dependencyUpdates.length > 0) {\n    await syncDataSiloDependencies(client, { input: dependencyUpdates, logger });\n  }\n\n  // Update processing activities\n  if (processingActivities) {\n    const processingActivitySuccess = await syncProcessingActivities(client, processingActivities, {\n      logger,\n    });\n    encounteredError = encounteredError || !processingActivitySuccess;\n  }\n\n  if (publishToPrivacyCenter) {\n    // TODO: https://transcend.height.app/T-23779\n  }\n\n  return encounteredError;\n}\n/* eslint-enable max-lines */\n","import { getEntries } from '@transcend-io/type-utils';\n\nimport { TranscendInput } from '../codecs.js';\n\n/**\n * Combine a set of TranscendInput yaml files into a single yaml\n *\n * @param base - Base input\n * @param inputs - The list of inputs\n * @returns Merged input\n */\nexport function mergeTranscendInputs(\n  base: TranscendInput,\n  ...inputs: TranscendInput[]\n): TranscendInput {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  const cloned: any = JSON.parse(JSON.stringify(base));\n  inputs.forEach((input) => {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    getEntries(input).forEach(([key, value]: [any, any]) => {\n      if (cloned[key] === undefined) {\n        cloned[key] = value;\n      } else if (Array.isArray(value)) {\n        cloned[key] = [...cloned[key], ...value];\n      } else {\n        cloned[key] = value;\n      }\n    });\n  });\n  return cloned;\n}\n","/**\n * Parse variables from string\n *\n * @param variables - Variables as string\n * @returns Variables as object\n */\nexport function parseVariablesFromString(variables: string): {\n  [k in string]: string;\n} {\n  // Parse out the variables\n  const splitVars = variables.split(',').filter((x) => !!x);\n  const vars: { [k in string]: string } = {};\n  splitVars.forEach((variable) => {\n    const [k, v] = variable.split(':');\n    if (!k || !v) {\n      throw new Error(`Invalid variable: ${variable}. Expected format: key:value`);\n    }\n    vars[k] = v;\n  });\n  return vars;\n}\n"],"mappings":"8uCAgBA,eAAsB,EACpB,CACE,aAAc,EAAY,EAAE,CAC5B,gBAAiB,EAAe,EAAE,CAClC,wBAAyB,EAAuB,EAAE,CAClD,YAAY,EAAE,EAEhB,EACA,EAAW,GACiC,CAC5C,IAAM,EAAuB,EAAK,CAChC,GAAG,EAAQ,EAAU,IAAK,GAAS,EAAK,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CACtE,GAAG,EAAQ,EAAqB,KAAK,CAAE,sBAAuB,GAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC5F,GAAG,EAAQ,EAAU,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC9E,GAAG,EAAa,IAAK,GAAY,EAAQ,KAAK,CAC/C,CAAC,CACF,GAAI,EAAqB,SAAW,GAAK,CAAC,EACxC,MAAO,EAAE,CAGX,IAAM,EAAmB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CACjE,EAAoB,EAAM,EAAkB,OAAO,CAEnD,EAAsB,EAC1B,EACA,EAAiB,KAAK,CAAE,UAAW,EAAK,CACzC,CAED,GAAI,EAAoB,OAAS,EAAG,CAClC,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAoB,OAAO,uBAAuB,CAAC,CAC1F,IAAK,IAAM,KAAmB,EAAqB,CACjD,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAgB,KAAK,CAAC,CAC1E,IAAM,EAAU,MAAM,EAAkB,EAAQ,CAAE,MAAO,EAAiB,SAAQ,CAAC,CACnF,EAAO,KAAK,EAAO,MAAM,wBAAwB,EAAgB,GAAG,CAAC,CACrE,EAAkB,GAAmB,GAIzC,OAAO,ECtBT,eAAsB,EACpB,EACA,EACA,CACE,WACA,qBACA,kBAcD,CACD,IAAI,EAAmB,GAGjB,EAAK,IAAI,MAAM,CAAC,SAAS,CAC/B,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,iBAAiB,CAAC,CAU1E,IAAM,EAA0B,EAPN,MAAM,EAAkB,EAAQ,CACxD,OAAQ,EAAU,KAAK,CAAE,WAAY,EAAM,CAC3C,WACA,SACD,CAAC,CAGuF,QAAQ,CAG3F,EAAoB,EAAU,QAAQ,CAAE,WAAY,CAAC,EAAwB,GAAO,CACtF,EAAkB,OAAS,IAC7B,EAAO,KACL,EAAO,QAAQ,aAAa,EAAkB,OAAO,oCAAoC,CAC1F,CAID,MAAM,EADU,EAAM,EAAmB,GAAkB,CAClC,KAAO,IAA0B,CACxD,GAAM,CACJ,gBAAiB,CAAE,cACjB,MAAM,EAMP,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,EAAsB,IAAK,IAAW,CAC3C,KAAM,EAAM,eAAiB,EAAM,gBACnC,MAAO,EAAM,MACb,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC3B,EAAE,CACJ,CACD,SACD,CAAC,CAGF,EAAU,QAAS,GAAS,CAC1B,EAAwB,EAAK,OAAS,GACtC,EACF,CAEF,EAAO,KAAK,EAAO,MAAM,yBAAyB,EAAkB,OAAO,eAAe,CAAC,EAI7F,IAAM,EAAiB,EAAM,EAAW,GAAkB,CAC1D,MAAM,EAAU,EAAgB,MAAO,EAAqB,IAAQ,CAClE,EAAO,KACL,EAAO,QACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,aACzC,EAAoB,OACrB,cACF,CACF,CACD,MAAM,EAMH,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,CACL,UAAW,EAAoB,IAAK,IAAW,CAC7C,GAAI,EAAwB,EAAM,OAAO,GACzC,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,IAAK,EAAM,IACX,QAAS,EAAM,QACf,YAAa,EAAM,YACnB,YAAa,EAAM,iBACnB,OAAQ,CAAC,EAAM,SACf,YAAa,EAAM,OACnB,UAAW,EAAM,MAGjB,yBAA0B,EAAM,yBAA2B,IAAA,GAAY,EAAE,CACzE,SAAU,EAAM,iBACZ,EAAe,EAAM,kBAAkB,GACvC,IAAA,GACJ,wBAAyB,EAAM,iBAC3B,EAA8B,EAAM,iBAAkB,EAAmB,CACzE,IAAA,GACJ,WAAY,EAAM,WAClB,qBAAsB,EAAM,qBAE5B,mBAAoB,EAAM,oBAAoB,wBAC9C,gCAAiC,EAAM,oBAAoB,kBAC3D,2BAA4B,EAAM,oBAAoB,aACtD,+CACE,EAAM,oBAAoB,kCAC5B,qCAAsC,EAAM,oBAAoB,wBAChE,yBAA0B,EAAM,oBAAoB,+BACrD,EAAE,CACJ,CACF,CACD,SACD,CAAC,CACF,EAAO,KACL,EAAO,MACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,YACzC,EAAoB,OACrB,eACF,CACF,EACD,CAKF,IAAM,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAA0B,EAAU,QAAQ,CAAE,aAAa,EAAE,IAAO,EAAW,OAAS,EAAE,CAC1F,EAAkB,EACrB,KAAK,CAAE,aAAa,EAAE,IAAO,EAAW,OAAO,CAC/C,QAAQ,EAAK,IAAU,EAAM,EAAO,EAAE,CACzC,EAAO,KACL,EAAO,QACL,YAAY,EAAgB,qBAAqB,EAAwB,OAAO,iBACjF,CACF,CACD,EAAY,MAAM,EAAiB,EAAE,CACrC,IAAI,EAAQ,EAEZ,MAAM,EACJ,EACA,MAAO,CAAE,aAAY,WAAY,CAC3B,GACF,MAAM,EAAU,EAAY,KAAO,IAAc,CAC/C,IAAM,EAAS,EAAU,OACrB,EAAU,OAAO,KACd,CAAE,MAAK,cAAa,aAAY,WAAU,aAAY,GAAG,MAEvD,CACC,KAAM,EACN,cACA,WAAa,EAET,EAAW,IAAK,IAAc,CAC5B,GAAG,EACH,KAAM,EAAS,MAAQ,QACxB,EAAE,CAJH,IAAA,GAKJ,SAAW,EAEP,EAAS,IAAK,IAAa,CACzB,GAAG,EACH,KAAM,EAAQ,MAAQ,QACvB,EAAE,CAJH,IAAA,GAKJ,aACA,+BAAgC,EAAK,qCACrC,+BAAgC,EAAK,qCACtC,EACJ,CACD,IAAA,GAEE,EAAU,CACd,WAAY,EAAwB,GAAO,GAC3C,KAAM,EAAU,KAChB,KAAM,EAAU,IAChB,MAAO,EAAU,MACjB,YAAa,EAAU,YACvB,GAAI,EAAU,OACV,CACE,YAAa,EAAU,OACxB,CACD,EAAE,CACN,GAAI,EAAU,MACV,CACE,UAAW,EAAU,MACtB,CACD,EAAE,CACN,GAAI,EAAU,uBACV,CAAE,kBAAmB,EAAU,uBAAwB,CACvD,EAAE,CACN,iBAAmB,EAAU,0BAEzB,OAAO,QAAQ,EAAU,0BAA0B,CAAC,KAAK,CAAC,EAAK,MAAY,CACzE,YAAa,EACb,eAAgB,EACjB,EAAE,CAJH,IAAA,GAKJ,eAAgB,EAAU,oBAAsB,EAAE,CAClD,cAAe,EAChB,CAGK,GAAyB,EAAQ,eAAiB,EAAE,EAAE,KAAK,CAAE,UAAW,EAAK,CAC7E,EAAsB,EAAsB,QAC/C,EAAM,IAAU,EAAsB,QAAQ,EAAK,GAAK,EAC1D,CACD,GAAI,EAAoB,OAAS,EAC/B,EAAO,KACL,EAAO,IACL,8BACE,EAAU,IACX,6DAA6D,EAAoB,KAChF;EACD,GACF,CACF,CACD,EAAmB,QAEnB,GAAI,CACF,MAAM,EAAmB,EAAQ,EAA6B,CAC5D,UAAW,EACX,SACD,CAAC,OACK,EAAK,CACZ,EAAO,KACL,EAAO,IACL,iCAAiC,EAAU,IAAI,mBAAmB,EAAM,SAAS,EAAI,UACtF,CACF,CACD,EAAmB,GAGvB,GAAS,EACT,EAAY,OAAO,EAAM,EACzB,EAGN,CACE,YAAa,GACd,CACF,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EASvB,OAPA,EAAO,KACL,EAAO,MACL,WAAW,EAAU,OAAO,oBAAoB,EAAgB,mBAC9D,EAAY,IACb,YACF,CACF,CACM,CACL,QAAS,CAAC,EACV,kBAAmB,EAAM,GAA0B,CAAE,QAAS,EAAG,CAClE,CCtPH,eAAsB,EACpB,EACA,EACA,CACE,WAAW,GAEX,yBAAyB,GACzB,kBAAkB,GAClB,6BAA6B,IAWb,CAClB,IAAI,EAAmB,GAEvB,EAAO,KAAK,EAAO,QAAQ,gCAAgC,EAAS,KAAK,CAAC,CAE1E,GAAM,CACJ,YACA,aACA,UACA,cACA,gBAAiB,EACjB,oBAAqB,EACrB,YACA,UACA,kBAAmB,EACnB,aAAc,EACd,aAAc,EACd,UACA,gBAAiB,EACjB,kBAAmB,EACnB,SACA,kBAAmB,EACnB,cAAe,EACf,UACA,kBAAmB,EACnB,wBAAyB,EACzB,sBAAuB,EACvB,eAAgB,EAChB,0BAA2B,EAC3B,QACA,iBAAkB,EAClB,WACA,WACA,cACE,EAEE,CAAC,EAAkB,EAAoB,IAAkB,MAAM,QAAQ,IAAI,CAE/E,GAAa,EACT,EAAiC,EAAQ,CACvC,QACA,YAAa,CAAC,EACd,SACD,CAAC,CACD,EAAE,CAEP,GAAa,GAAgB,GAAa,EACtC,EAA2B,EAAO,EAAO,CACzC,EAAE,CAEN,GACA,EACG,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,CAClD,QAAQ,EAAK,IAAQ,EAAM,EAAI,OAAQ,EAAE,CAAG,EAC3C,EAAa,EAAQ,CAAE,aAAc,EAAO,SAAQ,CAAC,CACrD,EAAE,CACP,CAAC,CAGF,GAAI,EAAgB,CAClB,EAAO,KAAK,EAAO,QAAQ,6BAA6B,CAAC,CACzD,GAAI,CACF,MAAM,EAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC5D,EAAO,KAAK,EAAO,MAAM,uCAAuC,CAAC,OAC1D,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,qCAAqC,EAAI,UAAU,CAAC,EAKhF,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAE1C,GAAI,EAAgB,CAClB,IAAM,EAAiB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CACnF,IAAuC,CAAC,EAE1C,GAAI,EAAc,CAChB,IAAM,EAAiB,MAAM,GAAiB,EAAQ,EAAc,CAAE,SAAQ,CAAC,CAC/E,IAAuC,CAAC,EAG1C,GAAI,EAAO,CACT,IAAM,EAAe,MAAM,GAAU,EAAQ,EAAO,CAAE,SAAQ,CAAC,CAC/D,IAAuC,CAAC,EA0B1C,GAtBI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,sBAAsB,CAAC,CAC/E,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAChD,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,oBAAoB,CAAC,EAIxE,EAAkB,CACpB,IAAM,EAAwB,MAAM,GAAqB,EAAQ,EAAkB,CAAE,SAAQ,CAAC,CAC9F,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAoB,CACtB,IAAM,EAA4B,MAAM,GAAuB,EAAQ,EAAoB,CACzF,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,GAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAQ,CACV,IAAM,EAAgB,MAAM,GAAW,EAAQ,EAAQ,CAAE,SAAQ,CAAC,CAClE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,EAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,EAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAuB,CACzB,IAAM,EAA+B,MAAM,EACzC,EACA,EACA,CAAE,SAAQ,CACX,CACD,IAAuC,CAAC,EAI1C,GAAI,EAAY,CAEd,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAW,OAAO,iBAAiB,CAAC,CAC3E,IAAM,EAAqB,MAAM,EAAmB,EAAQ,CAAE,SAAQ,CAAC,CACvE,MAAM,EACJ,EACA,KAAO,IAAc,CACnB,IAAM,EAAW,EAAmB,KAAM,GAAS,EAAK,OAAS,EAAU,KAAK,CAEhF,EAAO,KAAK,EAAO,QAAQ,sBAAsB,EAAU,KAAK,MAAM,CAAC,CACvE,GAAI,CACF,MAAM,GAAc,EAAQ,EAAW,CACrC,kBAAmB,EACnB,6BACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,kCAAkC,EAAU,KAAK,IAAI,CAAC,OACxE,EAAK,CACZ,EAAmB,GACnB,EAAO,MACL,EAAO,IAAI,6BAA6B,EAAU,KAAK,OAAO,EAAI,UAAU,CAC7E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAW,OAAO,eAAe,CAAC,CAIxE,GAAI,EAAa,CACf,IAAM,EAAqB,MAAM,EAAgB,EAAQ,EAAa,CAAE,SAAQ,CAAC,CACjF,IAAuC,CAAC,EAqE1C,GAjEI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,gBAAgB,CAAC,CACzE,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,CACzB,MAAO,EACP,mBACA,qBACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,cAAc,CAAC,EAIlE,IAEF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAY,OAAO,kBAAkB,CAAC,CAC7E,MAAM,EACJ,EACA,KAAO,IAAe,CACpB,IAAM,EAAW,EAAiB,EAAW,MAC7C,GAAI,CAAC,EACH,MAAU,MACR,wCAAwC,EAAW,KAAK,yCACzD,CAGH,EAAO,KAAK,EAAO,QAAQ,uBAAuB,EAAW,KAAK,MAAM,CAAC,CACzE,GAAI,CACF,MAAM,GAAe,EAAQ,CAC3B,MAAO,EACP,qBACA,aAAc,EAAS,GACvB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,mCAAmC,EAAW,KAAK,IAAI,CAAC,OAC1E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,8BAA8B,EAAW,KAAK,OAAO,EAAI,UAAU,CAC/E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAY,OAAO,gBAAgB,CAAC,EAItE,EAAS,CAEX,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAQ,OAAO,cAAc,CAAC,CACrE,IAAM,EAAkB,MAAM,EAAgB,EAAQ,CAAE,SAAQ,CAAC,CACjE,MAAM,EACJ,EACA,KAAO,IAAW,CAChB,IAAM,EAAW,EAAgB,KAAM,GAAQ,EAAI,OAAS,EAAO,KAAK,CACxE,GAAI,CAAC,EACH,MAAU,MACR,oCAAoC,EAAO,KAAK,qDACjD,CAGH,EAAO,KAAK,EAAO,QAAQ,mBAAmB,EAAO,KAAK,MAAM,CAAC,CACjE,GAAI,CACF,MAAM,EACJ,EACA,CACE,SACA,SAAU,EAAS,GACnB,YAAa,CAAC,EACf,CACD,CAAE,SAAQ,CACX,CACD,EAAO,KAAK,EAAO,MAAM,+BAA+B,EAAO,KAAK,IAAI,CAAC,OAClE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,0BAA0B,EAAO,KAAK,OAAO,EAAI,UAAU,CAAC,GAGxF,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAQ,OAAO,YAAY,CAAC,CAIlE,GAAI,EAAc,CAEhB,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAa,OAAO,oBAAoB,CAAC,CAChF,IAAM,EAAuB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CAC3E,MAAM,EACJ,EACA,KAAO,IAAgB,CACrB,IAAM,EAAW,EAAqB,KAAM,GAAS,EAAK,OAAS,EAAY,KAAK,CACpF,GAAI,CAAC,EACH,MAAU,MACR,0CAA0C,EAAY,KAAK,qDAC5D,CAGH,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAY,KAAK,MAAM,CAAC,CAC5E,GAAI,CACF,MAAM,GAAgB,EAAQ,CAC5B,MAAO,EACP,cAAe,EAAS,GACxB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,qCAAqC,EAAY,KAAK,IAAI,CAAC,OAC7E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,gCAAgC,EAAY,KAAK,OAAO,EAAI,UAAU,CAClF,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAa,OAAO,kBAAkB,CAAC,CAI7E,GAAI,EAAW,CACb,IAAM,EAAkB,MAAM,GAAc,EAAQ,EAAW,CAAE,kBAAiB,SAAQ,CAAC,CAC3F,IAAuC,CAAC,EAI1C,GAAI,EAAe,CACjB,IAAM,EAAuB,MAAM,GAAkB,EAAQ,EAAe,CAAE,SAAQ,CAAC,CACvF,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAiB,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAC5E,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CACxE,IAAuC,CAAC,EAI1C,IAAM,EAA0C,EAAE,CAElD,GAAI,EAAW,CACb,GAAM,CAAE,UAAS,qBAAsB,MAAM,EAAc,EAAW,EAAQ,CAC5E,qBACA,eAAgB,GAChB,WACD,CAAC,CACF,GAAW,QAAS,GAAa,CAE3B,EAAS,0BACX,EAAkB,KAAK,CACrB,EAAkB,EAAS,OAC3B,EAAS,yBACV,CAAC,EAEJ,CACF,IAAuC,CAAC,EAS1C,GALI,EAAkB,OAAS,GAC7B,MAAM,GAAyB,EAAQ,CAAE,MAAO,EAAmB,SAAQ,CAAC,CAI1E,EAAsB,CACxB,IAAM,EAA4B,MAAM,GAAyB,EAAQ,EAAsB,CAC7F,SACD,CAAC,CACF,IAAuC,CAAC,EAO1C,OAAO,ECneT,SAAgB,EACd,EACA,GAAG,EACa,CAEhB,IAAM,EAAc,KAAK,MAAM,KAAK,UAAU,EAAK,CAAC,CAapD,OAZA,EAAO,QAAS,GAAU,CAExB,EAAW,EAAM,CAAC,SAAS,CAAC,EAAK,KAAuB,CAClD,EAAO,KAAS,IAAA,GAClB,EAAO,GAAO,EACL,MAAM,QAAQ,EAAM,CAC7B,EAAO,GAAO,CAAC,GAAG,EAAO,GAAM,GAAG,EAAM,CAExC,EAAO,GAAO,GAEhB,EACF,CACK,ECvBT,SAAgB,EAAyB,EAEvC,CAEA,IAAM,EAAY,EAAU,MAAM,IAAI,CAAC,OAAQ,GAAM,CAAC,CAAC,EAAE,CACnD,EAAkC,EAAE,CAQ1C,OAPA,EAAU,QAAS,GAAa,CAC9B,GAAM,CAAC,EAAG,GAAK,EAAS,MAAM,IAAI,CAClC,GAAI,CAAC,GAAK,CAAC,EACT,MAAU,MAAM,qBAAqB,EAAS,8BAA8B,CAE9E,EAAK,GAAK,GACV,CACK"}
+{"version":3,"file":"parseVariablesFromString-BeKOGw5n.mjs","names":[],"sources":["../src/lib/graphql/ensureAllDataSubjectsExist.ts","../src/lib/graphql/syncDataSilos.ts","../src/lib/graphql/syncConfigurationToTranscend.ts","../src/lib/mergeTranscendInputs.ts","../src/lib/helpers/parseVariablesFromString.ts"],"sourcesContent":["import { createDataSubject, fetchAllDataSubjects, type DataSubject } from '@transcend-io/sdk';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { keyBy, flatten, uniq, difference } from 'lodash-es';\n\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\n/**\n * Fetch all of the data subjects in the organization\n *\n * @param input - Input to fetch\n * @param client - GraphQL client\n * @param fetchAll - When true, always fetch all subjects\n * @returns The list of data subjects\n */\nexport async function ensureAllDataSubjectsExist(\n  {\n    'data-silos': dataSilos = [],\n    'data-subjects': dataSubjects = [],\n    'processing-activities': processingActivities = [],\n    enrichers = [],\n  }: TranscendInput,\n  client: GraphQLClient,\n  fetchAll = false,\n): Promise<{ [type in string]: DataSubject }> {\n  const expectedDataSubjects = uniq([\n    ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []),\n    ...flatten(processingActivities.map(({ dataSubjectTypes }) => dataSubjectTypes ?? []) ?? []),\n    ...flatten(enrichers.map((enricher) => enricher['data-subjects'] || []) || []),\n    ...dataSubjects.map((subject) => subject.type),\n  ]);\n  if (expectedDataSubjects.length === 0 && !fetchAll) {\n    return {};\n  }\n\n  const internalSubjects = await fetchAllDataSubjects(client, { logger });\n  const dataSubjectByName = keyBy(internalSubjects, 'type');\n\n  const missingDataSubjects = difference(\n    expectedDataSubjects,\n    internalSubjects.map(({ type }) => type),\n  );\n\n  if (missingDataSubjects.length > 0) {\n    logger.info(colors.magenta(`Creating ${missingDataSubjects.length} new data subjects...`));\n    for (const dataSubjectType of missingDataSubjects) {\n      logger.info(colors.magenta(`Creating data subject ${dataSubjectType}...`));\n      const created = await createDataSubject(client, { input: dataSubjectType, logger });\n      logger.info(colors.green(`Created data subject ${dataSubjectType}!`));\n      dataSubjectByName[dataSubjectType] = created;\n    }\n  }\n\n  return dataSubjectByName;\n}\n","import {\n  makeGraphQLRequest,\n  ApiKey,\n  convertToDataSubjectBlockList,\n  type DataSubject,\n  fetchAllDataSilos,\n  type DataSilo,\n  CREATE_DATA_SILOS,\n  UPDATE_DATA_SILOS,\n  UPDATE_OR_CREATE_DATA_POINT,\n} from '@transcend-io/sdk';\nimport { apply } from '@transcend-io/type-utils';\nimport { mapSeries, map } from '@transcend-io/utils';\n/* eslint-disable max-lines */\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\nimport { chunk, keyBy } from 'lodash-es';\n\nimport { DataSiloInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nconst BATCH_SILOS_LIMIT = 20;\n\n/**\n * Sync a data silo configuration\n *\n * @param dataSilos - Data silos to sync\n * @param client - GraphQL client\n * @param options - Options\n * @returns Data silo info\n */\nexport async function syncDataSilos(\n  dataSilos: DataSiloInput[],\n  client: GraphQLClient,\n  {\n    pageSize,\n    dataSubjectsByName,\n    apiKeysByTitle,\n  }: {\n    /** Page size */\n    pageSize: number;\n    /** The data subjects in the organization */\n    dataSubjectsByName: { [type in string]: DataSubject };\n    /** API key title to API key */\n    apiKeysByTitle: { [title in string]: ApiKey };\n  },\n): Promise<{\n  /** Whether successfully updated */\n  success: boolean;\n  /** A mapping between data silo title to data silo ID */\n  dataSiloTitleToId: { [k in string]: string };\n}> {\n  let encounteredError = false;\n\n  // Time duration\n  const t0 = new Date().getTime();\n  logger.info(colors.magenta(`Syncing \"${dataSilos.length}\" data silos...`));\n\n  // Determine the set of data silos that already exist\n  const existingDataSilos = await fetchAllDataSilos(client, {\n    titles: dataSilos.map(({ title }) => title),\n    pageSize,\n    logger,\n  });\n\n  // Create a mapping of title -> existing silo, if it exists\n  const existingDataSiloByTitle = keyBy<Pick<DataSilo, 'id' | 'title'>>(existingDataSilos, 'title');\n\n  // Create new silos that do not exist\n  const newDataSiloInputs = dataSilos.filter(({ title }) => !existingDataSiloByTitle[title]);\n  if (newDataSiloInputs.length > 0) {\n    logger.info(\n      colors.magenta(`Creating \"${newDataSiloInputs.length}\" data silos that did not exist...`),\n    );\n\n    // Batch the creation\n    const chunked = chunk(newDataSiloInputs, BATCH_SILOS_LIMIT);\n    await mapSeries(chunked, async (dependencyUpdateChunk) => {\n      const {\n        createDataSilos: { dataSilos },\n      } = await makeGraphQLRequest<{\n        /** Mutation result */\n        createDataSilos: {\n          /** New data silos */\n          dataSilos: Pick<DataSilo, 'id' | 'title'>[];\n        };\n      }>(client, CREATE_DATA_SILOS, {\n        variables: {\n          input: dependencyUpdateChunk.map((input) => ({\n            name: input['outer-type'] || input.integrationName,\n            title: input.title,\n            country: input.country,\n            countrySubDivision: input.countrySubDivision,\n          })),\n        },\n        logger,\n      });\n\n      // save mapping of title and id\n      dataSilos.forEach((silo) => {\n        existingDataSiloByTitle[silo.title] = silo;\n      });\n    });\n\n    logger.info(colors.green(`Successfully created \"${newDataSiloInputs.length}\" data silos!`));\n  }\n\n  // Batch the updates\n  const chunkedUpdates = chunk(dataSilos, BATCH_SILOS_LIMIT);\n  await mapSeries(chunkedUpdates, async (dataSiloUpdateChunk, ind) => {\n    logger.info(\n      colors.magenta(\n        `[Batch ${ind + 1}/${chunkedUpdates.length}] Syncing \"${\n          dataSiloUpdateChunk.length\n        }\" data silos`,\n      ),\n    );\n    await makeGraphQLRequest<{\n      /** Mutation result */\n      updateDataSilos: {\n        /** New data silos */\n        dataSilos: Pick<DataSilo, 'id' | 'title'>[];\n      };\n    }>(client, UPDATE_DATA_SILOS, {\n      variables: {\n        input: {\n          dataSilos: dataSiloUpdateChunk.map((input) => ({\n            id: existingDataSiloByTitle[input.title].id,\n            country: input.country,\n            countrySubDivision: input.countrySubDivision,\n            url: input.url,\n            headers: input.headers,\n            description: input.description,\n            identifiers: input['identity-keys'],\n            isLive: !input.disabled,\n            ownerEmails: input.owners,\n            teamNames: input.teams,\n            // clear out if not specified, otherwise the update needs to be applied after\n            // all data silos are created\n            dependedOnDataSiloTitles: input['deletion-dependencies'] ? undefined : [],\n            apiKeyId: input['api-key-title']\n              ? apiKeysByTitle[input['api-key-title']].id\n              : undefined,\n            dataSubjectBlockListIds: input['data-subjects']\n              ? convertToDataSubjectBlockList(input['data-subjects'], dataSubjectsByName)\n              : undefined,\n            attributes: input.attributes,\n            businessEntityTitles: input.businessEntityTitles,\n            // AVC settings\n            notifyEmailAddress: input['email-settings']?.['notify-email-address'],\n            promptAVendorEmailSendFrequency: input['email-settings']?.['send-frequency'],\n            promptAVendorEmailSendType: input['email-settings']?.['send-type'],\n            promptAVendorEmailIncludeIdentifiersAttachment:\n              input['email-settings']?.['include-identifiers-attachment'],\n            promptAVendorEmailCompletionLinkType: input['email-settings']?.['completion-link-type'],\n            manualWorkRetryFrequency: input['email-settings']?.['manual-work-retry-frequency'],\n          })),\n        },\n      },\n      logger,\n    });\n    logger.info(\n      colors.green(\n        `[Batch ${ind + 1}/${chunkedUpdates.length}] Synced \"${\n          dataSiloUpdateChunk.length\n        }\" data silos!`,\n      ),\n    );\n  });\n\n  // Sync datapoints\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const dataSilosWithDataPoints = dataSilos.filter(({ datapoints = [] }) => datapoints.length > 0);\n  const totalDataPoints = dataSilos\n    .map(({ datapoints = [] }) => datapoints.length)\n    .reduce((acc, count) => acc + count, 0);\n  logger.info(\n    colors.magenta(\n      `Syncing \"${totalDataPoints}\" datapoints from \"${dataSilosWithDataPoints.length}\" data silos...`,\n    ),\n  );\n  progressBar.start(totalDataPoints, 0);\n  let total = 0;\n\n  await map(\n    dataSilosWithDataPoints,\n    async ({ datapoints, title }) => {\n      if (datapoints) {\n        await mapSeries(datapoints, async (datapoint) => {\n          const fields = datapoint.fields\n            ? datapoint.fields.map(\n                ({ key, description, categories, purposes, attributes, ...rest }) =>\n                  // TODO: Support setting title separately from the 'key/name'\n                  ({\n                    name: key,\n                    description,\n                    categories: !categories\n                      ? undefined\n                      : categories.map((category) => ({\n                          ...category,\n                          name: category.name || 'Other',\n                        })),\n                    purposes: !purposes\n                      ? undefined\n                      : purposes.map((purpose) => ({\n                          ...purpose,\n                          name: purpose.name || 'Other',\n                        })),\n                    attributes,\n                    accessRequestVisibilityEnabled: rest['access-request-visibility-enabled'],\n                    erasureRequestRedactionEnabled: rest['erasure-request-redaction-enabled'],\n                  }),\n              )\n            : undefined;\n\n          const payload = {\n            dataSiloId: existingDataSiloByTitle[title].id,\n            path: datapoint.path,\n            name: datapoint.key,\n            title: datapoint.title,\n            description: datapoint.description,\n            ...(datapoint.owners\n              ? {\n                  ownerEmails: datapoint.owners,\n                }\n              : {}),\n            ...(datapoint.teams\n              ? {\n                  teamNames: datapoint.teams,\n                }\n              : {}),\n            ...(datapoint['data-collection-tag']\n              ? { dataCollectionTag: datapoint['data-collection-tag'] }\n              : {}),\n            querySuggestions: !datapoint['privacy-action-queries']\n              ? undefined\n              : Object.entries(datapoint['privacy-action-queries']).map(([key, value]) => ({\n                  requestType: key,\n                  suggestedQuery: value,\n                })),\n            enabledActions: datapoint['privacy-actions'] || [], // clear out when not specified\n            subDataPoints: fields,\n          };\n\n          // Ensure no duplicate sub-datapoints are provided\n          const subDataPointsToUpdate = (payload.subDataPoints || []).map(({ name }) => name);\n          const duplicateDataPoints = subDataPointsToUpdate.filter(\n            (name, index) => subDataPointsToUpdate.indexOf(name) !== index,\n          );\n          if (duplicateDataPoints.length > 0) {\n            logger.info(\n              colors.red(\n                `\\nCannot update datapoint \"${\n                  datapoint.key\n                }\" as it has duplicate sub-datapoints with the same name: \\n${duplicateDataPoints.join(\n                  '\\n',\n                )}`,\n              ),\n            );\n            encounteredError = true;\n          } else {\n            try {\n              await makeGraphQLRequest(client, UPDATE_OR_CREATE_DATA_POINT, {\n                variables: payload,\n                logger,\n              });\n            } catch (err) {\n              logger.info(\n                colors.red(\n                  `\\nFailed to update datapoint \"${datapoint.key}\" for data silo \"${title}\"! - \\n${err.message}`,\n                ),\n              );\n              encounteredError = true;\n            }\n          }\n          total += 1;\n          progressBar.update(total);\n        });\n      }\n    },\n    {\n      concurrency: 10,\n    },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Synced \"${dataSilos.length}\" data silos and \"${totalDataPoints}\" datapoints in \"${\n        totalTime / 1000\n      }\" seconds!`,\n    ),\n  );\n  return {\n    success: !encounteredError,\n    dataSiloTitleToId: apply(existingDataSiloByTitle, ({ id }) => id),\n  };\n}\n/* eslint-enable max-lines */\n","import {\n  fetchAllActions,\n  fetchAllAttributes,\n  fetchAllDataSubjects,\n  fetchApiKeys,\n  fetchIdentifiersAndCreateMissing,\n  syncAction,\n  syncActionItemCollections,\n  syncActionItems,\n  syncAgentFiles,\n  syncAgentFunctions,\n  syncAgents,\n  syncAttribute,\n  syncBusinessEntities,\n  syncDataCategories,\n  syncConsentManager,\n  syncCookies,\n  syncDataFlows,\n  syncDataSubject,\n  syncDataSiloDependencies,\n  syncEnricher,\n  syncIdentifier,\n  syncIntlMessages,\n  syncPartitions,\n  syncPolicies,\n  syncPrivacyCenter,\n  syncProcessingActivities,\n  syncProcessingPurposes,\n  syncPromptGroups,\n  syncPromptPartials,\n  syncPrompts,\n  syncTeams,\n  syncTemplate,\n  syncVendors,\n  type Identifier,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { GraphQLClient } from 'graphql-request';\n\n/* eslint-disable max-lines */\nimport { TranscendInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\nimport { ensureAllDataSubjectsExist } from './ensureAllDataSubjectsExist.js';\nimport { syncDataSilos } from './syncDataSilos.js';\n\nconst CONCURRENCY = 10;\n\n/**\n * Sync the yaml input back to Transcend using the GraphQL APIs\n *\n * @param input - The yml input\n * @param client - GraphQL client\n * @param pageSize - Page size\n * @returns True if an error was encountered\n */\nexport async function syncConfigurationToTranscend(\n  input: TranscendInput,\n  client: GraphQLClient,\n  {\n    pageSize = 50,\n    // TODO: https://transcend.height.app/T-23779\n    publishToPrivacyCenter = true,\n    classifyService = false,\n    deleteExtraAttributeValues = false,\n  }: {\n    /** Page size */\n    pageSize?: number;\n    /** When true, skip publishing to privacy center */\n    publishToPrivacyCenter?: boolean;\n    /** When true, delete any attributes being synced up */\n    deleteExtraAttributeValues?: boolean;\n    /** classify data flow service if missing */\n    classifyService?: boolean;\n  },\n): Promise<boolean> {\n  let encounteredError = false;\n\n  logger.info(colors.magenta(`Fetching data with page size ${pageSize}...`));\n\n  const {\n    templates,\n    attributes,\n    actions,\n    identifiers,\n    'data-subjects': dataSubjects,\n    'business-entities': businessEntities,\n    enrichers,\n    cookies,\n    'consent-manager': consentManager,\n    'data-silos': dataSilos,\n    'data-flows': dataFlows,\n    prompts,\n    'prompt-groups': promptGroups,\n    'prompt-partials': promptPartials,\n    agents,\n    'agent-functions': agentFunctions,\n    'agent-files': agentFiles,\n    vendors,\n    'data-categories': dataCategories,\n    'processing-activities': processingActivities,\n    'processing-purposes': processingPurposes,\n    'action-items': actionItems,\n    'action-item-collections': actionItemCollections,\n    teams,\n    'privacy-center': privacyCenter,\n    messages,\n    policies,\n    partitions,\n  } = input;\n\n  const [identifierByName, dataSubjectsByName, apiKeyTitleMap] = await Promise.all([\n    // Ensure all identifiers are created and create a map from name -> identifier.id\n    enrichers || identifiers\n      ? fetchIdentifiersAndCreateMissing(client, {\n          input,\n          skipPublish: !publishToPrivacyCenter,\n          logger,\n        })\n      : ({} as { [k in string]: Identifier }),\n    // Grab all data subjects in the organization\n    dataSilos || dataSubjects || enrichers || processingActivities\n      ? ensureAllDataSubjectsExist(input, client)\n      : {},\n    // Grab API keys\n    dataSilos &&\n    dataSilos\n      .map((dataSilo) => dataSilo['api-key-title'] || [])\n      .reduce((acc, lst) => acc + lst.length, 0) > 0\n      ? fetchApiKeys(client, { apiKeyInputs: input, logger })\n      : {},\n  ]);\n\n  // Sync consent manager\n  if (consentManager) {\n    logger.info(colors.magenta('Syncing consent manager...'));\n    try {\n      await syncConsentManager(client, consentManager, { logger });\n      logger.info(colors.green('Successfully synced consent manager!'));\n    } catch (err) {\n      encounteredError = true;\n      logger.error(colors.red(`Failed to sync consent manager! - ${err.message}`));\n    }\n  }\n\n  // Sync prompts\n  if (prompts) {\n    const promptsSuccess = await syncPrompts(client, prompts, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n  if (promptPartials) {\n    const promptsSuccess = await syncPromptPartials(client, promptPartials, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n  if (promptGroups) {\n    const promptsSuccess = await syncPromptGroups(client, promptGroups, { logger });\n    encounteredError = encounteredError || !promptsSuccess;\n  }\n\n  if (teams) {\n    const teamsSuccess = await syncTeams(client, teams, { logger });\n    encounteredError = encounteredError || !teamsSuccess;\n  }\n\n  // Sync email templates\n  if (templates) {\n    logger.info(colors.magenta(`Syncing \"${templates.length}\" email templates...`));\n    await map(\n      templates,\n      async (template) => {\n        logger.info(colors.magenta(`Syncing template \"${template.title}\"...`));\n        try {\n          await syncTemplate(client, template, { logger });\n          logger.info(colors.green(`Successfully synced template \"${template.title}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync template \"${template.title}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${templates.length}\" email templates!`));\n  }\n\n  // Sync business entities\n  if (businessEntities) {\n    const businessEntitySuccess = await syncBusinessEntities(client, businessEntities, { logger });\n    encounteredError = encounteredError || !businessEntitySuccess;\n  }\n\n  // Sync vendors\n  if (vendors) {\n    const vendorsSuccess = await syncVendors(client, vendors, { logger });\n    encounteredError = encounteredError || !vendorsSuccess;\n  }\n\n  // Sync data categories\n  if (dataCategories) {\n    const dataCategoriesSuccess = await syncDataCategories(client, dataCategories, { logger });\n    encounteredError = encounteredError || !dataCategoriesSuccess;\n  }\n\n  // Sync processing purposes\n  if (processingPurposes) {\n    const processingPurposesSuccess = await syncProcessingPurposes(client, processingPurposes, {\n      logger,\n    });\n    encounteredError = encounteredError || !processingPurposesSuccess;\n  }\n\n  // Sync partitions\n  if (partitions) {\n    const partitionsSuccess = await syncPartitions(client, partitions, { logger });\n    encounteredError = encounteredError || !partitionsSuccess;\n  }\n\n  // Sync agents\n  if (agents) {\n    const agentsSuccess = await syncAgents(client, agents, { logger });\n    encounteredError = encounteredError || !agentsSuccess;\n  }\n\n  // Sync agent functions\n  if (agentFunctions) {\n    const agentFunctionsSuccess = await syncAgentFunctions(client, agentFunctions, { logger });\n    encounteredError = encounteredError || !agentFunctionsSuccess;\n  }\n\n  // Sync agent files\n  if (agentFiles) {\n    const agentFilesSuccess = await syncAgentFiles(client, agentFiles, { logger });\n    encounteredError = encounteredError || !agentFilesSuccess;\n  }\n\n  // Sync cookies\n  if (cookies) {\n    const cookiesSuccess = await syncCookies(client, cookies, { logger });\n    encounteredError = encounteredError || !cookiesSuccess;\n  }\n\n  // Sync action item collections\n  if (actionItemCollections) {\n    const actionItemCollectionsSuccess = await syncActionItemCollections(\n      client,\n      actionItemCollections,\n      { logger },\n    );\n    encounteredError = encounteredError || !actionItemCollectionsSuccess;\n  }\n\n  // Sync attributes\n  if (attributes) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${attributes.length}\" attributes...`));\n    const existingAttributes = await fetchAllAttributes(client, { logger });\n    await map(\n      attributes,\n      async (attribute) => {\n        const existing = existingAttributes.find((attr) => attr.name === attribute.name);\n\n        logger.info(colors.magenta(`Syncing attribute \"${attribute.name}\"...`));\n        try {\n          await syncAttribute(client, attribute, {\n            existingAttribute: existing,\n            deleteExtraAttributeValues,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced attribute \"${attribute.name}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(\n            colors.red(`Failed to sync attribute \"${attribute.name}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${attributes.length}\" attributes!`));\n  }\n\n  // Sync action items\n  if (actionItems) {\n    const actionItemsSuccess = await syncActionItems(client, actionItems, { logger });\n    encounteredError = encounteredError || !actionItemsSuccess;\n  }\n\n  // Sync enrichers\n  if (enrichers) {\n    logger.info(colors.magenta(`Syncing \"${enrichers.length}\" enrichers...`));\n    await map(\n      enrichers,\n      async (enricher) => {\n        logger.info(colors.magenta(`Syncing enricher \"${enricher.title}\"...`));\n        try {\n          await syncEnricher(client, {\n            input: enricher,\n            identifierByName,\n            dataSubjectsByName,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced enricher \"${enricher.title}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync enricher \"${enricher.title}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${enrichers.length}\" enrichers!`));\n  }\n\n  // Sync identifiers\n  if (identifiers) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${identifiers.length}\" identifiers...`));\n    await map(\n      identifiers,\n      async (identifier) => {\n        const existing = identifierByName[identifier.name];\n        if (!existing) {\n          throw new Error(\n            `Failed to find identifier with name: ${identifier.type}. Should have been auto-created by cli.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing identifier \"${identifier.type}\"...`));\n        try {\n          await syncIdentifier(client, {\n            input: identifier,\n            dataSubjectsByName,\n            identifierId: existing.id,\n            skipPublish: !publishToPrivacyCenter,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced identifier \"${identifier.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.info(\n            colors.red(`Failed to sync identifier \"${identifier.type}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${identifiers.length}\" identifiers!`));\n  }\n\n  // Sync actions\n  if (actions) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${actions.length}\" actions...`));\n    const existingActions = await fetchAllActions(client, { logger });\n    await map(\n      actions,\n      async (action) => {\n        const existing = existingActions.find((act) => act.type === action.type);\n        if (!existing) {\n          throw new Error(\n            `Failed to find action with type: ${action.type}. Should have already existing in the organization.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing action \"${action.type}\"...`));\n        try {\n          await syncAction(\n            client,\n            {\n              action,\n              actionId: existing.id,\n              skipPublish: !publishToPrivacyCenter,\n            },\n            { logger },\n          );\n          logger.info(colors.green(`Successfully synced action \"${action.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.error(colors.red(`Failed to sync action \"${action.type}\"! - ${err.message}`));\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${actions.length}\" actions!`));\n  }\n\n  // Sync data subjects\n  if (dataSubjects) {\n    // Fetch existing\n    logger.info(colors.magenta(`Syncing \"${dataSubjects.length}\" data subjects...`));\n    const existingDataSubjects = await fetchAllDataSubjects(client, { logger });\n    await map(\n      dataSubjects,\n      async (dataSubject) => {\n        const existing = existingDataSubjects.find((subj) => subj.type === dataSubject.type);\n        if (!existing) {\n          throw new Error(\n            `Failed to find data subject with type: ${dataSubject.type}. Should have already existing in the organization.`,\n          );\n        }\n\n        logger.info(colors.magenta(`Syncing data subject \"${dataSubject.type}\"...`));\n        try {\n          await syncDataSubject(client, {\n            input: dataSubject,\n            dataSubjectId: existing.id,\n            skipPublish: !publishToPrivacyCenter,\n            logger,\n          });\n          logger.info(colors.green(`Successfully synced data subject \"${dataSubject.type}\"!`));\n        } catch (err) {\n          encounteredError = true;\n          logger.info(\n            colors.red(`Failed to sync data subject \"${dataSubject.type}\"! - ${err.message}`),\n          );\n        }\n      },\n      {\n        concurrency: CONCURRENCY,\n      },\n    );\n    logger.info(colors.green(`Synced \"${dataSubjects.length}\" data subjects!`));\n  }\n\n  // Sync data flows\n  if (dataFlows) {\n    const syncedDataFlows = await syncDataFlows(client, dataFlows, { classifyService, logger });\n    encounteredError = encounteredError || !syncedDataFlows;\n  }\n\n  // Sync privacy center\n  if (privacyCenter) {\n    const privacyCenterSuccess = await syncPrivacyCenter(client, privacyCenter, { logger });\n    encounteredError = encounteredError || !privacyCenterSuccess;\n  }\n\n  // Sync messages\n  if (messages) {\n    const messagesSuccess = await syncIntlMessages(client, messages, { logger });\n    encounteredError = encounteredError || !messagesSuccess;\n  }\n\n  // Sync policies\n  if (policies) {\n    const policiesSuccess = await syncPolicies(client, policies, { logger });\n    encounteredError = encounteredError || !policiesSuccess;\n  }\n\n  // Store dependency updates\n  const dependencyUpdates: [string, string[]][] = [];\n  // Sync data silos\n  if (dataSilos) {\n    const { success, dataSiloTitleToId } = await syncDataSilos(dataSilos, client, {\n      dataSubjectsByName,\n      apiKeysByTitle: apiKeyTitleMap,\n      pageSize,\n    });\n    dataSilos?.forEach((dataSilo) => {\n      // Queue up dependency update\n      if (dataSilo['deletion-dependencies']) {\n        dependencyUpdates.push([\n          dataSiloTitleToId[dataSilo.title],\n          dataSilo['deletion-dependencies'],\n        ]);\n      }\n    });\n    encounteredError = encounteredError || !success;\n  }\n\n  // Dependencies updated at the end after all data silos are created\n  if (dependencyUpdates.length > 0) {\n    await syncDataSiloDependencies(client, { input: dependencyUpdates, logger });\n  }\n\n  // Update processing activities\n  if (processingActivities) {\n    const processingActivitySuccess = await syncProcessingActivities(client, processingActivities, {\n      logger,\n    });\n    encounteredError = encounteredError || !processingActivitySuccess;\n  }\n\n  if (publishToPrivacyCenter) {\n    // TODO: https://transcend.height.app/T-23779\n  }\n\n  return encounteredError;\n}\n/* eslint-enable max-lines */\n","import { getEntries } from '@transcend-io/type-utils';\n\nimport { TranscendInput } from '../codecs.js';\n\n/**\n * Combine a set of TranscendInput yaml files into a single yaml\n *\n * @param base - Base input\n * @param inputs - The list of inputs\n * @returns Merged input\n */\nexport function mergeTranscendInputs(\n  base: TranscendInput,\n  ...inputs: TranscendInput[]\n): TranscendInput {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  const cloned: any = JSON.parse(JSON.stringify(base));\n  inputs.forEach((input) => {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    getEntries(input).forEach(([key, value]: [any, any]) => {\n      if (cloned[key] === undefined) {\n        cloned[key] = value;\n      } else if (Array.isArray(value)) {\n        cloned[key] = [...cloned[key], ...value];\n      } else {\n        cloned[key] = value;\n      }\n    });\n  });\n  return cloned;\n}\n","/**\n * Parse variables from string\n *\n * @param variables - Variables as string\n * @returns Variables as object\n */\nexport function parseVariablesFromString(variables: string): {\n  [k in string]: string;\n} {\n  // Parse out the variables\n  const splitVars = variables.split(',').filter((x) => !!x);\n  const vars: { [k in string]: string } = {};\n  splitVars.forEach((variable) => {\n    const [k, v] = variable.split(':');\n    if (!k || !v) {\n      throw new Error(`Invalid variable: ${variable}. Expected format: key:value`);\n    }\n    vars[k] = v;\n  });\n  return vars;\n}\n"],"mappings":"8uCAgBA,eAAsB,EACpB,CACE,aAAc,EAAY,EAAE,CAC5B,gBAAiB,EAAe,EAAE,CAClC,wBAAyB,EAAuB,EAAE,CAClD,YAAY,EAAE,EAEhB,EACA,EAAW,GACiC,CAC5C,IAAM,EAAuB,EAAK,CAChC,GAAG,EAAQ,EAAU,IAAK,GAAS,EAAK,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CACtE,GAAG,EAAQ,EAAqB,KAAK,CAAE,sBAAuB,GAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC5F,GAAG,EAAQ,EAAU,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,EAAI,EAAE,CAAC,CAC9E,GAAG,EAAa,IAAK,GAAY,EAAQ,KAAK,CAC/C,CAAC,CACF,GAAI,EAAqB,SAAW,GAAK,CAAC,EACxC,MAAO,EAAE,CAGX,IAAM,EAAmB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CACjE,EAAoB,EAAM,EAAkB,OAAO,CAEnD,EAAsB,EAC1B,EACA,EAAiB,KAAK,CAAE,UAAW,EAAK,CACzC,CAED,GAAI,EAAoB,OAAS,EAAG,CAClC,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAoB,OAAO,uBAAuB,CAAC,CAC1F,IAAK,IAAM,KAAmB,EAAqB,CACjD,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAgB,KAAK,CAAC,CAC1E,IAAM,EAAU,MAAM,EAAkB,EAAQ,CAAE,MAAO,EAAiB,SAAQ,CAAC,CACnF,EAAO,KAAK,EAAO,MAAM,wBAAwB,EAAgB,GAAG,CAAC,CACrE,EAAkB,GAAmB,GAIzC,OAAO,ECtBT,eAAsB,EACpB,EACA,EACA,CACE,WACA,qBACA,kBAcD,CACD,IAAI,EAAmB,GAGjB,EAAK,IAAI,MAAM,CAAC,SAAS,CAC/B,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,iBAAiB,CAAC,CAU1E,IAAM,EAA0B,EAAsC,MAPtC,EAAkB,EAAQ,CACxD,OAAQ,EAAU,KAAK,CAAE,WAAY,EAAM,CAC3C,WACA,SACD,CAAC,CAGuF,QAAQ,CAG3F,EAAoB,EAAU,QAAQ,CAAE,WAAY,CAAC,EAAwB,GAAO,CACtF,EAAkB,OAAS,IAC7B,EAAO,KACL,EAAO,QAAQ,aAAa,EAAkB,OAAO,oCAAoC,CAC1F,CAID,MAAM,EADU,EAAM,EAAmB,GAClB,CAAE,KAAO,IAA0B,CACxD,GAAM,CACJ,gBAAiB,CAAE,cACjB,MAAM,EAMP,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,EAAsB,IAAK,IAAW,CAC3C,KAAM,EAAM,eAAiB,EAAM,gBACnC,MAAO,EAAM,MACb,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC3B,EAAE,CACJ,CACD,SACD,CAAC,CAGF,EAAU,QAAS,GAAS,CAC1B,EAAwB,EAAK,OAAS,GACtC,EACF,CAEF,EAAO,KAAK,EAAO,MAAM,yBAAyB,EAAkB,OAAO,eAAe,CAAC,EAI7F,IAAM,EAAiB,EAAM,EAAW,GAAkB,CAC1D,MAAM,EAAU,EAAgB,MAAO,EAAqB,IAAQ,CAClE,EAAO,KACL,EAAO,QACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,aACzC,EAAoB,OACrB,cACF,CACF,CACD,MAAM,EAMH,EAAQ,EAAmB,CAC5B,UAAW,CACT,MAAO,CACL,UAAW,EAAoB,IAAK,IAAW,CAC7C,GAAI,EAAwB,EAAM,OAAO,GACzC,QAAS,EAAM,QACf,mBAAoB,EAAM,mBAC1B,IAAK,EAAM,IACX,QAAS,EAAM,QACf,YAAa,EAAM,YACnB,YAAa,EAAM,iBACnB,OAAQ,CAAC,EAAM,SACf,YAAa,EAAM,OACnB,UAAW,EAAM,MAGjB,yBAA0B,EAAM,yBAA2B,IAAA,GAAY,EAAE,CACzE,SAAU,EAAM,iBACZ,EAAe,EAAM,kBAAkB,GACvC,IAAA,GACJ,wBAAyB,EAAM,iBAC3B,EAA8B,EAAM,iBAAkB,EAAmB,CACzE,IAAA,GACJ,WAAY,EAAM,WAClB,qBAAsB,EAAM,qBAE5B,mBAAoB,EAAM,oBAAoB,wBAC9C,gCAAiC,EAAM,oBAAoB,kBAC3D,2BAA4B,EAAM,oBAAoB,aACtD,+CACE,EAAM,oBAAoB,kCAC5B,qCAAsC,EAAM,oBAAoB,wBAChE,yBAA0B,EAAM,oBAAoB,+BACrD,EAAE,CACJ,CACF,CACD,SACD,CAAC,CACF,EAAO,KACL,EAAO,MACL,UAAU,EAAM,EAAE,GAAG,EAAe,OAAO,YACzC,EAAoB,OACrB,eACF,CACF,EACD,CAKF,IAAM,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAA0B,EAAU,QAAQ,CAAE,aAAa,EAAE,IAAO,EAAW,OAAS,EAAE,CAC1F,EAAkB,EACrB,KAAK,CAAE,aAAa,EAAE,IAAO,EAAW,OAAO,CAC/C,QAAQ,EAAK,IAAU,EAAM,EAAO,EAAE,CACzC,EAAO,KACL,EAAO,QACL,YAAY,EAAgB,qBAAqB,EAAwB,OAAO,iBACjF,CACF,CACD,EAAY,MAAM,EAAiB,EAAE,CACrC,IAAI,EAAQ,EAEZ,MAAM,EACJ,EACA,MAAO,CAAE,aAAY,WAAY,CAC3B,GACF,MAAM,EAAU,EAAY,KAAO,IAAc,CAC/C,IAAM,EAAS,EAAU,OACrB,EAAU,OAAO,KACd,CAAE,MAAK,cAAa,aAAY,WAAU,aAAY,GAAG,MAEvD,CACC,KAAM,EACN,cACA,WAAa,EAET,EAAW,IAAK,IAAc,CAC5B,GAAG,EACH,KAAM,EAAS,MAAQ,QACxB,EAAE,CAJH,IAAA,GAKJ,SAAW,EAEP,EAAS,IAAK,IAAa,CACzB,GAAG,EACH,KAAM,EAAQ,MAAQ,QACvB,EAAE,CAJH,IAAA,GAKJ,aACA,+BAAgC,EAAK,qCACrC,+BAAgC,EAAK,qCACtC,EACJ,CACD,IAAA,GAEE,EAAU,CACd,WAAY,EAAwB,GAAO,GAC3C,KAAM,EAAU,KAChB,KAAM,EAAU,IAChB,MAAO,EAAU,MACjB,YAAa,EAAU,YACvB,GAAI,EAAU,OACV,CACE,YAAa,EAAU,OACxB,CACD,EAAE,CACN,GAAI,EAAU,MACV,CACE,UAAW,EAAU,MACtB,CACD,EAAE,CACN,GAAI,EAAU,uBACV,CAAE,kBAAmB,EAAU,uBAAwB,CACvD,EAAE,CACN,iBAAmB,EAAU,0BAEzB,OAAO,QAAQ,EAAU,0BAA0B,CAAC,KAAK,CAAC,EAAK,MAAY,CACzE,YAAa,EACb,eAAgB,EACjB,EAAE,CAJH,IAAA,GAKJ,eAAgB,EAAU,oBAAsB,EAAE,CAClD,cAAe,EAChB,CAGK,GAAyB,EAAQ,eAAiB,EAAE,EAAE,KAAK,CAAE,UAAW,EAAK,CAC7E,EAAsB,EAAsB,QAC/C,EAAM,IAAU,EAAsB,QAAQ,EAAK,GAAK,EAC1D,CACD,GAAI,EAAoB,OAAS,EAC/B,EAAO,KACL,EAAO,IACL,8BACE,EAAU,IACX,6DAA6D,EAAoB,KAChF;EACD,GACF,CACF,CACD,EAAmB,QAEnB,GAAI,CACF,MAAM,EAAmB,EAAQ,EAA6B,CAC5D,UAAW,EACX,SACD,CAAC,OACK,EAAK,CACZ,EAAO,KACL,EAAO,IACL,iCAAiC,EAAU,IAAI,mBAAmB,EAAM,SAAS,EAAI,UACtF,CACF,CACD,EAAmB,GAGvB,GAAS,EACT,EAAY,OAAO,EAAM,EACzB,EAGN,CACE,YAAa,GACd,CACF,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EASvB,OAPA,EAAO,KACL,EAAO,MACL,WAAW,EAAU,OAAO,oBAAoB,EAAgB,mBAC9D,EAAY,IACb,YACF,CACF,CACM,CACL,QAAS,CAAC,EACV,kBAAmB,EAAM,GAA0B,CAAE,QAAS,EAAG,CAClE,CCtPH,eAAsB,EACpB,EACA,EACA,CACE,WAAW,GAEX,yBAAyB,GACzB,kBAAkB,GAClB,6BAA6B,IAWb,CAClB,IAAI,EAAmB,GAEvB,EAAO,KAAK,EAAO,QAAQ,gCAAgC,EAAS,KAAK,CAAC,CAE1E,GAAM,CACJ,YACA,aACA,UACA,cACA,gBAAiB,EACjB,oBAAqB,EACrB,YACA,UACA,kBAAmB,EACnB,aAAc,EACd,aAAc,EACd,UACA,gBAAiB,EACjB,kBAAmB,EACnB,SACA,kBAAmB,EACnB,cAAe,EACf,UACA,kBAAmB,EACnB,wBAAyB,EACzB,sBAAuB,EACvB,eAAgB,EAChB,0BAA2B,EAC3B,QACA,iBAAkB,EAClB,WACA,WACA,cACE,EAEE,CAAC,EAAkB,EAAoB,IAAkB,MAAM,QAAQ,IAAI,CAE/E,GAAa,EACT,EAAiC,EAAQ,CACvC,QACA,YAAa,CAAC,EACd,SACD,CAAC,CACD,EAAE,CAEP,GAAa,GAAgB,GAAa,EACtC,EAA2B,EAAO,EAAO,CACzC,EAAE,CAEN,GACA,EACG,IAAK,GAAa,EAAS,kBAAoB,EAAE,CAAC,CAClD,QAAQ,EAAK,IAAQ,EAAM,EAAI,OAAQ,EAAE,CAAG,EAC3C,EAAa,EAAQ,CAAE,aAAc,EAAO,SAAQ,CAAC,CACrD,EAAE,CACP,CAAC,CAGF,GAAI,EAAgB,CAClB,EAAO,KAAK,EAAO,QAAQ,6BAA6B,CAAC,CACzD,GAAI,CACF,MAAM,EAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC5D,EAAO,KAAK,EAAO,MAAM,uCAAuC,CAAC,OAC1D,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,qCAAqC,EAAI,UAAU,CAAC,EAKhF,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAE1C,GAAI,EAAgB,CAClB,IAAM,EAAiB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CACnF,IAAuC,CAAC,EAE1C,GAAI,EAAc,CAChB,IAAM,EAAiB,MAAM,GAAiB,EAAQ,EAAc,CAAE,SAAQ,CAAC,CAC/E,IAAuC,CAAC,EAG1C,GAAI,EAAO,CACT,IAAM,EAAe,MAAM,GAAU,EAAQ,EAAO,CAAE,SAAQ,CAAC,CAC/D,IAAuC,CAAC,EA0B1C,GAtBI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,sBAAsB,CAAC,CAC/E,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAChD,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,oBAAoB,CAAC,EAIxE,EAAkB,CACpB,IAAM,EAAwB,MAAM,GAAqB,EAAQ,EAAkB,CAAE,SAAQ,CAAC,CAC9F,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,GAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAoB,CACtB,IAAM,EAA4B,MAAM,GAAuB,EAAQ,EAAoB,CACzF,SACD,CAAC,CACF,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,GAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAQ,CACV,IAAM,EAAgB,MAAM,GAAW,EAAQ,EAAQ,CAAE,SAAQ,CAAC,CAClE,IAAuC,CAAC,EAI1C,GAAI,EAAgB,CAClB,IAAM,EAAwB,MAAM,EAAmB,EAAQ,EAAgB,CAAE,SAAQ,CAAC,CAC1F,IAAuC,CAAC,EAI1C,GAAI,EAAY,CACd,IAAM,EAAoB,MAAM,EAAe,EAAQ,EAAY,CAAE,SAAQ,CAAC,CAC9E,IAAuC,CAAC,EAI1C,GAAI,EAAS,CACX,IAAM,EAAiB,MAAM,GAAY,EAAQ,EAAS,CAAE,SAAQ,CAAC,CACrE,IAAuC,CAAC,EAI1C,GAAI,EAAuB,CACzB,IAAM,EAA+B,MAAM,EACzC,EACA,EACA,CAAE,SAAQ,CACX,CACD,IAAuC,CAAC,EAI1C,GAAI,EAAY,CAEd,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAW,OAAO,iBAAiB,CAAC,CAC3E,IAAM,EAAqB,MAAM,EAAmB,EAAQ,CAAE,SAAQ,CAAC,CACvE,MAAM,EACJ,EACA,KAAO,IAAc,CACnB,IAAM,EAAW,EAAmB,KAAM,GAAS,EAAK,OAAS,EAAU,KAAK,CAEhF,EAAO,KAAK,EAAO,QAAQ,sBAAsB,EAAU,KAAK,MAAM,CAAC,CACvE,GAAI,CACF,MAAM,GAAc,EAAQ,EAAW,CACrC,kBAAmB,EACnB,6BACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,kCAAkC,EAAU,KAAK,IAAI,CAAC,OACxE,EAAK,CACZ,EAAmB,GACnB,EAAO,MACL,EAAO,IAAI,6BAA6B,EAAU,KAAK,OAAO,EAAI,UAAU,CAC7E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAW,OAAO,eAAe,CAAC,CAIxE,GAAI,EAAa,CACf,IAAM,EAAqB,MAAM,EAAgB,EAAQ,EAAa,CAAE,SAAQ,CAAC,CACjF,IAAuC,CAAC,EAqE1C,GAjEI,IACF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAU,OAAO,gBAAgB,CAAC,CACzE,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,EAAO,KAAK,EAAO,QAAQ,qBAAqB,EAAS,MAAM,MAAM,CAAC,CACtE,GAAI,CACF,MAAM,GAAa,EAAQ,CACzB,MAAO,EACP,mBACA,qBACA,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,iCAAiC,EAAS,MAAM,IAAI,CAAC,OACvE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,4BAA4B,EAAS,MAAM,OAAO,EAAI,UAAU,CAAC,GAG7F,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAU,OAAO,cAAc,CAAC,EAIlE,IAEF,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAY,OAAO,kBAAkB,CAAC,CAC7E,MAAM,EACJ,EACA,KAAO,IAAe,CACpB,IAAM,EAAW,EAAiB,EAAW,MAC7C,GAAI,CAAC,EACH,MAAU,MACR,wCAAwC,EAAW,KAAK,yCACzD,CAGH,EAAO,KAAK,EAAO,QAAQ,uBAAuB,EAAW,KAAK,MAAM,CAAC,CACzE,GAAI,CACF,MAAM,GAAe,EAAQ,CAC3B,MAAO,EACP,qBACA,aAAc,EAAS,GACvB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,mCAAmC,EAAW,KAAK,IAAI,CAAC,OAC1E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,8BAA8B,EAAW,KAAK,OAAO,EAAI,UAAU,CAC/E,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAY,OAAO,gBAAgB,CAAC,EAItE,EAAS,CAEX,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAQ,OAAO,cAAc,CAAC,CACrE,IAAM,EAAkB,MAAM,EAAgB,EAAQ,CAAE,SAAQ,CAAC,CACjE,MAAM,EACJ,EACA,KAAO,IAAW,CAChB,IAAM,EAAW,EAAgB,KAAM,GAAQ,EAAI,OAAS,EAAO,KAAK,CACxE,GAAI,CAAC,EACH,MAAU,MACR,oCAAoC,EAAO,KAAK,qDACjD,CAGH,EAAO,KAAK,EAAO,QAAQ,mBAAmB,EAAO,KAAK,MAAM,CAAC,CACjE,GAAI,CACF,MAAM,EACJ,EACA,CACE,SACA,SAAU,EAAS,GACnB,YAAa,CAAC,EACf,CACD,CAAE,SAAQ,CACX,CACD,EAAO,KAAK,EAAO,MAAM,+BAA+B,EAAO,KAAK,IAAI,CAAC,OAClE,EAAK,CACZ,EAAmB,GACnB,EAAO,MAAM,EAAO,IAAI,0BAA0B,EAAO,KAAK,OAAO,EAAI,UAAU,CAAC,GAGxF,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAQ,OAAO,YAAY,CAAC,CAIlE,GAAI,EAAc,CAEhB,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAa,OAAO,oBAAoB,CAAC,CAChF,IAAM,EAAuB,MAAM,EAAqB,EAAQ,CAAE,SAAQ,CAAC,CAC3E,MAAM,EACJ,EACA,KAAO,IAAgB,CACrB,IAAM,EAAW,EAAqB,KAAM,GAAS,EAAK,OAAS,EAAY,KAAK,CACpF,GAAI,CAAC,EACH,MAAU,MACR,0CAA0C,EAAY,KAAK,qDAC5D,CAGH,EAAO,KAAK,EAAO,QAAQ,yBAAyB,EAAY,KAAK,MAAM,CAAC,CAC5E,GAAI,CACF,MAAM,GAAgB,EAAQ,CAC5B,MAAO,EACP,cAAe,EAAS,GACxB,YAAa,CAAC,EACd,SACD,CAAC,CACF,EAAO,KAAK,EAAO,MAAM,qCAAqC,EAAY,KAAK,IAAI,CAAC,OAC7E,EAAK,CACZ,EAAmB,GACnB,EAAO,KACL,EAAO,IAAI,gCAAgC,EAAY,KAAK,OAAO,EAAI,UAAU,CAClF,GAGL,CACE,YAAa,GACd,CACF,CACD,EAAO,KAAK,EAAO,MAAM,WAAW,EAAa,OAAO,kBAAkB,CAAC,CAI7E,GAAI,EAAW,CACb,IAAM,EAAkB,MAAM,GAAc,EAAQ,EAAW,CAAE,kBAAiB,SAAQ,CAAC,CAC3F,IAAuC,CAAC,EAI1C,GAAI,EAAe,CACjB,IAAM,EAAuB,MAAM,GAAkB,EAAQ,EAAe,CAAE,SAAQ,CAAC,CACvF,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAiB,EAAQ,EAAU,CAAE,SAAQ,CAAC,CAC5E,IAAuC,CAAC,EAI1C,GAAI,EAAU,CACZ,IAAM,EAAkB,MAAM,GAAa,EAAQ,EAAU,CAAE,SAAQ,CAAC,CACxE,IAAuC,CAAC,EAI1C,IAAM,EAA0C,EAAE,CAElD,GAAI,EAAW,CACb,GAAM,CAAE,UAAS,qBAAsB,MAAM,EAAc,EAAW,EAAQ,CAC5E,qBACA,eAAgB,GAChB,WACD,CAAC,CACF,GAAW,QAAS,GAAa,CAE3B,EAAS,0BACX,EAAkB,KAAK,CACrB,EAAkB,EAAS,OAC3B,EAAS,yBACV,CAAC,EAEJ,CACF,IAAuC,CAAC,EAS1C,GALI,EAAkB,OAAS,GAC7B,MAAM,GAAyB,EAAQ,CAAE,MAAO,EAAmB,SAAQ,CAAC,CAI1E,EAAsB,CACxB,IAAM,EAA4B,MAAM,GAAyB,EAAQ,EAAsB,CAC7F,SACD,CAAC,CACF,IAAuC,CAAC,EAO1C,OAAO,ECneT,SAAgB,EACd,EACA,GAAG,EACa,CAEhB,IAAM,EAAc,KAAK,MAAM,KAAK,UAAU,EAAK,CAAC,CAapD,OAZA,EAAO,QAAS,GAAU,CAExB,EAAW,EAAM,CAAC,SAAS,CAAC,EAAK,KAAuB,CAClD,EAAO,KAAS,IAAA,GAClB,EAAO,GAAO,EACL,MAAM,QAAQ,EAAM,CAC7B,EAAO,GAAO,CAAC,GAAG,EAAO,GAAM,GAAG,EAAM,CAExC,EAAO,GAAO,GAEhB,EACF,CACK,ECvBT,SAAgB,EAAyB,EAEvC,CAEA,IAAM,EAAY,EAAU,MAAM,IAAI,CAAC,OAAQ,GAAM,CAAC,CAAC,EAAE,CACnD,EAAkC,EAAE,CAQ1C,OAPA,EAAU,QAAS,GAAa,CAC9B,GAAM,CAAC,EAAG,GAAK,EAAS,MAAM,IAAI,CAClC,GAAI,CAAC,GAAK,CAAC,EACT,MAAU,MAAM,qBAAqB,EAAS,8BAA8B,CAE9E,EAAK,GAAK,GACV,CACK"}

package/dist/pullAllDatapoints-Bbmky50p.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"pullAllDatapoints-Bbmky50p.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n  type DataCategoryType,\n  SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport {\n  makeGraphQLRequest,\n  DATAPOINT_EXPORT,\n  DATA_SILO_EXPORT,\n  SUB_DATA_POINTS_COUNT,\n  type DataSiloAttributeValue,\n} from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines */\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nexport interface DataSiloCsvPreview {\n  /** ID of dataSilo */\n  id: string;\n  /** Name of dataSilo */\n  title: string;\n}\n\nexport interface DataPointCsvPreview {\n  /** ID of dataPoint */\n  id: string;\n  /** The path to this data point */\n  path: string[];\n  /** Description */\n  description: {\n    /** Default message */\n    defaultMessage: string;\n  };\n  /** Name */\n  name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n  /** ID of subDatapoint */\n  id: string;\n  /** Name (or key) of the subdatapoint */\n  name: string;\n  /** The description */\n  description?: string;\n  /** Personal data category */\n  categories: DataCategoryInput[];\n  /** Data point ID */\n  dataPointId: string;\n  /** The data silo ID */\n  dataSiloId: string;\n  /** The processing purpose for this sub datapoint */\n  purposes: ProcessingPurposeInput[];\n  /** Attribute attached to subdatapoint */\n  attributeValues?: DataSiloAttributeValue[];\n  /** Data category guesses that are output by the classifier */\n  pendingCategoryGuesses?: {\n    /** Data category being guessed */\n    category: DataCategoryInput;\n    /** Status of guess */\n    status: SubDataPointDataSubCategoryGuessStatus;\n    /** classifier version that produced the guess */\n    classifierVersion: number;\n  }[];\n}\n\nexport interface DatapointFilterOptions {\n  /** IDs of data silos to filter down */\n  dataSiloIds?: string[];\n  /** Whether to include guessed categories, defaults to only approved categories */\n  includeGuessedCategories?: boolean;\n  /** Whether or not to include attributes */\n  includeAttributes?: boolean;\n  /** Parent categories to filter down for */\n  parentCategories?: DataCategoryType[];\n  /** Sub categories to filter down for */\n  subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/**\n * Pull subdatapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n */\nasync function pullSubDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<SubDataPointCsvPreview[]> {\n  const subDataPoints: SubDataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Filters\n  const filterBy = {\n    ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n    ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n    // if parentCategories or subCategories and not includeGuessedCategories\n    ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n      ? // then only show data points with approved data categories\n        { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n      : {}),\n    ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n  };\n\n  // Build a GraphQL client\n  const {\n    subDataPoints: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Query response */\n    subDataPoints: {\n      /** Count */\n      totalCount: number;\n    };\n  }>(client, SUB_DATA_POINTS_COUNT, {\n    variables: { filterBy },\n    logger,\n  });\n\n  logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n  progressBar.start(totalCount, 0);\n  let total = 0;\n  let shouldContinue = false;\n  let cursor: string | undefined;\n  let offset = 0;\n  do {\n    try {\n      const {\n        subDataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        subDataPoints: {\n          /** List of matches */\n          nodes: SubDataPointCsvPreview[];\n        };\n      }>(\n        client,\n        gql`\n          query TranscendCliSubDataPointCsvExport(\n            $filterBy: SubDataPointFiltersInput\n            $first: Int!\n            $offset: Int!\n          ) {\n            subDataPoints(\n              filterBy: $filterBy\n              first: $first\n              offset: $offset\n              useMaster: false\n            ) {\n              nodes {\n                id\n                name\n                description\n                dataPointId\n                dataSiloId\n                purposes {\n                  name\n                  purpose\n                }\n                categories {\n                  name\n                  category\n                }\n                ${\n                  includeGuessedCategories\n                    ? `pendingCategoryGuesses {\n                  category {\n                    name\n                    category\n                  }\n                  status\n                  classifierVersion\n                }`\n                    : ''\n                }\n                ${\n                  includeAttributes\n                    ? `attributeValues {\n                  attributeKey {\n                    name\n                  }\n                  name\n                }`\n                    : ''\n                }\n              }\n            }\n          }\n        `,\n        {\n          variables: {\n            first: pageSize,\n            offset,\n            filterBy: {\n              ...filterBy,\n              // TODO: https://transcend.height.app/T-40484 - add cursor support\n              // ...(cursor ? { cursor: { id: cursor } } : {}),\n            },\n          },\n          logger,\n        },\n      );\n\n      cursor = nodes[nodes.length - 1]?.id as string;\n      subDataPoints.push(...nodes);\n      shouldContinue = nodes.length === pageSize;\n      total += nodes.length;\n      offset += nodes.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n      );\n      throw err;\n    }\n  } while (shouldContinue);\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  const sorted = sortBy(subDataPoints, 'name');\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return sorted;\n}\n\n/**\n * Pull datapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n */\nasync function pullDatapoints(\n  client: GraphQLClient,\n  {\n    dataPointIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data points to filter down */\n    dataPointIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataPointCsvPreview[]> {\n  const dataPoints: DataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n  // Group by 100\n  const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n  progressBar.start(dataPointIds.length, 0);\n  let total = 0;\n  await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n    try {\n      const {\n        dataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataPoints: {\n          /** List of matches */\n          nodes: DataPointCsvPreview[];\n        };\n      }>(client, DATAPOINT_EXPORT, {\n        variables: {\n          first: pageSize,\n          filterBy: {\n            ids: dataPointIdsGroup,\n          },\n        },\n        logger,\n      });\n\n      dataPoints.push(...nodes);\n      total += dataPointIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataPoints;\n}\n\n/**\n * Pull data silo information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n */\nasync function pullDataSilos(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data silos to filter down */\n    dataSiloIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataSiloCsvPreview[]> {\n  const dataSilos: DataSiloCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n  // Group by 100\n  const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n  progressBar.start(dataSiloIds.length, 0);\n  let total = 0;\n  await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n    try {\n      const {\n        dataSilos: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataSilos: {\n          /** List of matches */\n          nodes: DataSiloCsvPreview[];\n        };\n      }>(client, DATA_SILO_EXPORT, {\n        variables: {\n          first: pageSize,\n          filterBy: {\n            ids: dataSiloIdsGroup,\n          },\n        },\n        logger,\n      });\n\n      dataSilos.push(...nodes);\n      total += dataSiloIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataSilos;\n}\n\n/**\n * Pull all datapoints from the data inventory.\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n */\nexport async function pullAllDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<\n  (SubDataPointCsvPreview & {\n    /** Data point information */\n    dataPoint: DataPointCsvPreview;\n    /** Data silo information */\n    dataSilo: DataSiloCsvPreview;\n  })[]\n> {\n  // Subdatapoint information\n  const subDatapoints = await pullSubDatapoints(client, {\n    dataSiloIds,\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories,\n    subCategories,\n    pageSize,\n  });\n\n  // The datapoint ids to grab\n  const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n  const dataPoints = await pullDatapoints(client, {\n    dataPointIds,\n  });\n  const dataPointById = keyBy(dataPoints, 'id');\n\n  // The data silo IDs to grab\n  const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n  const dataSilos = await pullDataSilos(client, {\n    dataSiloIds: allDataSiloIds,\n  });\n  const dataSiloById = keyBy(dataSilos, 'id');\n\n  return subDatapoints.map((subDataPoint) => ({\n    ...subDataPoint,\n    dataPoint: dataPointById[subDataPoint.dataPointId],\n    dataSilo: dataSiloById[subDataPoint.dataSiloId],\n  }));\n}\n/* eslint-enable max-lines */\n"],"mappings":"wcA0FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,UAAW,CAAE,WAAU,CACvB,SACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,UAAW,CACT,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACD,SACD,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAHH,MAAM,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAAC,CAGzE,CAAC,CACsC,KAAK,CAOvC,EAAe,EAHH,MAAM,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAAC,CAG1E,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}
+{"version":3,"file":"pullAllDatapoints-Bbmky50p.mjs","names":[],"sources":["../src/lib/data-inventory/pullAllDatapoints.ts"],"sourcesContent":["import {\n  type DataCategoryType,\n  SubDataPointDataSubCategoryGuessStatus,\n} from '@transcend-io/privacy-types';\nimport {\n  makeGraphQLRequest,\n  DATAPOINT_EXPORT,\n  DATA_SILO_EXPORT,\n  SUB_DATA_POINTS_COUNT,\n  type DataSiloAttributeValue,\n} from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { gql, type GraphQLClient } from 'graphql-request';\n/* eslint-disable max-lines */\nimport { keyBy, uniq, chunk, sortBy } from 'lodash-es';\n\nimport type { DataCategoryInput, ProcessingPurposeInput } from '../../codecs.js';\nimport { logger } from '../../logger.js';\n\nexport interface DataSiloCsvPreview {\n  /** ID of dataSilo */\n  id: string;\n  /** Name of dataSilo */\n  title: string;\n}\n\nexport interface DataPointCsvPreview {\n  /** ID of dataPoint */\n  id: string;\n  /** The path to this data point */\n  path: string[];\n  /** Description */\n  description: {\n    /** Default message */\n    defaultMessage: string;\n  };\n  /** Name */\n  name: string;\n}\n\nexport interface SubDataPointCsvPreview {\n  /** ID of subDatapoint */\n  id: string;\n  /** Name (or key) of the subdatapoint */\n  name: string;\n  /** The description */\n  description?: string;\n  /** Personal data category */\n  categories: DataCategoryInput[];\n  /** Data point ID */\n  dataPointId: string;\n  /** The data silo ID */\n  dataSiloId: string;\n  /** The processing purpose for this sub datapoint */\n  purposes: ProcessingPurposeInput[];\n  /** Attribute attached to subdatapoint */\n  attributeValues?: DataSiloAttributeValue[];\n  /** Data category guesses that are output by the classifier */\n  pendingCategoryGuesses?: {\n    /** Data category being guessed */\n    category: DataCategoryInput;\n    /** Status of guess */\n    status: SubDataPointDataSubCategoryGuessStatus;\n    /** classifier version that produced the guess */\n    classifierVersion: number;\n  }[];\n}\n\nexport interface DatapointFilterOptions {\n  /** IDs of data silos to filter down */\n  dataSiloIds?: string[];\n  /** Whether to include guessed categories, defaults to only approved categories */\n  includeGuessedCategories?: boolean;\n  /** Whether or not to include attributes */\n  includeAttributes?: boolean;\n  /** Parent categories to filter down for */\n  parentCategories?: DataCategoryType[];\n  /** Sub categories to filter down for */\n  subCategories?: string[]; // TODO: https://transcend.height.app/T-40482 - do by name not ID\n}\n\n/**\n * Pull subdatapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The subdatapoints\n */\nasync function pullSubDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<SubDataPointCsvPreview[]> {\n  const subDataPoints: SubDataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Filters\n  const filterBy = {\n    ...(parentCategories.length > 0 ? { category: parentCategories } : {}),\n    ...(subCategories.length > 0 ? { subCategoryIds: subCategories } : {}),\n    // if parentCategories or subCategories and not includeGuessedCategories\n    ...(parentCategories.length + subCategories.length > 0 && !includeGuessedCategories\n      ? // then only show data points with approved data categories\n        { status: SubDataPointDataSubCategoryGuessStatus.Approved }\n      : {}),\n    ...(dataSiloIds.length > 0 ? { dataSilos: dataSiloIds } : {}),\n  };\n\n  // Build a GraphQL client\n  const {\n    subDataPoints: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Query response */\n    subDataPoints: {\n      /** Count */\n      totalCount: number;\n    };\n  }>(client, SUB_DATA_POINTS_COUNT, {\n    variables: { filterBy },\n    logger,\n  });\n\n  logger.info(colors.magenta('[Step 1/3] Pulling in all subdatapoints'));\n\n  progressBar.start(totalCount, 0);\n  let total = 0;\n  let shouldContinue = false;\n  let cursor: string | undefined;\n  let offset = 0;\n  do {\n    try {\n      const {\n        subDataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        subDataPoints: {\n          /** List of matches */\n          nodes: SubDataPointCsvPreview[];\n        };\n      }>(\n        client,\n        gql`\n          query TranscendCliSubDataPointCsvExport(\n            $filterBy: SubDataPointFiltersInput\n            $first: Int!\n            $offset: Int!\n          ) {\n            subDataPoints(\n              filterBy: $filterBy\n              first: $first\n              offset: $offset\n              useMaster: false\n            ) {\n              nodes {\n                id\n                name\n                description\n                dataPointId\n                dataSiloId\n                purposes {\n                  name\n                  purpose\n                }\n                categories {\n                  name\n                  category\n                }\n                ${\n                  includeGuessedCategories\n                    ? `pendingCategoryGuesses {\n                  category {\n                    name\n                    category\n                  }\n                  status\n                  classifierVersion\n                }`\n                    : ''\n                }\n                ${\n                  includeAttributes\n                    ? `attributeValues {\n                  attributeKey {\n                    name\n                  }\n                  name\n                }`\n                    : ''\n                }\n              }\n            }\n          }\n        `,\n        {\n          variables: {\n            first: pageSize,\n            offset,\n            filterBy: {\n              ...filterBy,\n              // TODO: https://transcend.height.app/T-40484 - add cursor support\n              // ...(cursor ? { cursor: { id: cursor } } : {}),\n            },\n          },\n          logger,\n        },\n      );\n\n      cursor = nodes[nodes.length - 1]?.id as string;\n      subDataPoints.push(...nodes);\n      shouldContinue = nodes.length === pageSize;\n      total += nodes.length;\n      offset += nodes.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for cursor ${cursor} and offset ${offset}`),\n      );\n      throw err;\n    }\n  } while (shouldContinue);\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  const sorted = sortBy(subDataPoints, 'name');\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${sorted.length} subdatapoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return sorted;\n}\n\n/**\n * Pull datapoint information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints\n */\nasync function pullDatapoints(\n  client: GraphQLClient,\n  {\n    dataPointIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data points to filter down */\n    dataPointIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataPointCsvPreview[]> {\n  const dataPoints: DataPointCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 2/3] Fetching metadata for ${dataPointIds.length} datapoints`));\n\n  // Group by 100\n  const dataPointsGrouped = chunk(dataPointIds, pageSize);\n\n  progressBar.start(dataPointIds.length, 0);\n  let total = 0;\n  await mapSeries(dataPointsGrouped, async (dataPointIdsGroup) => {\n    try {\n      const {\n        dataPoints: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataPoints: {\n          /** List of matches */\n          nodes: DataPointCsvPreview[];\n        };\n      }>(client, DATAPOINT_EXPORT, {\n        variables: {\n          first: pageSize,\n          filterBy: {\n            ids: dataPointIdsGroup,\n          },\n        },\n        logger,\n      });\n\n      dataPoints.push(...nodes);\n      total += dataPointIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching subdatapoints for IDs ${dataPointIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataPoints.length} dataPoints in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataPoints;\n}\n\n/**\n * Pull data silo information\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The data silos\n */\nasync function pullDataSilos(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    pageSize = 100,\n  }: {\n    /** IDs of data silos to filter down */\n    dataSiloIds: string[];\n    /** Page size to pull in */\n    pageSize?: number;\n  },\n): Promise<DataSiloCsvPreview[]> {\n  const dataSilos: DataSiloCsvPreview[] = [];\n\n  // Time duration\n  const t0 = new Date().getTime();\n\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  logger.info(colors.magenta(`[Step 3/3] Fetching metadata for ${dataSiloIds.length} data silos`));\n\n  // Group by 100\n  const dataSilosGrouped = chunk(dataSiloIds, pageSize);\n\n  progressBar.start(dataSiloIds.length, 0);\n  let total = 0;\n  await mapSeries(dataSilosGrouped, async (dataSiloIdsGroup) => {\n    try {\n      const {\n        dataSilos: { nodes },\n      } = await makeGraphQLRequest<{\n        /** Query response */\n        dataSilos: {\n          /** List of matches */\n          nodes: DataSiloCsvPreview[];\n        };\n      }>(client, DATA_SILO_EXPORT, {\n        variables: {\n          first: pageSize,\n          filterBy: {\n            ids: dataSiloIdsGroup,\n          },\n        },\n        logger,\n      });\n\n      dataSilos.push(...nodes);\n      total += dataSiloIdsGroup.length;\n      progressBar.update(total);\n    } catch (err) {\n      logger.error(\n        colors.red(`An error fetching data silos for IDs ${dataSiloIdsGroup.join(', ')}`),\n      );\n      throw err;\n    }\n  });\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled in ${dataSilos.length} data silos in ${totalTime / 1000} seconds!`,\n    ),\n  );\n  return dataSilos;\n}\n\n/**\n * Pull all datapoints from the data inventory.\n *\n * @param client - Client to use for the request\n * @param options - Options\n * @returns The datapoints and data silos\n */\nexport async function pullAllDatapoints(\n  client: GraphQLClient,\n  {\n    dataSiloIds = [],\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories = [],\n    subCategories = [],\n    pageSize = 1000,\n  }: DatapointFilterOptions & {\n    /** Page size to pull in */\n    pageSize?: number;\n  } = {},\n): Promise<\n  (SubDataPointCsvPreview & {\n    /** Data point information */\n    dataPoint: DataPointCsvPreview;\n    /** Data silo information */\n    dataSilo: DataSiloCsvPreview;\n  })[]\n> {\n  // Subdatapoint information\n  const subDatapoints = await pullSubDatapoints(client, {\n    dataSiloIds,\n    includeGuessedCategories,\n    includeAttributes,\n    parentCategories,\n    subCategories,\n    pageSize,\n  });\n\n  // The datapoint ids to grab\n  const dataPointIds = uniq(subDatapoints.map((point) => point.dataPointId));\n  const dataPoints = await pullDatapoints(client, {\n    dataPointIds,\n  });\n  const dataPointById = keyBy(dataPoints, 'id');\n\n  // The data silo IDs to grab\n  const allDataSiloIds = uniq(subDatapoints.map((point) => point.dataSiloId));\n  const dataSilos = await pullDataSilos(client, {\n    dataSiloIds: allDataSiloIds,\n  });\n  const dataSiloById = keyBy(dataSilos, 'id');\n\n  return subDatapoints.map((subDataPoint) => ({\n    ...subDataPoint,\n    dataPoint: dataPointById[subDataPoint.dataPointId],\n    dataSilo: dataSiloById[subDataPoint.dataSiloId],\n  }));\n}\n/* eslint-enable max-lines */\n"],"mappings":"wcA0FA,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAC6B,CACnC,IAAM,EAA0C,EAAE,CAG5C,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAG/E,EAAW,CACf,GAAI,EAAiB,OAAS,EAAI,CAAE,SAAU,EAAkB,CAAG,EAAE,CACrE,GAAI,EAAc,OAAS,EAAI,CAAE,eAAgB,EAAe,CAAG,EAAE,CAErE,GAAI,EAAiB,OAAS,EAAc,OAAS,GAAK,CAAC,EAEvD,CAAE,OAAQ,EAAuC,SAAU,CAC3D,EAAE,CACN,GAAI,EAAY,OAAS,EAAI,CAAE,UAAW,EAAa,CAAG,EAAE,CAC7D,CAGK,CACJ,cAAe,CAAE,eACf,MAAM,EAMP,EAAQ,EAAuB,CAChC,UAAW,CAAE,WAAU,CACvB,SACD,CAAC,CAEF,EAAO,KAAK,EAAO,QAAQ,0CAA0C,CAAC,CAEtE,EAAY,MAAM,EAAY,EAAE,CAChC,IAAI,EAAQ,EACR,EAAiB,GACjB,EACA,EAAS,EACb,EACE,IAAI,CACF,GAAM,CACJ,cAAe,CAAE,UACf,MAAM,EAOR,EACA,CAAG;;;;;;;;;;;;;;;;;;;;;;;;;;kBA2BO,EACI;;;;;;;mBAQA,GACL;kBAEC,EACI;;;;;mBAMA,GACL;;;;UAKT,CACE,UAAW,CACT,MAAO,EACP,SACA,SAAU,CACR,GAAG,EAGJ,CACF,CACD,SACD,CACF,CAED,EAAS,EAAM,EAAM,OAAS,IAAI,GAClC,EAAc,KAAK,GAAG,EAAM,CAC5B,EAAiB,EAAM,SAAW,EAClC,GAAS,EAAM,OACf,GAAU,EAAM,OAChB,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,8CAA8C,EAAO,cAAc,IAAS,CACxF,CACK,QAED,GAET,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAEjB,EAAS,EAAO,EAAe,OAAO,CAO5C,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAO,OAAO,oBAAoB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,eAAe,EAAE,CACjB,WAAW,KAOmB,CAChC,IAAM,EAAoC,EAAE,CAGtC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAa,OAAO,aAAa,CAAC,CAGjG,IAAM,EAAoB,EAAM,EAAc,EAAS,CAEvD,EAAY,MAAM,EAAa,OAAQ,EAAE,CACzC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAmB,KAAO,IAAsB,CAC9D,GAAI,CACF,GAAM,CACJ,WAAY,CAAE,UACZ,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAW,KAAK,GAAG,EAAM,CACzB,GAAS,EAAkB,OAC3B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,2CAA2C,EAAkB,KAAK,KAAK,GAAG,CACtF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAW,OAAO,iBAAiB,EAAY,IAAK,WAC/E,CACF,CACM,EAUT,eAAe,EACb,EACA,CACE,cAAc,EAAE,CAChB,WAAW,KAOkB,CAC/B,IAAM,EAAkC,EAAE,CAGpC,EAAK,IAAI,MAAM,CAAC,SAAS,CAGzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAErF,EAAO,KAAK,EAAO,QAAQ,oCAAoC,EAAY,OAAO,aAAa,CAAC,CAGhG,IAAM,EAAmB,EAAM,EAAa,EAAS,CAErD,EAAY,MAAM,EAAY,OAAQ,EAAE,CACxC,IAAI,EAAQ,EACZ,MAAM,EAAU,EAAkB,KAAO,IAAqB,CAC5D,GAAI,CACF,GAAM,CACJ,UAAW,CAAE,UACX,MAAM,EAMP,EAAQ,EAAkB,CAC3B,UAAW,CACT,MAAO,EACP,SAAU,CACR,IAAK,EACN,CACF,CACD,SACD,CAAC,CAEF,EAAU,KAAK,GAAG,EAAM,CACxB,GAAS,EAAiB,OAC1B,EAAY,OAAO,EAAM,OAClB,EAAK,CAIZ,MAHA,EAAO,MACL,EAAO,IAAI,wCAAwC,EAAiB,KAAK,KAAK,GAAG,CAClF,CACK,IAER,CAEF,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAOvB,OALA,EAAO,KACL,EAAO,MACL,0BAA0B,EAAU,OAAO,iBAAiB,EAAY,IAAK,WAC9E,CACF,CACM,EAUT,eAAsB,EACpB,EACA,CACE,cAAc,EAAE,CAChB,2BACA,oBACA,mBAAmB,EAAE,CACrB,gBAAgB,EAAE,CAClB,WAAW,KAIT,EAAE,CAQN,CAEA,IAAM,EAAgB,MAAM,EAAkB,EAAQ,CACpD,cACA,2BACA,oBACA,mBACA,gBACA,WACD,CAAC,CAOI,EAAgB,EAAM,MAHH,EAAe,EAAQ,CAC9C,aAFmB,EAAK,EAAc,IAAK,GAAU,EAAM,YAAY,CAE3D,CACb,CAAC,CACsC,KAAK,CAOvC,EAAe,EAAM,MAHH,EAAc,EAAQ,CAC5C,YAFqB,EAAK,EAAc,IAAK,GAAU,EAAM,WAAW,CAE7C,CAC5B,CAAC,CACoC,KAAK,CAE3C,OAAO,EAAc,IAAK,IAAkB,CAC1C,GAAG,EACH,UAAW,EAAc,EAAa,aACtC,SAAU,EAAa,EAAa,YACrC,EAAE"}

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-D2oMDSTM.mjs → pullChunkedCustomSiloOutstandingIdentifiers-CPY9eRLF.mjs}

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-BOycJ_AQ.mjs";import{t}from"./logger-Bj782ZYD.mjs";import{decodeCodec as n}from"@transcend-io/type-utils";import r from"colors";import*as i from"io-ts";import{REDUCED_REQUESTS_FOR_DATA_SILO_COUNT as a,buildTranscendGraphQLClient as o,createSombraGotInstance as s,makeGraphQLRequest as c,withTransientRetry as l}from"@transcend-io/sdk";import{mapSeries as u}from"@transcend-io/utils";import d from"cli-progress";async function f(e,{dataSiloId:n}){let{listReducedRequestsForDataSilo:{totalCount:r}}=await c(e,a,{variables:{input:{dataSiloId:n,isResolved:!1}},logger:t});return r}const p=i.type({id:i.string,name:i.string,partition:i.string}),m=i.intersection([i.type({identifier:i.string,type:i.string,coreIdentifier:i.string,dataSiloId:i.string,requestId:i.string,nonce:i.string,requestCreatedAt:i.string,daysUntilOverdue:i.number,attributes:i.array(i.type({key:i.string,values:i.array(i.string)}))}),i.partial({partition:i.union([p,i.null])})]);async function h(e,{dataSiloId:r,limit:a=100,offset:o=0,requestType:s}){try{let c=await l(`pullCronPageOfIdentifiers`,()=>e.get(`v1/data-silo/${r}/pending-requests/${s}`,{searchParams:{offset:o,limit:a}}).json(),{logger:t,maxAttempts:6,baseDelayMs:500}),{items:u}=n(i.type({items:i.array(m)}),c);return u}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function g({dataSiloId:n,auth:i,sombraAuth:a,actions:c,apiPageSize:l=100,savePageSize:p=1e3,onSave:m,transcendUrl:g=e,skipRequestCount:_=!1}){if(p%l!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${p}, apiPageSize: ${l}`);let v=await s(g,i,{logger:t,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),y=o(g,i),b=0;_||(b=await f(y,{dataSiloId:n})),t.info(r.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${n}" for requests of types "${c.join(`", "`)}"`));let x=new Date().getTime(),S=new d.SingleBar({},d.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await u(c,async e=>{let i=0,a=!0;for(;a;){let o=await h(v,{dataSiloId:n,limit:l,offset:i,requestType:e}),s=o.map(t=>(C.add(t.requestId),{...t,action:e})),c=s.map(({attributes:e,partition:t,...n})=>({...n,...t?{partitionId:t.id,partitionName:t.name,partitionKey:t.partition}:{},...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...s),T.push(...c),T.length>=p&&(await m(T),T=[]),a=o.length===l,i+=l,_?t.info(r.magenta(`Pulled ${o.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await m(T),_||S.stop();let E=new Date().getTime()-x;return t.info(r.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{f as a,h as i,m as n,p as r,g as t};
-//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-D2oMDSTM.mjs.map
+import{a as e}from"./constants-CeIAxxnY.mjs";import{t}from"./logger-Bj782ZYD.mjs";import{decodeCodec as n}from"@transcend-io/type-utils";import r from"colors";import*as i from"io-ts";import{REDUCED_REQUESTS_FOR_DATA_SILO_COUNT as a,buildTranscendGraphQLClient as o,createSombraGotInstance as s,makeGraphQLRequest as c,withTransientRetry as l}from"@transcend-io/sdk";import{mapSeries as u}from"@transcend-io/utils";import d from"cli-progress";async function f(e,{dataSiloId:n}){let{listReducedRequestsForDataSilo:{totalCount:r}}=await c(e,a,{variables:{input:{dataSiloId:n,isResolved:!1}},logger:t});return r}const p=i.type({id:i.string,name:i.string,partition:i.string}),m=i.intersection([i.type({identifier:i.string,type:i.string,coreIdentifier:i.string,dataSiloId:i.string,requestId:i.string,nonce:i.string,requestCreatedAt:i.string,daysUntilOverdue:i.number,attributes:i.array(i.type({key:i.string,values:i.array(i.string)}))}),i.partial({partition:i.union([p,i.null])})]);async function h(e,{dataSiloId:r,limit:a=100,offset:o=0,requestType:s}){try{let c=await l(`pullCronPageOfIdentifiers`,()=>e.get(`v1/data-silo/${r}/pending-requests/${s}`,{searchParams:{offset:o,limit:a}}).json(),{logger:t,maxAttempts:6,baseDelayMs:500}),{items:u}=n(i.type({items:i.array(m)}),c);return u}catch(e){throw Error(`Received an error from server: ${e?.response?.body||e?.message}`)}}async function g({dataSiloId:n,auth:i,sombraAuth:a,actions:c,apiPageSize:l=100,savePageSize:p=1e3,onSave:m,transcendUrl:g=e,skipRequestCount:_=!1}){if(p%l!==0)throw Error(`savePageSize must be a multiple of apiPageSize. savePageSize: ${p}, apiPageSize: ${l}`);let v=await s(g,i,{logger:t,sombraApiKey:a,sombraUrl:process.env.SOMBRA_URL}),y=o(g,i),b=0;_||(b=await f(y,{dataSiloId:n})),t.info(r.magenta(`Pulling ${_?`all`:b} outstanding request identifiers for data silo: "${n}" for requests of types "${c.join(`", "`)}"`));let x=new Date().getTime(),S=new d.SingleBar({},d.Presets.shades_classic),C=new Set,w=[],T=[];_||S.start(b,0),await u(c,async e=>{let i=0,a=!0;for(;a;){let o=await h(v,{dataSiloId:n,limit:l,offset:i,requestType:e}),s=o.map(t=>(C.add(t.requestId),{...t,action:e})),c=s.map(({attributes:e,partition:t,...n})=>({...n,...t?{partitionId:t.id,partitionName:t.name,partitionKey:t.partition}:{},...e.reduce((e,t)=>Object.assign(e,{[t.key]:t.values.join(`,`)}),{})}));w.push(...s),T.push(...c),T.length>=p&&(await m(T),T=[]),a=o.length===l,i+=l,_?t.info(r.magenta(`Pulled ${o.length} outstanding identifiers for ${C.size} requests`)):S.update(C.size)}}),T.length>0&&await m(T),_||S.stop();let E=new Date().getTime()-x;return t.info(r.green(`Successfully pulled ${w.length} outstanding identifiers from ${C.size} requests in "${E/1e3}" seconds!`)),{identifiers:w}}export{f as a,h as i,m as n,p as r,g as t};
+//# sourceMappingURL=pullChunkedCustomSiloOutstandingIdentifiers-CPY9eRLF.mjs.map

package/dist/{pullChunkedCustomSiloOutstandingIdentifiers-D2oMDSTM.mjs.map → pullChunkedCustomSiloOutstandingIdentifiers-CPY9eRLF.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-D2oMDSTM.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { makeGraphQLRequest, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from '@transcend-io/sdk';\nimport { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/**\n * Get number of open requests for a data silo\n *\n * @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n */\nexport async function fetchRequestDataSiloActiveCount(\n  client: GraphQLClient,\n  {\n    dataSiloId,\n  }: {\n    /** Data silo ID */\n    dataSiloId: string;\n  },\n): Promise<number> {\n  const {\n    listReducedRequestsForDataSilo: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Requests */\n    listReducedRequestsForDataSilo: {\n      /** Total count */\n      totalCount: number;\n    };\n  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n    variables: {\n      input: {\n        dataSiloId,\n        isResolved: false,\n      },\n    },\n    logger,\n  });\n\n  return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { withTransientRetry } from '@transcend-io/sdk';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\n/** Consent partition metadata when a DSR is scoped to a partition */\nexport const CronPartition = t.type({\n  /** ID of the partition */\n  id: t.string,\n  /** The human readable name of the partition */\n  name: t.string,\n  /** The unique identifying partition value */\n  partition: t.string,\n});\n\n/** Type override */\nexport type CronPartition = t.TypeOf<typeof CronPartition>;\n\nexport const CronIdentifier = t.intersection([\n  t.type({\n    /** The identifier value */\n    identifier: t.string,\n    /** The type of identifier */\n    type: t.string,\n    /** The core identifier of the request */\n    coreIdentifier: t.string,\n    /** The ID of the underlying data silo */\n    dataSiloId: t.string,\n    /** The ID of the underlying request */\n    requestId: t.string,\n    /** The request nonce */\n    nonce: t.string,\n    /** The time the request was created */\n    requestCreatedAt: t.string,\n    /** The number of days until the request is overdue */\n    daysUntilOverdue: t.number,\n    /** Request attributes */\n    attributes: t.array(\n      t.type({\n        key: t.string,\n        values: t.array(t.string),\n      }),\n    ),\n  }),\n  t.partial({\n    /** The consent partition that scopes this request, when applicable */\n    partition: t.union([CronPartition, t.null]),\n  }),\n]);\n\n/** Type override */\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/**\n * Pull a offset of identifiers for a cron job\n *\n * @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n */\nexport async function pullCronPageOfIdentifiers(\n  sombra: Got,\n  {\n    dataSiloId,\n    limit = 100,\n    offset = 0,\n    requestType,\n  }: {\n    /** Data Silo ID */\n    dataSiloId: string;\n    /** Type of request */\n    requestType: RequestAction;\n    /** Number of identifiers to pull in */\n    limit?: number;\n    /** Page to pull in */\n    offset?: number;\n  },\n): Promise<CronIdentifier[]> {\n  try {\n    // `GET pending-requests` is a read and therefore safe to retry on transient\n    // gateway / network errors. Customers running `transcend cron\n    // pull-identifiers` against large silos would otherwise see the command\n    // abort on a single 502 from the Sombra reverse tunnel.\n    const response = await withTransientRetry(\n      'pullCronPageOfIdentifiers',\n      () =>\n        sombra\n          .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n            searchParams: {\n              offset,\n              limit,\n            },\n          })\n          .json(),\n      { logger, maxAttempts: 6, baseDelayMs: 500 },\n    );\n\n    const { items } = decodeCodec(\n      t.type({\n        items: t.array(CronIdentifier),\n      }),\n      response,\n    );\n    return items;\n  } catch (err) {\n    throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n  }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { buildTranscendGraphQLClient, createSombraGotInstance } from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchRequestDataSiloActiveCount } from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/**\n * A CSV formatted identifier\n */\nexport type CsvFormattedIdentifier = {\n  [k in string]: string | null | boolean | number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n  /** The request action that the identifier relates to */\n  action: RequestAction;\n}\n\n/**\n * Pull the set of identifiers outstanding for a cron or AVC integration\n *\n * This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n *\n * @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n */\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n  dataSiloId,\n  auth,\n  sombraAuth,\n  actions,\n  apiPageSize = 100,\n  savePageSize = 1000,\n  onSave,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  skipRequestCount = false,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Data Silo ID to pull down jobs for */\n  dataSiloId: string;\n  /** The request actions to fetch */\n  actions: RequestAction[];\n  /** How many identifiers to pull in a single call to the backend */\n  apiPageSize: number;\n  /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */\n  savePageSize: number;\n  /** Callback function called when a chunk of identifiers is ready to be saved */\n  onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Skip request count */\n  skipRequestCount?: boolean;\n}): Promise<{\n  /** Raw Identifiers */\n  identifiers: CronIdentifierWithAction[];\n}> {\n  // Validate savePageSize\n  if (savePageSize % apiPageSize !== 0) {\n    throw new Error(\n      `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n    );\n  }\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  let totalRequestCount = 0;\n  if (!skipRequestCount) {\n    totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n      dataSiloId,\n    });\n  }\n\n  logger.info(\n    colors.magenta(\n      `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n        `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const foundRequestIds = new Set<string>();\n\n  // identifiers found in total\n  const identifiers: CronIdentifierWithAction[] = [];\n  // current chunk of identifiers to be saved\n  let currentChunk: CsvFormattedIdentifier[] = [];\n\n  // map over each action\n  if (!skipRequestCount) {\n    progressBar.start(totalRequestCount, 0);\n  }\n  await mapSeries(actions, async (action) => {\n    let offset = 0;\n    let shouldContinue = true;\n\n    // Fetch a page of identifiers\n    while (shouldContinue) {\n      const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n        dataSiloId,\n        limit: apiPageSize,\n        offset,\n        requestType: action,\n      });\n\n      const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n        (identifier) => {\n          foundRequestIds.add(identifier.requestId);\n          return {\n            ...identifier,\n            action,\n          };\n        },\n      );\n\n      const csvFormattedIdentifiers = identifiersWithAction.map(\n        ({ attributes, partition, ...identifier }) => ({\n          ...identifier,\n          ...(partition\n            ? {\n                partitionId: partition.id,\n                partitionName: partition.name,\n                partitionKey: partition.partition,\n              }\n            : {}),\n          ...attributes.reduce(\n            (acc, val) =>\n              Object.assign(acc, {\n                [val.key]: val.values.join(','),\n              }),\n            {},\n          ),\n        }),\n      );\n\n      identifiers.push(...identifiersWithAction);\n      currentChunk.push(...csvFormattedIdentifiers);\n\n      // Check if we've reached the savePageSize and call the onSave callback\n      if (currentChunk.length >= savePageSize) {\n        await onSave(currentChunk);\n        currentChunk = [];\n      }\n\n      shouldContinue = pageIdentifiers.length === apiPageSize;\n      offset += apiPageSize;\n      if (!skipRequestCount) {\n        progressBar.update(foundRequestIds.size);\n      } else {\n        logger.info(\n          colors.magenta(\n            `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n          ),\n        );\n      }\n    }\n  });\n\n  // Save any remaining identifiers in the current chunk\n  if (currentChunk.length > 0) {\n    await onSave(currentChunk);\n  }\n\n  if (!skipRequestCount) {\n    progressBar.stop();\n  }\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n        foundRequestIds.size\n      } requests in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n\n  return { identifiers };\n}\n"],"mappings":"0bAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,UAAW,CACT,MAAO,CACL,aACA,WAAY,GACb,CACF,CACD,SACD,CAAC,CAEF,OAAO,EC9BT,MAAa,EAAgB,EAAE,KAAK,CAElC,GAAI,EAAE,OAEN,KAAM,EAAE,OAER,UAAW,EAAE,OACd,CAAC,CAKW,EAAiB,EAAE,aAAa,CAC3C,EAAE,KAAK,CAEL,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CACF,EAAE,QAAQ,CAER,UAAW,EAAE,MAAM,CAAC,EAAe,EAAE,KAAK,CAAC,CAC5C,CAAC,CACH,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAKF,IAAM,EAAW,MAAM,EACrB,gCAEE,EACG,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CACX,CAAE,SAAQ,YAAa,EAAG,YAAa,IAAK,CAC7C,CAEK,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC7E5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGI,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,YAAW,GAAG,MAAkB,CAC7C,GAAG,EACH,GAAI,EACA,CACE,YAAa,EAAU,GACvB,cAAe,EAAU,KACzB,aAAc,EAAU,UACzB,CACD,EAAE,CACN,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SAAS,CACR,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}
+{"version":3,"file":"pullChunkedCustomSiloOutstandingIdentifiers-CPY9eRLF.mjs","names":[],"sources":["../src/lib/graphql/fetchRequestDataSiloActiveCount.ts","../src/lib/cron/pullCronPageOfIdentifiers.ts","../src/lib/cron/pullChunkedCustomSiloOutstandingIdentifiers.ts"],"sourcesContent":["import { makeGraphQLRequest, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT } from '@transcend-io/sdk';\nimport { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/**\n * Get number of open requests for a data silo\n *\n * @param client - GraphQL client\n * @param options - Filter options\n * @returns List of request identifiers\n */\nexport async function fetchRequestDataSiloActiveCount(\n  client: GraphQLClient,\n  {\n    dataSiloId,\n  }: {\n    /** Data silo ID */\n    dataSiloId: string;\n  },\n): Promise<number> {\n  const {\n    listReducedRequestsForDataSilo: { totalCount },\n  } = await makeGraphQLRequest<{\n    /** Requests */\n    listReducedRequestsForDataSilo: {\n      /** Total count */\n      totalCount: number;\n    };\n  }>(client, REDUCED_REQUESTS_FOR_DATA_SILO_COUNT, {\n    variables: {\n      input: {\n        dataSiloId,\n        isResolved: false,\n      },\n    },\n    logger,\n  });\n\n  return totalCount;\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { withTransientRetry } from '@transcend-io/sdk';\nimport { decodeCodec } from '@transcend-io/type-utils';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\n\n/** Consent partition metadata when a DSR is scoped to a partition */\nexport const CronPartition = t.type({\n  /** ID of the partition */\n  id: t.string,\n  /** The human readable name of the partition */\n  name: t.string,\n  /** The unique identifying partition value */\n  partition: t.string,\n});\n\n/** Type override */\nexport type CronPartition = t.TypeOf<typeof CronPartition>;\n\nexport const CronIdentifier = t.intersection([\n  t.type({\n    /** The identifier value */\n    identifier: t.string,\n    /** The type of identifier */\n    type: t.string,\n    /** The core identifier of the request */\n    coreIdentifier: t.string,\n    /** The ID of the underlying data silo */\n    dataSiloId: t.string,\n    /** The ID of the underlying request */\n    requestId: t.string,\n    /** The request nonce */\n    nonce: t.string,\n    /** The time the request was created */\n    requestCreatedAt: t.string,\n    /** The number of days until the request is overdue */\n    daysUntilOverdue: t.number,\n    /** Request attributes */\n    attributes: t.array(\n      t.type({\n        key: t.string,\n        values: t.array(t.string),\n      }),\n    ),\n  }),\n  t.partial({\n    /** The consent partition that scopes this request, when applicable */\n    partition: t.union([CronPartition, t.null]),\n  }),\n]);\n\n/** Type override */\nexport type CronIdentifier = t.TypeOf<typeof CronIdentifier>;\n\n/**\n * Pull a offset of identifiers for a cron job\n *\n * @see https://docs.transcend.io/docs/api-reference/GET/v1/data-silo/(id)/pending-requests/(type)\n * @param sombra - Sombra instance configured to make requests\n * @param options - Additional options\n * @returns Successfully submitted request\n */\nexport async function pullCronPageOfIdentifiers(\n  sombra: Got,\n  {\n    dataSiloId,\n    limit = 100,\n    offset = 0,\n    requestType,\n  }: {\n    /** Data Silo ID */\n    dataSiloId: string;\n    /** Type of request */\n    requestType: RequestAction;\n    /** Number of identifiers to pull in */\n    limit?: number;\n    /** Page to pull in */\n    offset?: number;\n  },\n): Promise<CronIdentifier[]> {\n  try {\n    // `GET pending-requests` is a read and therefore safe to retry on transient\n    // gateway / network errors. Customers running `transcend cron\n    // pull-identifiers` against large silos would otherwise see the command\n    // abort on a single 502 from the Sombra reverse tunnel.\n    const response = await withTransientRetry(\n      'pullCronPageOfIdentifiers',\n      () =>\n        sombra\n          .get(`v1/data-silo/${dataSiloId}/pending-requests/${requestType}`, {\n            searchParams: {\n              offset,\n              limit,\n            },\n          })\n          .json(),\n      { logger, maxAttempts: 6, baseDelayMs: 500 },\n    );\n\n    const { items } = decodeCodec(\n      t.type({\n        items: t.array(CronIdentifier),\n      }),\n      response,\n    );\n    return items;\n  } catch (err) {\n    throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n  }\n}\n","import { RequestAction } from '@transcend-io/privacy-types';\nimport { buildTranscendGraphQLClient, createSombraGotInstance } from '@transcend-io/sdk';\nimport { mapSeries } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchRequestDataSiloActiveCount } from '../graphql/index.js';\nimport { pullCronPageOfIdentifiers, CronIdentifier } from './pullCronPageOfIdentifiers.js';\n\n/**\n * A CSV formatted identifier\n */\nexport type CsvFormattedIdentifier = {\n  [k in string]: string | null | boolean | number;\n};\n\nexport interface CronIdentifierWithAction extends CronIdentifier {\n  /** The request action that the identifier relates to */\n  action: RequestAction;\n}\n\n/**\n * Pull the set of identifiers outstanding for a cron or AVC integration\n *\n * This function is designed to be used in a loop, and will call the onSave callback\n * with a chunk of identifiers when the savePageSize is reached.\n *\n * @param options - Options\n * @returns The identifiers and identifiers formatted for CSV\n */\nexport async function pullChunkedCustomSiloOutstandingIdentifiers({\n  dataSiloId,\n  auth,\n  sombraAuth,\n  actions,\n  apiPageSize = 100,\n  savePageSize = 1000,\n  onSave,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  skipRequestCount = false,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Data Silo ID to pull down jobs for */\n  dataSiloId: string;\n  /** The request actions to fetch */\n  actions: RequestAction[];\n  /** How many identifiers to pull in a single call to the backend */\n  apiPageSize: number;\n  /** How many identifiers to save at a time (usually to a CSV file, should be a multiple of apiPageSize) */\n  savePageSize: number;\n  /** Callback function called when a chunk of identifiers is ready to be saved */\n  onSave: (chunk: CsvFormattedIdentifier[]) => Promise<void>;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Skip request count */\n  skipRequestCount?: boolean;\n}): Promise<{\n  /** Raw Identifiers */\n  identifiers: CronIdentifierWithAction[];\n}> {\n  // Validate savePageSize\n  if (savePageSize % apiPageSize !== 0) {\n    throw new Error(\n      `savePageSize must be a multiple of apiPageSize. savePageSize: ${savePageSize}, apiPageSize: ${apiPageSize}`,\n    );\n  }\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  // Create GraphQL client to connect to Transcend backend\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  let totalRequestCount = 0;\n  if (!skipRequestCount) {\n    totalRequestCount = await fetchRequestDataSiloActiveCount(client, {\n      dataSiloId,\n    });\n  }\n\n  logger.info(\n    colors.magenta(\n      `Pulling ${skipRequestCount ? 'all' : totalRequestCount} outstanding request identifiers ` +\n        `for data silo: \"${dataSiloId}\" for requests of types \"${actions.join('\", \"')}\"`,\n    ),\n  );\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  const foundRequestIds = new Set<string>();\n\n  // identifiers found in total\n  const identifiers: CronIdentifierWithAction[] = [];\n  // current chunk of identifiers to be saved\n  let currentChunk: CsvFormattedIdentifier[] = [];\n\n  // map over each action\n  if (!skipRequestCount) {\n    progressBar.start(totalRequestCount, 0);\n  }\n  await mapSeries(actions, async (action) => {\n    let offset = 0;\n    let shouldContinue = true;\n\n    // Fetch a page of identifiers\n    while (shouldContinue) {\n      const pageIdentifiers = await pullCronPageOfIdentifiers(sombra, {\n        dataSiloId,\n        limit: apiPageSize,\n        offset,\n        requestType: action,\n      });\n\n      const identifiersWithAction: CronIdentifierWithAction[] = pageIdentifiers.map(\n        (identifier) => {\n          foundRequestIds.add(identifier.requestId);\n          return {\n            ...identifier,\n            action,\n          };\n        },\n      );\n\n      const csvFormattedIdentifiers = identifiersWithAction.map(\n        ({ attributes, partition, ...identifier }) => ({\n          ...identifier,\n          ...(partition\n            ? {\n                partitionId: partition.id,\n                partitionName: partition.name,\n                partitionKey: partition.partition,\n              }\n            : {}),\n          ...attributes.reduce(\n            (acc, val) =>\n              Object.assign(acc, {\n                [val.key]: val.values.join(','),\n              }),\n            {},\n          ),\n        }),\n      );\n\n      identifiers.push(...identifiersWithAction);\n      currentChunk.push(...csvFormattedIdentifiers);\n\n      // Check if we've reached the savePageSize and call the onSave callback\n      if (currentChunk.length >= savePageSize) {\n        await onSave(currentChunk);\n        currentChunk = [];\n      }\n\n      shouldContinue = pageIdentifiers.length === apiPageSize;\n      offset += apiPageSize;\n      if (!skipRequestCount) {\n        progressBar.update(foundRequestIds.size);\n      } else {\n        logger.info(\n          colors.magenta(\n            `Pulled ${pageIdentifiers.length} outstanding identifiers for ${foundRequestIds.size} requests`,\n          ),\n        );\n      }\n    }\n  });\n\n  // Save any remaining identifiers in the current chunk\n  if (currentChunk.length > 0) {\n    await onSave(currentChunk);\n  }\n\n  if (!skipRequestCount) {\n    progressBar.stop();\n  }\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully pulled ${identifiers.length} outstanding identifiers from ${\n        foundRequestIds.size\n      } requests in \"${totalTime / 1000}\" seconds!`,\n    ),\n  );\n\n  return { identifiers };\n}\n"],"mappings":"0bAYA,eAAsB,EACpB,EACA,CACE,cAKe,CACjB,GAAM,CACJ,+BAAgC,CAAE,eAChC,MAAM,EAMP,EAAQ,EAAsC,CAC/C,UAAW,CACT,MAAO,CACL,aACA,WAAY,GACb,CACF,CACD,SACD,CAAC,CAEF,OAAO,EC9BT,MAAa,EAAgB,EAAE,KAAK,CAElC,GAAI,EAAE,OAEN,KAAM,EAAE,OAER,UAAW,EAAE,OACd,CAAC,CAKW,EAAiB,EAAE,aAAa,CAC3C,EAAE,KAAK,CAEL,WAAY,EAAE,OAEd,KAAM,EAAE,OAER,eAAgB,EAAE,OAElB,WAAY,EAAE,OAEd,UAAW,EAAE,OAEb,MAAO,EAAE,OAET,iBAAkB,EAAE,OAEpB,iBAAkB,EAAE,OAEpB,WAAY,EAAE,MACZ,EAAE,KAAK,CACL,IAAK,EAAE,OACP,OAAQ,EAAE,MAAM,EAAE,OAAO,CAC1B,CAAC,CACH,CACF,CAAC,CACF,EAAE,QAAQ,CAER,UAAW,EAAE,MAAM,CAAC,EAAe,EAAE,KAAK,CAAC,CAC5C,CAAC,CACH,CAAC,CAaF,eAAsB,EACpB,EACA,CACE,aACA,QAAQ,IACR,SAAS,EACT,eAWyB,CAC3B,GAAI,CAKF,IAAM,EAAW,MAAM,EACrB,gCAEE,EACG,IAAI,gBAAgB,EAAW,oBAAoB,IAAe,CACjE,aAAc,CACZ,SACA,QACD,CACF,CAAC,CACD,MAAM,CACX,CAAE,SAAQ,YAAa,EAAG,YAAa,IAAK,CAC7C,CAEK,CAAE,SAAU,EAChB,EAAE,KAAK,CACL,MAAO,EAAE,MAAM,EAAe,CAC/B,CAAC,CACF,EACD,CACD,OAAO,QACA,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EC7E5F,eAAsB,EAA4C,CAChE,aACA,OACA,aACA,UACA,cAAc,IACd,eAAe,IACf,SACA,eAAe,EACf,mBAAmB,IAuBlB,CAED,GAAI,EAAe,IAAgB,EACjC,MAAU,MACR,iEAAiE,EAAa,iBAAiB,IAChG,CAIH,IAAM,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGI,EAAS,EAA4B,EAAc,EAAK,CAE1D,EAAoB,EACnB,IACH,EAAoB,MAAM,EAAgC,EAAQ,CAChE,aACD,CAAC,EAGJ,EAAO,KACL,EAAO,QACL,WAAW,EAAmB,MAAQ,EAAkB,mDACnC,EAAW,2BAA2B,EAAQ,KAAK,OAAO,CAAC,GACjF,CACF,CAGD,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAC/E,EAAkB,IAAI,IAGtB,EAA0C,EAAE,CAE9C,EAAyC,EAAE,CAG1C,GACH,EAAY,MAAM,EAAmB,EAAE,CAEzC,MAAM,EAAU,EAAS,KAAO,IAAW,CACzC,IAAI,EAAS,EACT,EAAiB,GAGrB,KAAO,GAAgB,CACrB,IAAM,EAAkB,MAAM,EAA0B,EAAQ,CAC9D,aACA,MAAO,EACP,SACA,YAAa,EACd,CAAC,CAEI,EAAoD,EAAgB,IACvE,IACC,EAAgB,IAAI,EAAW,UAAU,CAClC,CACL,GAAG,EACH,SACD,EAEJ,CAEK,EAA0B,EAAsB,KACnD,CAAE,aAAY,YAAW,GAAG,MAAkB,CAC7C,GAAG,EACH,GAAI,EACA,CACE,YAAa,EAAU,GACvB,cAAe,EAAU,KACzB,aAAc,EAAU,UACzB,CACD,EAAE,CACN,GAAG,EAAW,QACX,EAAK,IACJ,OAAO,OAAO,EAAK,EAChB,EAAI,KAAM,EAAI,OAAO,KAAK,IAAI,CAChC,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAED,EAAY,KAAK,GAAG,EAAsB,CAC1C,EAAa,KAAK,GAAG,EAAwB,CAGzC,EAAa,QAAU,IACzB,MAAM,EAAO,EAAa,CAC1B,EAAe,EAAE,EAGnB,EAAiB,EAAgB,SAAW,EAC5C,GAAU,EACL,EAGH,EAAO,KACL,EAAO,QACL,UAAU,EAAgB,OAAO,+BAA+B,EAAgB,KAAK,WACtF,CACF,CAND,EAAY,OAAO,EAAgB,KAAK,GAS5C,CAGE,EAAa,OAAS,GACxB,MAAM,EAAO,EAAa,CAGvB,GACH,EAAY,MAAM,CAGpB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAUvB,OARA,EAAO,KACL,EAAO,MACL,uBAAuB,EAAY,OAAO,gCACxC,EAAgB,KACjB,gBAAgB,EAAY,IAAK,YACnC,CACF,CAEM,CAAE,cAAa"}

package/dist/pullConsentManagerMetrics-zKgjc3Ap.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"pullConsentManagerMetrics-zKgjc3Ap.mjs","names":[],"sources":["../src/lib/consent-manager/pullConsentManagerMetrics.ts"],"sourcesContent":["import { ConsentManagerMetricBin } from '@transcend-io/privacy-types';\nimport {\n  type ConsentManagerMetric,\n  fetchConsentManagerAnalyticsData,\n  fetchConsentManagerId,\n} from '@transcend-io/sdk';\nimport type { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/**\n * One second of time in ms\n */\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\nconst ONE_DAY = 24 * ONE_HOUR;\nconst ONE_WEEK = 7 * ONE_DAY;\n\n/**\n * Pull consent manager metrics in an organization\n *\n * @param client - GraphQL client\n * @param options - Options\n * @returns The consent manager metrics\n */\nexport async function pullConsentManagerMetrics(\n  client: GraphQLClient,\n  {\n    bin,\n    start,\n    end = new Date(),\n  }: {\n    /** Start date to pull metrics from */\n    start: Date;\n    /** End date to pull metrics from (assumes now) */\n    end?: Date;\n    /** Bin size to pull metrics */\n    bin: ConsentManagerMetricBin;\n  },\n): Promise<{\n  /** Privacy signal data */\n  PRIVACY_SIGNAL_TIMESERIES: ConsentManagerMetric[];\n  /** Consent changes data */\n  CONSENT_CHANGES_TIMESERIES: ConsentManagerMetric[];\n  /** Consent sessions by regime */\n  CONSENT_SESSIONS_BY_REGIME: ConsentManagerMetric[];\n}> {\n  // Grab the bundleId associated with this API key\n  const airgapBundleId = await fetchConsentManagerId(client, { logger });\n\n  // convert start and end to times\n  const startTime = Math.floor(start.getTime() / 1000);\n  const endTime = Math.floor(end.getTime() / 1000);\n  if (startTime > endTime) {\n    throw new Error('Received \"end\" date that happened before \"start\" date');\n  }\n\n  // do not allow hourly bins greater than 2 weeks\n  if (bin === '1h' && end.getTime() - start.getTime() > ONE_WEEK * 2) {\n    throw new Error('When using bin=1h, start and end time can be no more than 2 weeks apart');\n  }\n\n  // Pull in the metrics\n  const startDate = start.toISOString();\n  const endDate = end.toISOString();\n  const [privacySignalData, consentChangesData, consentSessionsByRegimeData] = await Promise.all([\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'PRIVACY_SIGNAL_TIMESERIES',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'CONSENT_CHANGES_TIMESERIES',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'CONSENT_SESSIONS_BY_REGIME',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n  ]);\n\n  return {\n    PRIVACY_SIGNAL_TIMESERIES: privacySignalData,\n    CONSENT_CHANGES_TIMESERIES: consentChangesData,\n    CONSENT_SESSIONS_BY_REGIME: consentSessionsByRegimeData,\n  };\n}\n"],"mappings":"0IA0BA,eAAsB,EACpB,EACA,CACE,MACA,QACA,MAAM,IAAI,MAgBX,CAED,IAAM,EAAiB,MAAM,EAAsB,EAAQ,CAAE,SAAQ,CAAC,CAKtE,GAFkB,KAAK,MAAM,EAAM,SAAS,CAAG,IAAK,CACpC,KAAK,MAAM,EAAI,SAAS,CAAG,IAAK,CAE9C,MAAU,MAAM,wDAAwD,CAI1E,GAAI,IAAQ,MAAQ,EAAI,SAAS,CAAG,EAAM,SAAS,CAAG,OAAW,EAC/D,MAAU,MAAM,0EAA0E,CAI5F,IAAM,EAAY,EAAM,aAAa,CAC/B,EAAU,EAAI,aAAa,CAC3B,CAAC,EAAmB,EAAoB,GAA+B,MAAM,QAAQ,IAAI,CAC7F,EACE,EACA,CACE,WAAY,4BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACD,EACE,EACA,CACE,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACD,EACE,EACA,CACE,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACF,CAAC,CAEF,MAAO,CACL,0BAA2B,EAC3B,2BAA4B,EAC5B,2BAA4B,EAC7B"}
+{"version":3,"file":"pullConsentManagerMetrics-zKgjc3Ap.mjs","names":[],"sources":["../src/lib/consent-manager/pullConsentManagerMetrics.ts"],"sourcesContent":["import { ConsentManagerMetricBin } from '@transcend-io/privacy-types';\nimport {\n  type ConsentManagerMetric,\n  fetchConsentManagerAnalyticsData,\n  fetchConsentManagerId,\n} from '@transcend-io/sdk';\nimport type { GraphQLClient } from 'graphql-request';\n\nimport { logger } from '../../logger.js';\n\n/**\n * One second of time in ms\n */\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\nconst ONE_DAY = 24 * ONE_HOUR;\nconst ONE_WEEK = 7 * ONE_DAY;\n\n/**\n * Pull consent manager metrics in an organization\n *\n * @param client - GraphQL client\n * @param options - Options\n * @returns The consent manager metrics\n */\nexport async function pullConsentManagerMetrics(\n  client: GraphQLClient,\n  {\n    bin,\n    start,\n    end = new Date(),\n  }: {\n    /** Start date to pull metrics from */\n    start: Date;\n    /** End date to pull metrics from (assumes now) */\n    end?: Date;\n    /** Bin size to pull metrics */\n    bin: ConsentManagerMetricBin;\n  },\n): Promise<{\n  /** Privacy signal data */\n  PRIVACY_SIGNAL_TIMESERIES: ConsentManagerMetric[];\n  /** Consent changes data */\n  CONSENT_CHANGES_TIMESERIES: ConsentManagerMetric[];\n  /** Consent sessions by regime */\n  CONSENT_SESSIONS_BY_REGIME: ConsentManagerMetric[];\n}> {\n  // Grab the bundleId associated with this API key\n  const airgapBundleId = await fetchConsentManagerId(client, { logger });\n\n  // convert start and end to times\n  const startTime = Math.floor(start.getTime() / 1000);\n  const endTime = Math.floor(end.getTime() / 1000);\n  if (startTime > endTime) {\n    throw new Error('Received \"end\" date that happened before \"start\" date');\n  }\n\n  // do not allow hourly bins greater than 2 weeks\n  if (bin === '1h' && end.getTime() - start.getTime() > ONE_WEEK * 2) {\n    throw new Error('When using bin=1h, start and end time can be no more than 2 weeks apart');\n  }\n\n  // Pull in the metrics\n  const startDate = start.toISOString();\n  const endDate = end.toISOString();\n  const [privacySignalData, consentChangesData, consentSessionsByRegimeData] = await Promise.all([\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'PRIVACY_SIGNAL_TIMESERIES',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'CONSENT_CHANGES_TIMESERIES',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n    fetchConsentManagerAnalyticsData(\n      client,\n      {\n        dataSource: 'CONSENT_SESSIONS_BY_REGIME',\n        startDate,\n        endDate,\n        forceRefetch: true,\n        airgapBundleId,\n        binInterval: bin,\n        smoothTimeseries: false,\n      },\n      { logger },\n    ),\n  ]);\n\n  return {\n    PRIVACY_SIGNAL_TIMESERIES: privacySignalData,\n    CONSENT_CHANGES_TIMESERIES: consentChangesData,\n    CONSENT_SESSIONS_BY_REGIME: consentSessionsByRegimeData,\n  };\n}\n"],"mappings":"0IA0BA,eAAsB,EACpB,EACA,CACE,MACA,QACA,MAAM,IAAI,MAgBX,CAED,IAAM,EAAiB,MAAM,EAAsB,EAAQ,CAAE,SAAQ,CAAC,CAKtE,GAFkB,KAAK,MAAM,EAAM,SAAS,CAAG,IAElC,CADG,KAAK,MAAM,EAAI,SAAS,CAAG,IACpB,CACrB,MAAU,MAAM,wDAAwD,CAI1E,GAAI,IAAQ,MAAQ,EAAI,SAAS,CAAG,EAAM,SAAS,CAAG,OAAW,EAC/D,MAAU,MAAM,0EAA0E,CAI5F,IAAM,EAAY,EAAM,aAAa,CAC/B,EAAU,EAAI,aAAa,CAC3B,CAAC,EAAmB,EAAoB,GAA+B,MAAM,QAAQ,IAAI,CAC7F,EACE,EACA,CACE,WAAY,4BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACD,EACE,EACA,CACE,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACD,EACE,EACA,CACE,WAAY,6BACZ,YACA,UACA,aAAc,GACd,iBACA,YAAa,EACb,iBAAkB,GACnB,CACD,CAAE,SAAQ,CACX,CACF,CAAC,CAEF,MAAO,CACL,0BAA2B,EAC3B,2BAA4B,EAC5B,2BAA4B,EAC7B"}

package/dist/{pullManualEnrichmentIdentifiersToCsv-CuMrsy2E.mjs → pullManualEnrichmentIdentifiersToCsv-CDCL8Ju4.mjs}

@@ -1,2 +1,2 @@
-import{a as e}from"./constants-BOycJ_AQ.mjs";import{t}from"./logger-Bj782ZYD.mjs";import{r as n}from"./fetchAllRequests-CHHdyb4Q.mjs";import{a as r}from"./writeCsv-C4pjXGsD.mjs";import{RequestStatus as i}from"@transcend-io/privacy-types";import{groupBy as a,uniq as o}from"lodash-es";import s from"colors";import{buildTranscendGraphQLClient as c,createSombraGotInstance as l,fetchAllRequestEnrichers as u,fetchAllRequestIdentifiers as d,validateSombraVersion as f}from"@transcend-io/sdk";import{map as p}from"@transcend-io/utils";async function m({file:m,auth:h,sombraAuth:g,requestActions:_=[],concurrency:v=100,transcendUrl:y=e}){let b=c(y,h),x=await l(y,h,{logger:t,sombraApiKey:g,sombraUrl:process.env.SOMBRA_URL});t.info(s.magenta(`Pulling manual enrichment requests, filtered for actions: ${_.join(`,`)}`));let S=await n(b,{actions:_,statuses:[i.Enriching]});await f(b,{logger:t});let C=[];await p(S,async e=>{let n=await u(b,{filterBy:{requestId:e.id},logger:t});if(n.filter(({status:e})=>e===`ACTION_REQUIRED`)){let r=await d(b,x,{filterBy:{requestId:e.id},skipSombraCheck:!0,logger:t});C.push({...e,requestIdentifiers:r,requestEnrichers:n})}},{concurrency:v});let w=C.map(({attributeValues:e,requestIdentifiers:t,requestEnrichers:n,...r})=>({...r,...Object.entries(a(t,`name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({value:e})=>e).join(`,`)}),{}),...Object.entries(a(e,`attributeKey.name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({name:e})=>e).join(`,`)}),{})}));return await r(m,w,o(w.map(e=>Object.keys(e)).flat())),t.info(s.green(`Successfully wrote ${C.length} requests to file "${m}"`)),C}export{m as t};
-//# sourceMappingURL=pullManualEnrichmentIdentifiersToCsv-CuMrsy2E.mjs.map
+import{a as e}from"./constants-CeIAxxnY.mjs";import{t}from"./logger-Bj782ZYD.mjs";import{r as n}from"./fetchAllRequests-CHHdyb4Q.mjs";import{a as r}from"./writeCsv-C4pjXGsD.mjs";import{RequestStatus as i}from"@transcend-io/privacy-types";import{groupBy as a,uniq as o}from"lodash-es";import s from"colors";import{buildTranscendGraphQLClient as c,createSombraGotInstance as l,fetchAllRequestEnrichers as u,fetchAllRequestIdentifiers as d,validateSombraVersion as f}from"@transcend-io/sdk";import{map as p}from"@transcend-io/utils";async function m({file:m,auth:h,sombraAuth:g,requestActions:_=[],concurrency:v=100,transcendUrl:y=e}){let b=c(y,h),x=await l(y,h,{logger:t,sombraApiKey:g,sombraUrl:process.env.SOMBRA_URL});t.info(s.magenta(`Pulling manual enrichment requests, filtered for actions: ${_.join(`,`)}`));let S=await n(b,{actions:_,statuses:[i.Enriching]});await f(b,{logger:t});let C=[];await p(S,async e=>{let n=await u(b,{filterBy:{requestId:e.id},logger:t});if(n.filter(({status:e})=>e===`ACTION_REQUIRED`)){let r=await d(b,x,{filterBy:{requestId:e.id},skipSombraCheck:!0,logger:t});C.push({...e,requestIdentifiers:r,requestEnrichers:n})}},{concurrency:v});let w=C.map(({attributeValues:e,requestIdentifiers:t,requestEnrichers:n,...r})=>({...r,...Object.entries(a(t,`name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({value:e})=>e).join(`,`)}),{}),...Object.entries(a(e,`attributeKey.name`)).reduce((e,[t,n])=>Object.assign(e,{[t]:n.map(({name:e})=>e).join(`,`)}),{})}));return await r(m,w,o(w.map(e=>Object.keys(e)).flat())),t.info(s.green(`Successfully wrote ${C.length} requests to file "${m}"`)),C}export{m as t};
+//# sourceMappingURL=pullManualEnrichmentIdentifiersToCsv-CDCL8Ju4.mjs.map

package/dist/{pullManualEnrichmentIdentifiersToCsv-CuMrsy2E.mjs.map → pullManualEnrichmentIdentifiersToCsv-CDCL8Ju4.mjs.map}

@@ -1 +1 @@
-{"version":3,"file":"pullManualEnrichmentIdentifiersToCsv-CuMrsy2E.mjs","names":[],"sources":["../src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts"],"sourcesContent":["import { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllRequestEnrichers,\n  fetchAllRequestIdentifiers,\n  validateSombraVersion,\n  type RequestEnricher,\n  type RequestIdentifier,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { groupBy, uniq } from 'lodash-es';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { type PrivacyRequest, fetchAllRequests } from '../graphql/index.js';\nimport { writeCsv } from '../helpers/writeCsv.js';\n\nexport interface PrivacyRequestWithIdentifiers extends PrivacyRequest {\n  /** Request Enrichers */\n  requestEnrichers: RequestEnricher[];\n  /** Request Identifiers */\n  requestIdentifiers: RequestIdentifier[];\n}\n\n/**\n * Pull the set of manual enrichment jobs to CSV\n *\n * @param options - Options\n * @returns List of requests with identifiers\n */\nexport async function pullManualEnrichmentIdentifiersToCsv({\n  file,\n  auth,\n  sombraAuth,\n  requestActions = [],\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** CSV file path */\n  file: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Sombra API key */\n  sombraAuth?: string;\n  /** Concurrency */\n  concurrency?: number;\n  /** The request actions to fetch */\n  requestActions?: RequestAction[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<PrivacyRequestWithIdentifiers[]> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  logger.info(\n    colors.magenta(\n      `Pulling manual enrichment requests, filtered for actions: ${requestActions.join(',')}`,\n    ),\n  );\n\n  // Pull all privacy requests\n  const allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    statuses: [RequestStatus.Enriching],\n  });\n\n  await validateSombraVersion(client, { logger });\n\n  // Requests to save\n  const savedRequests: PrivacyRequestWithIdentifiers[] = [];\n\n  // Filter down requests to what is needed\n  await map(\n    allRequests,\n    async (request) => {\n      // Fetch enrichers\n      const requestEnrichers = await fetchAllRequestEnrichers(client, {\n        filterBy: { requestId: request.id },\n        logger,\n      });\n\n      // Check if manual enrichment exists for that request\n      const hasManualEnrichment = requestEnrichers.filter(\n        ({ status }) => status === 'ACTION_REQUIRED',\n      );\n\n      // Save request to queue\n      if (hasManualEnrichment) {\n        const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, {\n          filterBy: { requestId: request.id },\n          skipSombraCheck: true,\n          logger,\n        });\n        savedRequests.push({\n          ...request,\n          requestIdentifiers,\n          requestEnrichers,\n        });\n      }\n    },\n    {\n      concurrency,\n    },\n  );\n\n  const data = savedRequests.map(\n    ({\n      attributeValues,\n      requestIdentifiers,\n      requestEnrichers, // eslint-disable-line @typescript-eslint/no-unused-vars\n      ...request\n    }) => ({\n      ...request,\n      // flatten identifiers\n      ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ value }) => value).join(','),\n          }),\n        {},\n      ),\n      // flatten attributes\n      ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ name }) => name).join(','),\n          }),\n        {},\n      ),\n    }),\n  );\n\n  // Write out to CSV\n  const headers = uniq(data.map((d) => Object.keys(d)).flat());\n  await writeCsv(file, data, headers);\n\n  logger.info(\n    colors.green(`Successfully wrote ${savedRequests.length} requests to file \"${file}\"`),\n  );\n\n  return savedRequests;\n}\n"],"mappings":"khBAgCA,eAAsB,EAAqC,CACzD,OACA,OACA,aACA,iBAAiB,EAAE,CACnB,cAAc,IACd,eAAe,GAc4B,CAE3C,IAAM,EAAS,EAA4B,EAAc,EAAK,CACxD,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAEF,EAAO,KACL,EAAO,QACL,6DAA6D,EAAe,KAAK,IAAI,GACtF,CACF,CAGD,IAAM,EAAc,MAAM,EAAiB,EAAQ,CACjD,QAAS,EACT,SAAU,CAAC,EAAc,UAAU,CACpC,CAAC,CAEF,MAAM,EAAsB,EAAQ,CAAE,SAAQ,CAAC,CAG/C,IAAM,EAAiD,EAAE,CAGzD,MAAM,EACJ,EACA,KAAO,IAAY,CAEjB,IAAM,EAAmB,MAAM,EAAyB,EAAQ,CAC9D,SAAU,CAAE,UAAW,EAAQ,GAAI,CACnC,SACD,CAAC,CAQF,GAL4B,EAAiB,QAC1C,CAAE,YAAa,IAAW,kBAC5B,CAGwB,CACvB,IAAM,EAAqB,MAAM,EAA2B,EAAQ,EAAQ,CAC1E,SAAU,CAAE,UAAW,EAAQ,GAAI,CACnC,gBAAiB,GACjB,SACD,CAAC,CACF,EAAc,KAAK,CACjB,GAAG,EACH,qBACA,mBACD,CAAC,GAGN,CACE,cACD,CACF,CAED,IAAM,EAAO,EAAc,KACxB,CACC,kBACA,qBACA,mBACA,GAAG,MACE,CACL,GAAG,EAEH,GAAG,OAAO,QAAQ,EAAQ,EAAoB,OAAO,CAAC,CAAC,QACpD,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,WAAY,EAAM,CAAC,KAAK,IAAI,CAClD,CAAC,CACJ,EAAE,CACH,CAED,GAAG,OAAO,QAAQ,EAAQ,EAAiB,oBAAoB,CAAC,CAAC,QAC9D,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,UAAW,EAAK,CAAC,KAAK,IAAI,CAChD,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAUD,OANA,MAAM,EAAS,EAAM,EADL,EAAK,EAAK,IAAK,GAAM,OAAO,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CACzB,CAEnC,EAAO,KACL,EAAO,MAAM,sBAAsB,EAAc,OAAO,qBAAqB,EAAK,GAAG,CACtF,CAEM"}
+{"version":3,"file":"pullManualEnrichmentIdentifiersToCsv-CDCL8Ju4.mjs","names":[],"sources":["../src/lib/manual-enrichment/pullManualEnrichmentIdentifiersToCsv.ts"],"sourcesContent":["import { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllRequestEnrichers,\n  fetchAllRequestIdentifiers,\n  validateSombraVersion,\n  type RequestEnricher,\n  type RequestIdentifier,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport { groupBy, uniq } from 'lodash-es';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { type PrivacyRequest, fetchAllRequests } from '../graphql/index.js';\nimport { writeCsv } from '../helpers/writeCsv.js';\n\nexport interface PrivacyRequestWithIdentifiers extends PrivacyRequest {\n  /** Request Enrichers */\n  requestEnrichers: RequestEnricher[];\n  /** Request Identifiers */\n  requestIdentifiers: RequestIdentifier[];\n}\n\n/**\n * Pull the set of manual enrichment jobs to CSV\n *\n * @param options - Options\n * @returns List of requests with identifiers\n */\nexport async function pullManualEnrichmentIdentifiersToCsv({\n  file,\n  auth,\n  sombraAuth,\n  requestActions = [],\n  concurrency = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n}: {\n  /** CSV file path */\n  file: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Sombra API key */\n  sombraAuth?: string;\n  /** Concurrency */\n  concurrency?: number;\n  /** The request actions to fetch */\n  requestActions?: RequestAction[];\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n}): Promise<PrivacyRequestWithIdentifiers[]> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  logger.info(\n    colors.magenta(\n      `Pulling manual enrichment requests, filtered for actions: ${requestActions.join(',')}`,\n    ),\n  );\n\n  // Pull all privacy requests\n  const allRequests = await fetchAllRequests(client, {\n    actions: requestActions,\n    statuses: [RequestStatus.Enriching],\n  });\n\n  await validateSombraVersion(client, { logger });\n\n  // Requests to save\n  const savedRequests: PrivacyRequestWithIdentifiers[] = [];\n\n  // Filter down requests to what is needed\n  await map(\n    allRequests,\n    async (request) => {\n      // Fetch enrichers\n      const requestEnrichers = await fetchAllRequestEnrichers(client, {\n        filterBy: { requestId: request.id },\n        logger,\n      });\n\n      // Check if manual enrichment exists for that request\n      const hasManualEnrichment = requestEnrichers.filter(\n        ({ status }) => status === 'ACTION_REQUIRED',\n      );\n\n      // Save request to queue\n      if (hasManualEnrichment) {\n        const requestIdentifiers = await fetchAllRequestIdentifiers(client, sombra, {\n          filterBy: { requestId: request.id },\n          skipSombraCheck: true,\n          logger,\n        });\n        savedRequests.push({\n          ...request,\n          requestIdentifiers,\n          requestEnrichers,\n        });\n      }\n    },\n    {\n      concurrency,\n    },\n  );\n\n  const data = savedRequests.map(\n    ({\n      attributeValues,\n      requestIdentifiers,\n      requestEnrichers, // eslint-disable-line @typescript-eslint/no-unused-vars\n      ...request\n    }) => ({\n      ...request,\n      // flatten identifiers\n      ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ value }) => value).join(','),\n          }),\n        {},\n      ),\n      // flatten attributes\n      ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce(\n        (acc, [key, values]) =>\n          Object.assign(acc, {\n            [key]: values.map(({ name }) => name).join(','),\n          }),\n        {},\n      ),\n    }),\n  );\n\n  // Write out to CSV\n  const headers = uniq(data.map((d) => Object.keys(d)).flat());\n  await writeCsv(file, data, headers);\n\n  logger.info(\n    colors.green(`Successfully wrote ${savedRequests.length} requests to file \"${file}\"`),\n  );\n\n  return savedRequests;\n}\n"],"mappings":"khBAgCA,eAAsB,EAAqC,CACzD,OACA,OACA,aACA,iBAAiB,EAAE,CACnB,cAAc,IACd,eAAe,GAc4B,CAE3C,IAAM,EAAS,EAA4B,EAAc,EAAK,CACxD,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAEF,EAAO,KACL,EAAO,QACL,6DAA6D,EAAe,KAAK,IAAI,GACtF,CACF,CAGD,IAAM,EAAc,MAAM,EAAiB,EAAQ,CACjD,QAAS,EACT,SAAU,CAAC,EAAc,UAAU,CACpC,CAAC,CAEF,MAAM,EAAsB,EAAQ,CAAE,SAAQ,CAAC,CAG/C,IAAM,EAAiD,EAAE,CAGzD,MAAM,EACJ,EACA,KAAO,IAAY,CAEjB,IAAM,EAAmB,MAAM,EAAyB,EAAQ,CAC9D,SAAU,CAAE,UAAW,EAAQ,GAAI,CACnC,SACD,CAAC,CAQF,GAL4B,EAAiB,QAC1C,CAAE,YAAa,IAAW,kBAIN,CAAE,CACvB,IAAM,EAAqB,MAAM,EAA2B,EAAQ,EAAQ,CAC1E,SAAU,CAAE,UAAW,EAAQ,GAAI,CACnC,gBAAiB,GACjB,SACD,CAAC,CACF,EAAc,KAAK,CACjB,GAAG,EACH,qBACA,mBACD,CAAC,GAGN,CACE,cACD,CACF,CAED,IAAM,EAAO,EAAc,KACxB,CACC,kBACA,qBACA,mBACA,GAAG,MACE,CACL,GAAG,EAEH,GAAG,OAAO,QAAQ,EAAQ,EAAoB,OAAO,CAAC,CAAC,QACpD,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,WAAY,EAAM,CAAC,KAAK,IAAI,CAClD,CAAC,CACJ,EAAE,CACH,CAED,GAAG,OAAO,QAAQ,EAAQ,EAAiB,oBAAoB,CAAC,CAAC,QAC9D,EAAK,CAAC,EAAK,KACV,OAAO,OAAO,EAAK,EAChB,GAAM,EAAO,KAAK,CAAE,UAAW,EAAK,CAAC,KAAK,IAAI,CAChD,CAAC,CACJ,EAAE,CACH,CACF,EACF,CAUD,OANA,MAAM,EAAS,EAAM,EADL,EAAK,EAAK,IAAK,GAAM,OAAO,KAAK,EAAE,CAAC,CAAC,MAAM,CACzB,CAAC,CAEnC,EAAO,KACL,EAAO,MAAM,sBAAsB,EAAc,OAAO,qBAAqB,EAAK,GAAG,CACtF,CAEM"}