@transcend-io/cli 4.74.0 → 4.75.0

package/README.md

@@ -77,26 +77,34 @@
     - [Authentication](#authentication-16)
     - [Arguments](#arguments-16)
     - [Usage](#usage-17)
-  - [tr-pull-consent-metrics](#tr-pull-consent-metrics)
+  - [tr-derive-data-silos-from-data-flows](#tr-derive-data-silos-from-data-flows)
     - [Authentication](#authentication-17)
     - [Arguments](#arguments-17)
     - [Usage](#usage-18)
-  - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv)
+  - [tr-derive-data-silos-from-data-flows-cross-instance](#tr-derive-data-silos-from-data-flows-cross-instance)
     - [Authentication](#authentication-18)
     - [Arguments](#arguments-18)
     - [Usage](#usage-19)
-  - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv)
+  - [tr-pull-consent-metrics](#tr-pull-consent-metrics)
     - [Authentication](#authentication-19)
     - [Arguments](#arguments-19)
     - [Usage](#usage-20)
-  - [tr-generate-api-keys](#tr-generate-api-keys)
+  - [tr-upload-data-flows-from-csv](#tr-upload-data-flows-from-csv)
     - [Authentication](#authentication-20)
     - [Arguments](#arguments-20)
     - [Usage](#usage-21)
-  - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint)
+  - [tr-upload-cookies-from-csv](#tr-upload-cookies-from-csv)
     - [Authentication](#authentication-21)
     - [Arguments](#arguments-21)
     - [Usage](#usage-22)
+  - [tr-generate-api-keys](#tr-generate-api-keys)
+    - [Authentication](#authentication-22)
+    - [Arguments](#arguments-22)
+    - [Usage](#usage-23)
+  - [tr-build-xdi-sync-endpoint](#tr-build-xdi-sync-endpoint)
+    - [Authentication](#authentication-23)
+    - [Arguments](#arguments-23)
+    - [Usage](#usage-24)
 - [Proxy usage](#proxy-usage)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -131,7 +139,9 @@ yarn tr-mark-request-data-silos-completed --auth=$TRANSCEND_API_KEY
 yarn tr-skip-request-data-silos --auth=$TRANSCEND_API_KEY
 yarn tr-retry-request-data-silos --auth=$TRANSCEND_API_KEY
 yarn tr-update-consent-manager --auth=$TRANSCEND_API_KEY
-yarn tr-consent-managers-to-business-entities --auth=$TRANSCEND_API_KEY
+yarn tr-consent-managers-to-business-entities
+yarn tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY
+yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY
 yarn tr-pull-consent-metrics --auth=$TRANSCEND_API_KEY
 yarn tr-upload-data-flows-from-csv --auth=$TRANSCEND_API_KEY
 yarn tr-upload-cookies-from-csv --auth=$TRANSCEND_API_KEY
@@ -162,7 +172,9 @@ tr-skip-request-data-silos --auth=$TRANSCEND_API_KEY
 tr-retry-request-data-silos --auth=$TRANSCEND_API_KEY
 tr-update-consent-manager --auth=$TRANSCEND_API_KEY
 tr-pull-consent-metrics --auth=$TRANSCEND_API_KEY
-tr-consent-managers-to-business-entities --auth=$TRANSCEND_API_KEY
+tr-consent-managers-to-business-entities
+tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY
+tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY
 tr-upload-data-flows-from-csv --auth=$TRANSCEND_API_KEY
 tr-generate-api-keys --auth=$TRANSCEND_API_KEY
 tr-build-xdi-sync-endpoint --auth=$TRANSCEND_API_KEY
@@ -1497,6 +1509,99 @@ Specify custom output file
 yarn tr-consent-managers-to-business-entities --consentManagerYmlFolder=./working/consent-managers/ --output=./custom.yml
 ```
 
+### tr-derive-data-silos-from-data-flows
+
+Given a folder of data flow `transcend.yml` configurations, convert those configurations to set of data silo `transcend.yml` configurations.
+
+#### Authentication
+
+In order to use this cli, you will first need to generate an API key on the Transcend Admin Dashboard (https://app.transcend.io/infrastructure/api-keys).
+
+The API does not need any scopes, any API key will work.
+
+#### Arguments
+
+| Argument           | Description                                                                   | Type                 | Default                  | Required |
+| ------------------ | ----------------------------------------------------------------------------- | -------------------- | ------------------------ | -------- |
+| auth               | The Transcend API key with the scopes necessary for the command.              | string               | N/A                      | true     |
+| dataFlowsYmlFolder | The folder that contains data flow yml files                                  | string - folder-path | N/A                      | true     |
+| dataSilosYmlFolder | The folder that contains data silo yml files                                  | string - folder-path | N/A                      | true     |
+| ignoreYmls         | The set of yml files that should be skipped when uploading                    | string[]             | []                       | false    |
+| transcendUrl       | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting. | string - URL         | https://api.transcend.io | false    |
+
+#### Usage
+
+Convert data flow configurations in folder `./working/data-flows/` to data silo configurations in folder `./working/data-silos/`
+
+```sh
+yarn tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY \
+ --dataFlowsYmlFolder=./working/data-flows/ \
+ --dataSilosYmlFolder=./working/data-silos/
+```
+
+Use with US backend
+
+```sh
+yarn tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY --transcendUrl=https://api.us.transcend.io \
+ --dataFlowsYmlFolder=./working/data-flows/ \
+ --dataSilosYmlFolder=./working/data-silos/
+```
+
+Skip a set of yml files
+
+```sh
+yarn tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY --ignoreYmls="Skip.yml,Other.yml" \
+ --dataFlowsYmlFolder=./working/data-flows/ \
+ --dataSilosYmlFolder=./working/data-silos/
+```
+
+### tr-derive-data-silos-from-data-flows-cross-instance
+
+Given a folder of data flow `transcend.yml` configurations, convert those configurations to a single `transcend.yml` configurations of all related data silos.
+
+#### Authentication
+
+In order to use this cli, you will first need to generate an API key on the Transcend Admin Dashboard (https://app.transcend.io/infrastructure/api-keys).
+
+The API does not need any scopes, any API key will work.
+
+#### Arguments
+
+| Argument           | Description                                                                   | Type                 | Default                  | Required |
+| ------------------ | ----------------------------------------------------------------------------- | -------------------- | ------------------------ | -------- |
+| auth               | The Transcend API key with the scopes necessary for the command.              | string               | N/A                      | true     |
+| dataFlowsYmlFolder | The folder that contains data flow yml files                                  | string - folder-path | N/A                      | true     |
+| output             | The output transcend.yml file containing the data silo configurations         | string - file-path   | ./transcend.yml          | false    |
+| ignoreYmls         | The set of yml files that should be skipped when uploading                    | string[]             | []                       | false    |
+| transcendUrl       | URL of the Transcend backend. Use https://api.us.transcend.io for US hosting. | string - URL         | https://api.transcend.io | false    |
+
+#### Usage
+
+Convert data flow configurations in folder `./working/data-flows/` to data silo configurations in file `./transcend.yml`
+
+```sh
+yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY --dataFlowsYmlFolder=./working/data-flows/
+```
+
+Use with US backend
+
+```sh
+yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY --dataFlowsYmlFolder=./working/data-flows/ \
+  --transcendUrl=https://api.us.transcend.io
+```
+
+Skip a set of yml files
+
+```sh
+yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY --dataFlowsYmlFolder=./working/data-flows/ --ignoreYmls="Skip.yml,Other.yml"
+```
+
+Convert data flow configurations in folder `./working/data-flows/` to data silo configurations in file `./output.yml`
+
+```sh
+yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY --dataFlowsYmlFolder=./working/data-flows/ --output=./output.yml
+```
+
 ### tr-pull-consent-metrics
 
 This command allows for pulling consent manager metrics for a Transcend account, or a set of Transcend accounts.

package/build/cli-consent-managers-to-business-entities.js

@@ -33,7 +33,7 @@ function main() {
         logger_1.logger.error(colors_1.default.red('Missing required arg: --consentManagerYmlFolder=./working/consent-managers/'));
         process.exit(1);
     }
-    // Ensure file is passed
+    // Ensure folder is passed
     if (!(0, fs_1.existsSync)(consentManagerYmlFolder) ||
         !(0, fs_1.lstatSync)(consentManagerYmlFolder).isDirectory()) {
         logger_1.logger.error(colors_1.default.red(`Folder does not exist: "${consentManagerYmlFolder}"`));

package/build/cli-consent-managers-to-business-entities.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-consent-managers-to-business-entities.js","sourceRoot":"","sources":["../src/cli-consent-managers-to-business-entities.ts"],"names":[],"mappings":";;;;;;AACA,yCAAuC;AACvC,uDAAsE;AACtE,2DAA4E;AAC5E,+BAA4B;AAC5B,gEAAiC;AACjC,oDAA4B;AAC5B,qCAAkC;AAClC,2BAA2C;AAE3C;;;;;;;;;;;;GAYG;AACH,SAAS,IAAI;IACX,+BAA+B;IAC/B,MAAM,EACJ,uBAAuB,EACvB,MAAM,GAAG,kCAAkC,GAC5C,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,qDAAqD;IACrD,IAAI,CAAC,uBAAuB,EAAE;QAC5B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6EAA6E,CAC9E,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,wBAAwB;IACxB,IACE,CAAC,IAAA,eAAU,EAAC,uBAAuB,CAAC;QACpC,CAAC,IAAA,cAAS,EAAC,uBAAuB,CAAC,CAAC,WAAW,EAAE,EACjD;QACA,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CAAC,2BAA2B,uBAAuB,GAAG,CAAC,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAA,oBAAS,EAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAClE,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,IAAA,qCAAiB,EAC7D,IAAA,WAAI,EAAC,uBAAuB,EAAE,SAAS,CAAC,CACzC,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,+BAA+B;IAC/B,MAAM,gBAAgB,GAAG,IAAA,mDAAiC,EAAC,MAAM,CAAC,CAAC;IAEnE,gBAAgB;IAChB,IAAA,sCAAkB,EAAC,MAAM,EAAE;QACzB,mBAAmB,EAAE,gBAAgB;KACtC,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,gBAAgB,CAAC,MAAM,+BAA+B,MAAM,GAAG,CACtF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}
+{"version":3,"file":"cli-consent-managers-to-business-entities.js","sourceRoot":"","sources":["../src/cli-consent-managers-to-business-entities.ts"],"names":[],"mappings":";;;;;;AACA,yCAAuC;AACvC,uDAAsE;AACtE,2DAA4E;AAC5E,+BAA4B;AAC5B,gEAAiC;AACjC,oDAA4B;AAC5B,qCAAkC;AAClC,2BAA2C;AAE3C;;;;;;;;;;;;GAYG;AACH,SAAS,IAAI;IACX,+BAA+B;IAC/B,MAAM,EACJ,uBAAuB,EACvB,MAAM,GAAG,kCAAkC,GAC5C,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,qDAAqD;IACrD,IAAI,CAAC,uBAAuB,EAAE;QAC5B,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,6EAA6E,CAC9E,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IACE,CAAC,IAAA,eAAU,EAAC,uBAAuB,CAAC;QACpC,CAAC,IAAA,cAAS,EAAC,uBAAuB,CAAC,CAAC,WAAW,EAAE,EACjD;QACA,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CAAC,2BAA2B,uBAAuB,GAAG,CAAC,CAClE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,6CAA6C;IAC7C,MAAM,MAAM,GAAG,IAAA,oBAAS,EAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAClE,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,IAAA,qCAAiB,EAC7D,IAAA,WAAI,EAAC,uBAAuB,EAAE,SAAS,CAAC,CACzC,CAAC;QACF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,+BAA+B;IAC/B,MAAM,gBAAgB,GAAG,IAAA,mDAAiC,EAAC,MAAM,CAAC,CAAC;IAEnE,gBAAgB;IAChB,IAAA,sCAAkB,EAAC,MAAM,EAAE;QACzB,mBAAmB,EAAE,gBAAgB;KACtC,CAAC,CAAC;IAEH,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,sBAAsB,gBAAgB,CAAC,MAAM,+BAA+B,MAAM,GAAG,CACtF,CACF,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-derive-data-silos-from-data-flows-cross-instance.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli-derive-data-silos-from-data-flows-cross-instance.d.ts.map

package/build/cli-derive-data-silos-from-data-flows-cross-instance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-derive-data-silos-from-data-flows-cross-instance.d.ts","sourceRoot":"","sources":["../src/cli-derive-data-silos-from-data-flows-cross-instance.ts"],"names":[],"mappings":""}

package/build/cli-derive-data-silos-from-data-flows-cross-instance.js

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const graphql_1 = require("./graphql");
+const path_1 = require("path");
+const yargs_parser_1 = __importDefault(require("yargs-parser"));
+const difference_1 = __importDefault(require("lodash/difference"));
+const colors_1 = __importDefault(require("colors"));
+const logger_1 = require("./logger");
+const constants_1 = require("./constants");
+const dataFlowsToDataSilos_1 = require("./consent-manager/dataFlowsToDataSilos");
+const fs_1 = require("fs");
+const api_keys_1 = require("./api-keys");
+const readTranscendYaml_1 = require("./readTranscendYaml");
+const requests_1 = require("./requests");
+/**
+ * Derive the set of data silos from a set of data flows.
+ *
+ * Requires an API key - no scope required on API key
+ *
+ * Dev Usage:
+ * yarn ts-node ./src/cli-derive-data-silos-from-data-flows-cross-instance.ts --auth=$TRANSCEND_API_KEY \
+ *   --dataFlowsYmlFolder=./working/data-flows/ \
+ *   --output=./transcend.yml
+ *
+ * Standard usage:
+ * yarn tr-derive-data-silos-from-data-flows-cross-instance --auth=$TRANSCEND_API_KEY \
+ *   --dataFlowsYmlFolder=./working/data-flows/ \
+ *   --output=./transcend.yml
+ */
+async function main() {
+    // Parse command line arguments
+    const { transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, dataFlowsYmlFolder, output = 'transcend.yml', ignoreYmls = '', } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    // Ensure auth is passed
+    if (!auth) {
+        logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
+        process.exit(1);
+    }
+    // Ensure folder is passed to dataFlowsYmlFolder
+    if (!dataFlowsYmlFolder) {
+        logger_1.logger.error(colors_1.default.red('Missing required arg: --dataFlowsYmlFolder=./working/data-flows/'));
+        process.exit(1);
+    }
+    // Ensure folder is passed
+    if (!(0, fs_1.existsSync)(dataFlowsYmlFolder) ||
+        !(0, fs_1.lstatSync)(dataFlowsYmlFolder).isDirectory()) {
+        logger_1.logger.error(colors_1.default.red(`Folder does not exist: "${dataFlowsYmlFolder}"`));
+        process.exit(1);
+    }
+    // Ignore the data flows in these yml files
+    const instancesToIgnore = (0, requests_1.splitCsvToList)(ignoreYmls).map((x) => x.split('.')[0]);
+    // Map over each data flow yml file and convert to data silo configurations
+    const dataSiloInputs = (0, api_keys_1.listFiles)(dataFlowsYmlFolder).map((directory) => {
+        // read in the data flows for a specific instance
+        const { 'data-flows': dataFlows = [] } = (0, readTranscendYaml_1.readTranscendYaml)((0, path_1.join)(dataFlowsYmlFolder, directory));
+        // map the data flows to data silos
+        const { adTechDataSilos, siteTechDataSilos } = (0, dataFlowsToDataSilos_1.dataFlowsToDataSilos)(dataFlows, {
+            serviceToSupportedIntegration,
+            serviceToTitle,
+        });
+        return {
+            adTechDataSilos,
+            siteTechDataSilos,
+            organizationName: directory.split('.')[0],
+        };
+    });
+    // Mapping from service name to instances that have that service
+    const serviceToInstance = {};
+    dataSiloInputs.forEach(({ adTechDataSilos, siteTechDataSilos, organizationName }) => {
+        const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos];
+        allDataSilos.forEach((dataSilo) => {
+            const service = dataSilo['outer-type'] || dataSilo.integrationName;
+            // create mapping to instance
+            if (!serviceToInstance[service]) {
+                serviceToInstance[service] = [];
+            }
+            serviceToInstance[service].push(organizationName);
+            serviceToInstance[service] = [...new Set(serviceToInstance[service])];
+        });
+    });
+    // List of ad tech integrations
+    const adTechIntegrations = [
+        ...new Set(dataSiloInputs
+            .map(({ adTechDataSilos }) => adTechDataSilos.map((silo) => silo['outer-type'] || silo.integrationName))
+            .flat()),
+    ];
+    // List of site tech integrations
+    const siteTechIntegrations = (0, difference_1.default)([
+        ...new Set(dataSiloInputs
+            .map(({ siteTechDataSilos }) => siteTechDataSilos.map((silo) => silo['outer-type'] || silo.integrationName))
+            .flat()),
+    ], adTechIntegrations);
+    // Mapping from service name to list of
+    const serviceToFoundOnDomain = {};
+    dataSiloInputs.forEach(({ adTechDataSilos, siteTechDataSilos }) => {
+        const allDataSilos = [...adTechDataSilos, ...siteTechDataSilos];
+        allDataSilos.forEach((dataSilo) => {
+            var _a;
+            const service = dataSilo['outer-type'] || dataSilo.integrationName;
+            const foundOnDomain = (_a = dataSilo.attributes) === null || _a === void 0 ? void 0 : _a.find((attr) => attr.key === 'Found On Domain');
+            // create mapping to instance
+            if (!serviceToFoundOnDomain[service]) {
+                serviceToFoundOnDomain[service] = [];
+            }
+            serviceToFoundOnDomain[service].push(...((foundOnDomain === null || foundOnDomain === void 0 ? void 0 : foundOnDomain.values) || []));
+            serviceToFoundOnDomain[service] = [
+                ...new Set(serviceToFoundOnDomain[service]),
+            ];
+        });
+    });
+    // Fetch all integrations in the catalog
+    const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
+    const { serviceToTitle, serviceToSupportedIntegration } = await (0, graphql_1.fetchAndIndexCatalogs)(client);
+    // construct the aggregated data silo inputs
+    const dataSilos = [...adTechIntegrations, ...siteTechIntegrations].map((service) => ({
+        title: serviceToTitle[service],
+        ...(serviceToSupportedIntegration[service]
+            ? { integrationName: service }
+            : { integrationName: 'promptAPerson', 'outer-type': service }),
+        attributes: [
+            {
+                key: 'Tech Type',
+                values: ['Ad Tech'],
+            },
+            {
+                key: 'Business Units',
+                values: (0, difference_1.default)(serviceToInstance[service] || [], instancesToIgnore),
+            },
+            {
+                key: 'Found On Domain',
+                values: serviceToFoundOnDomain[service] || [],
+            },
+        ],
+    }));
+    // Log output
+    logger_1.logger.log(`Total Services: ${dataSilos.length}`);
+    logger_1.logger.log(`Ad Tech Services: ${adTechIntegrations.length}`);
+    logger_1.logger.log(`Site Tech Services: ${siteTechIntegrations.length}`);
+    // Write to yaml
+    (0, readTranscendYaml_1.writeTranscendYaml)(output, {
+        'data-silos': dataSilos,
+    });
+}
+main();
+//# sourceMappingURL=cli-derive-data-silos-from-data-flows-cross-instance.js.map

package/build/cli-derive-data-silos-from-data-flows-cross-instance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-derive-data-silos-from-data-flows-cross-instance.js","sourceRoot":"","sources":["../src/cli-derive-data-silos-from-data-flows-cross-instance.ts"],"names":[],"mappings":";;;;;;AACA,uCAA+E;AAC/E,+BAA4B;AAC5B,gEAAiC;AACjC,mEAA2C;AAC3C,oDAA4B;AAC5B,qCAAkC;AAClC,2CAAoD;AACpD,iFAA8E;AAE9E,2BAA2C;AAC3C,yCAAuC;AACvC,2DAA4E;AAC5E,yCAA4C;AAE5C;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,kBAAkB,EAClB,MAAM,GAAG,eAAe,EACxB,UAAU,GAAG,EAAE,GAChB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gDAAgD;IAChD,IAAI,CAAC,kBAAkB,EAAE;QACvB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,kEAAkE,CACnE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IACE,CAAC,IAAA,eAAU,EAAC,kBAAkB,CAAC;QAC/B,CAAC,IAAA,cAAS,EAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,EAC5C;QACA,eAAM,CAAC,KAAK,CAAC,gBAAM,CAAC,GAAG,CAAC,2BAA2B,kBAAkB,GAAG,CAAC,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,2CAA2C;IAC3C,MAAM,iBAAiB,GAAG,IAAA,yBAAc,EAAC,UAAU,CAAC,CAAC,GAAG,CACtD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CACvB,CAAC;IAEF,2EAA2E;IAC3E,MAAM,cAAc,GAAG,IAAA,oBAAS,EAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QACrE,iDAAiD;QACjD,MAAM,EAAE,YAAY,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,IAAA,qCAAiB,EACxD,IAAA,WAAI,EAAC,kBAAkB,EAAE,SAAS,CAAC,CACpC,CAAC;QAEF,mCAAmC;QACnC,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,GAAG,IAAA,2CAAoB,EACjE,SAA4B,EAC5B;YACE,6BAA6B;YAC7B,cAAc;SACf,CACF,CAAC;QAEF,OAAO;YACL,eAAe;YACf,iBAAiB;YACjB,gBAAgB,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC1C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,gEAAgE;IAChE,MAAM,iBAAiB,GAAgC,EAAE,CAAC;IAC1D,cAAc,CAAC,OAAO,CACpB,CAAC,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,EAAE,EAAE;QAC3D,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,EAAE,GAAG,iBAAiB,CAAC,CAAC;QAChE,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,eAAe,CAAC;YACnE,6BAA6B;YAC7B,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE;gBAC/B,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;aACjC;YACD,iBAAiB,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACnD,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CACF,CAAC;IAEF,+BAA+B;IAC/B,MAAM,kBAAkB,GAAG;QACzB,GAAG,IAAI,GAAG,CACR,cAAc;aACX,GAAG,CAAC,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,CAC3B,eAAe,CAAC,GAAG,CACjB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,eAAe,CACrD,CACF;aACA,IAAI,EAAE,CACV;KACF,CAAC;IAEF,iCAAiC;IACjC,MAAM,oBAAoB,GAAG,IAAA,oBAAU,EACrC;QACE,GAAG,IAAI,GAAG,CACR,cAAc;aACX,GAAG,CAAC,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAC7B,iBAAiB,CAAC,GAAG,CACnB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,eAAe,CACrD,CACF;aACA,IAAI,EAAE,CACV;KACF,EACD,kBAAkB,CACnB,CAAC;IAEF,uCAAuC;IACvC,MAAM,sBAAsB,GAAgC,EAAE,CAAC;IAC/D,cAAc,CAAC,OAAO,CAAC,CAAC,EAAE,eAAe,EAAE,iBAAiB,EAAE,EAAE,EAAE;QAChE,MAAM,YAAY,GAAG,CAAC,GAAG,eAAe,EAAE,GAAG,iBAAiB,CAAC,CAAC;QAChE,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;;YAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,eAAe,CAAC;YACnE,MAAM,aAAa,GAAG,MAAA,QAAQ,CAAC,UAAU,0CAAE,IAAI,CAC7C,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CACzC,CAAC;YACF,6BAA6B;YAC7B,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE;gBACpC,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;aACtC;YACD,sBAAsB,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,MAAM,KAAI,EAAE,CAAC,CAAC,CAAC;YACxE,sBAAsB,CAAC,OAAO,CAAC,GAAG;gBAChC,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;aAC5C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,wCAAwC;IACxC,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,EAAE,cAAc,EAAE,6BAA6B,EAAE,GACrD,MAAM,IAAA,+BAAqB,EAAC,MAAM,CAAC,CAAC;IAEtC,4CAA4C;IAC5C,MAAM,SAAS,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,oBAAoB,CAAC,CAAC,GAAG,CACpE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACZ,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC;QAC9B,GAAG,CAAC,6BAA6B,CAAC,OAAO,CAAC;YACxC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;YAC9B,CAAC,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;QAChE,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,WAAW;gBAChB,MAAM,EAAE,CAAC,SAAS,CAAC;aACpB;YACD;gBACE,GAAG,EAAE,gBAAgB;gBACrB,MAAM,EAAE,IAAA,oBAAU,EAChB,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,EAChC,iBAAiB,CAClB;aACF;YACD;gBACE,GAAG,EAAE,iBAAiB;gBACtB,MAAM,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,EAAE;aAC9C;SACF;KACF,CAAC,CACH,CAAC;IAEF,aAAa;IACb,eAAM,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,eAAM,CAAC,GAAG,CAAC,qBAAqB,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7D,eAAM,CAAC,GAAG,CAAC,uBAAuB,oBAAoB,CAAC,MAAM,EAAE,CAAC,CAAC;IAEjE,gBAAgB;IAChB,IAAA,sCAAkB,EAAC,MAAM,EAAE;QACzB,YAAY,EAAE,SAAS;KACxB,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/cli-derive-data-silos-from-data-flows.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli-derive-data-silos-from-data-flows.d.ts.map

package/build/cli-derive-data-silos-from-data-flows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-derive-data-silos-from-data-flows.d.ts","sourceRoot":"","sources":["../src/cli-derive-data-silos-from-data-flows.ts"],"names":[],"mappings":""}

package/build/cli-derive-data-silos-from-data-flows.js

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const graphql_1 = require("./graphql");
+const path_1 = require("path");
+const yargs_parser_1 = __importDefault(require("yargs-parser"));
+const colors_1 = __importDefault(require("colors"));
+const logger_1 = require("./logger");
+const constants_1 = require("./constants");
+const dataFlowsToDataSilos_1 = require("./consent-manager/dataFlowsToDataSilos");
+const requests_1 = require("./requests");
+const fs_1 = require("fs");
+const api_keys_1 = require("./api-keys");
+const readTranscendYaml_1 = require("./readTranscendYaml");
+/**
+ * Derive the set of data silos that can be derived from a set of data flows.
+ *
+ * Requires an API key - no scope required on API key
+ *
+ * Dev Usage:
+ * yarn ts-node ./src/cli-derive-data-silos-from-data-flows.ts --auth=$TRANSCEND_API_KEY \
+ *   --dataFlowsYmlFolder=./working/data-flows/ \
+ *   --dataSilosYmlFolder=./working/data-silos/
+ *
+ * Standard usage:
+ * yarn tr-derive-data-silos-from-data-flows --auth=$TRANSCEND_API_KEY \
+ *   --dataFlowsYmlFolder=./working/data-flows/ \
+ *   --dataSilosYmlFolder=./working/data-silos/
+ */
+async function main() {
+    // Parse command line arguments
+    const { transcendUrl = constants_1.DEFAULT_TRANSCEND_API, auth, dataFlowsYmlFolder, dataSilosYmlFolder, ignoreYmls = '', } = (0, yargs_parser_1.default)(process.argv.slice(2));
+    // Ensure auth is passed
+    if (!auth) {
+        logger_1.logger.error(colors_1.default.red('A Transcend API key must be provided. You can specify using --auth=$TRANSCEND_API_KEY'));
+        process.exit(1);
+    }
+    // Ensure folder is passed to dataFlowsYmlFolder
+    if (!dataFlowsYmlFolder) {
+        logger_1.logger.error(colors_1.default.red('Missing required arg: --dataFlowsYmlFolder=./working/data-flows/'));
+        process.exit(1);
+    }
+    // Ensure folder is passed
+    if (!(0, fs_1.existsSync)(dataFlowsYmlFolder) ||
+        !(0, fs_1.lstatSync)(dataFlowsYmlFolder).isDirectory()) {
+        logger_1.logger.error(colors_1.default.red(`Folder does not exist: "${dataFlowsYmlFolder}"`));
+        process.exit(1);
+    }
+    // Ensure folder is passed to dataSilosYmlFolder
+    if (!dataSilosYmlFolder) {
+        logger_1.logger.error(colors_1.default.red('Missing required arg: --dataSilosYmlFolder=./working/data-silos/'));
+        process.exit(1);
+    }
+    // Ensure folder is passed
+    if (!(0, fs_1.existsSync)(dataSilosYmlFolder) ||
+        !(0, fs_1.lstatSync)(dataSilosYmlFolder).isDirectory()) {
+        logger_1.logger.error(colors_1.default.red(`Folder does not exist: "${dataSilosYmlFolder}"`));
+        process.exit(1);
+    }
+    // Ignore the data flows in these yml files
+    const ymlsToIgnore = (0, requests_1.splitCsvToList)(ignoreYmls);
+    // Fetch all integrations in the catalog
+    const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
+    const { serviceToTitle, serviceToSupportedIntegration } = await (0, graphql_1.fetchAndIndexCatalogs)(client);
+    // List of each data flow yml file
+    (0, api_keys_1.listFiles)(dataFlowsYmlFolder).forEach((directory) => {
+        // read in the data flows for a specific instance
+        const { 'data-flows': dataFlows = [] } = (0, readTranscendYaml_1.readTranscendYaml)((0, path_1.join)(dataFlowsYmlFolder, directory));
+        // map the data flows to data silos
+        const { adTechDataSilos, siteTechDataSilos } = (0, dataFlowsToDataSilos_1.dataFlowsToDataSilos)(dataFlows, {
+            serviceToSupportedIntegration,
+            serviceToTitle,
+        });
+        // combine and write to yml file
+        const dataSilos = [...adTechDataSilos, ...siteTechDataSilos];
+        logger_1.logger.log(`Total Services: ${dataSilos.length}`);
+        logger_1.logger.log(`Ad Tech Services: ${adTechDataSilos.length}`);
+        logger_1.logger.log(`Site Tech Services: ${siteTechDataSilos.length}`);
+        (0, readTranscendYaml_1.writeTranscendYaml)((0, path_1.join)(dataSilosYmlFolder, directory), {
+            'data-silos': ymlsToIgnore.includes(directory) ? [] : dataSilos,
+        });
+    });
+}
+main();
+//# sourceMappingURL=cli-derive-data-silos-from-data-flows.js.map

package/build/cli-derive-data-silos-from-data-flows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-derive-data-silos-from-data-flows.js","sourceRoot":"","sources":["../src/cli-derive-data-silos-from-data-flows.ts"],"names":[],"mappings":";;;;;;AACA,uCAA+E;AAC/E,+BAA4B;AAC5B,gEAAiC;AACjC,oDAA4B;AAE5B,qCAAkC;AAClC,2CAAoD;AACpD,iFAA8E;AAE9E,yCAA4C;AAC5C,2BAA2C;AAC3C,yCAAuC;AACvC,2DAA4E;AAE5E;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,IAAI;IACjB,+BAA+B;IAC/B,MAAM,EACJ,YAAY,GAAG,iCAAqB,EACpC,IAAI,EACJ,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,GAAG,EAAE,GAChB,GAAG,IAAA,sBAAK,EAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAA8B,CAAC;IAE9D,wBAAwB;IACxB,IAAI,CAAC,IAAI,EAAE;QACT,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,uFAAuF,CACxF,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gDAAgD;IAChD,IAAI,CAAC,kBAAkB,EAAE;QACvB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,kEAAkE,CACnE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IACE,CAAC,IAAA,eAAU,EAAC,kBAAkB,CAAC;QAC/B,CAAC,IAAA,cAAS,EAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,EAC5C;QACA,eAAM,CAAC,KAAK,CAAC,gBAAM,CAAC,GAAG,CAAC,2BAA2B,kBAAkB,GAAG,CAAC,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,gDAAgD;IAChD,IAAI,CAAC,kBAAkB,EAAE;QACvB,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,kEAAkE,CACnE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,0BAA0B;IAC1B,IACE,CAAC,IAAA,eAAU,EAAC,kBAAkB,CAAC;QAC/B,CAAC,IAAA,cAAS,EAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,EAC5C;QACA,eAAM,CAAC,KAAK,CAAC,gBAAM,CAAC,GAAG,CAAC,2BAA2B,kBAAkB,GAAG,CAAC,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACjB;IAED,2CAA2C;IAC3C,MAAM,YAAY,GAAG,IAAA,yBAAc,EAAC,UAAU,CAAC,CAAC;IAEhD,wCAAwC;IACxC,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,EAAE,cAAc,EAAE,6BAA6B,EAAE,GACrD,MAAM,IAAA,+BAAqB,EAAC,MAAM,CAAC,CAAC;IAEtC,kCAAkC;IAClC,IAAA,oBAAS,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;QAClD,iDAAiD;QACjD,MAAM,EAAE,YAAY,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,IAAA,qCAAiB,EACxD,IAAA,WAAI,EAAC,kBAAkB,EAAE,SAAS,CAAC,CACpC,CAAC;QAEF,mCAAmC;QACnC,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,GAAG,IAAA,2CAAoB,EACjE,SAA4B,EAC5B;YACE,6BAA6B;YAC7B,cAAc;SACf,CACF,CAAC;QAEF,gCAAgC;QAChC,MAAM,SAAS,GAAG,CAAC,GAAG,eAAe,EAAE,GAAG,iBAAiB,CAAC,CAAC;QAC7D,eAAM,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,eAAM,CAAC,GAAG,CAAC,qBAAqB,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,eAAM,CAAC,GAAG,CAAC,uBAAuB,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,IAAA,sCAAkB,EAAC,IAAA,WAAI,EAAC,kBAAkB,EAAE,SAAS,CAAC,EAAE;YACtD,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;SAChE,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}

package/build/consent-manager/dataFlowsToDataSilos.d.ts

@@ -0,0 +1,19 @@
+import { DataFlowInput, DataSiloInput } from '../codecs';
+import { IndexedCatalogs } from '../graphql';
+/**
+ * Convert data flow configurations into a set of data silo configurations
+ *
+ * @param inputs - Data flow input to convert to data silos
+ * @param options - Additional options
+ * @returns Business entity configuration input
+ */
+export declare function dataFlowsToDataSilos(inputs: DataFlowInput[], { adTechPurposes, serviceToTitle, serviceToSupportedIntegration, }: IndexedCatalogs & {
+    /** List of purposes that are considered "Ad Tech"  */
+    adTechPurposes?: string[];
+}): {
+    /** List of data silo configurations for site-tech services */
+    siteTechDataSilos: DataSiloInput[];
+    /** List of data silo configurations for ad-tech services */
+    adTechDataSilos: DataSiloInput[];
+};
+//# sourceMappingURL=dataFlowsToDataSilos.d.ts.map

package/build/consent-manager/dataFlowsToDataSilos.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataFlowsToDataSilos.d.ts","sourceRoot":"","sources":["../../src/consent-manager/dataFlowsToDataSilos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE7C;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,aAAa,EAAE,EACvB,EACE,cAA+B,EAC/B,cAAc,EACd,6BAA6B,GAC9B,EAAE,eAAe,GAAG;IACnB,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B,GACA;IACD,8DAA8D;IAC9D,iBAAiB,EAAE,aAAa,EAAE,CAAC;IACnC,4DAA4D;IAC5D,eAAe,EAAE,aAAa,EAAE,CAAC;CAClC,CAiGA"}

package/build/consent-manager/dataFlowsToDataSilos.js

@@ -0,0 +1,95 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dataFlowsToDataSilos = void 0;
+const union_1 = __importDefault(require("lodash/union"));
+/**
+ * Convert data flow configurations into a set of data silo configurations
+ *
+ * @param inputs - Data flow input to convert to data silos
+ * @param options - Additional options
+ * @returns Business entity configuration input
+ */
+function dataFlowsToDataSilos(inputs, { adTechPurposes = ['SaleOfInfo'], serviceToTitle, serviceToSupportedIntegration, }) {
+    // List of site tech integrations
+    let siteTechIntegrations = [];
+    // List of ad tech integrations
+    const adTechIntegrations = [];
+    // Mapping from service name to list of
+    const serviceToFoundOnDomain = {};
+    // iterate over each flow
+    inputs.forEach((flow) => {
+        // process data flows with services
+        const { service, attributes = [] } = flow;
+        if (!service || service === 'internalService') {
+            return;
+        }
+        // create mapping to found on domain
+        const foundOnDomain = attributes.find((attr) => attr.key === 'Found on Domain');
+        // Create a list of all domains where the data flow was found
+        if (foundOnDomain) {
+            if (!serviceToFoundOnDomain[service]) {
+                serviceToFoundOnDomain[service] = [];
+            }
+            serviceToFoundOnDomain[service].push(...foundOnDomain.values.map((v) => v.replace('https://', '').replace('http://', '')));
+            serviceToFoundOnDomain[service] = [
+                ...new Set(serviceToFoundOnDomain[service]),
+            ];
+        }
+        // Keep track of ad tech
+        if ((0, union_1.default)(flow.trackingPurposes, adTechPurposes).length > 0) {
+            // add service to ad tech list
+            adTechIntegrations.push(service);
+            // remove from site tech list
+            if (siteTechIntegrations.includes(service)) {
+                siteTechIntegrations = siteTechIntegrations.filter((s) => s !== service);
+            }
+        }
+        else if (!adTechIntegrations.includes(service)) {
+            // add to site tech list
+            siteTechIntegrations.push(service);
+        }
+    });
+    // create the list of ad tech integrations
+    const adTechDataSilos = [...new Set(adTechIntegrations)].map((service) => ({
+        title: serviceToTitle[service],
+        ...(serviceToSupportedIntegration[service]
+            ? { integrationName: service }
+            : { integrationName: 'promptAPerson', 'outer-type': service }),
+        attributes: [
+            {
+                key: 'Tech Type',
+                values: ['Ad Tech'],
+            },
+            {
+                key: 'Found On Domain',
+                values: serviceToFoundOnDomain[service] || [],
+            },
+        ],
+    }));
+    // create the list of site tech integrations
+    const siteTechDataSilos = [...new Set(siteTechIntegrations)].map((service) => ({
+        title: serviceToTitle[service],
+        ...(serviceToSupportedIntegration[service]
+            ? { integrationName: service }
+            : { integrationName: 'promptAPerson', outerType: service }),
+        attributes: [
+            {
+                key: 'Tech Type',
+                values: ['Site Tech'],
+            },
+            {
+                key: 'Found On Domain',
+                values: serviceToFoundOnDomain[service] || [],
+            },
+        ],
+    }));
+    return {
+        siteTechDataSilos,
+        adTechDataSilos,
+    };
+}
+exports.dataFlowsToDataSilos = dataFlowsToDataSilos;
+//# sourceMappingURL=dataFlowsToDataSilos.js.map

package/build/consent-manager/dataFlowsToDataSilos.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataFlowsToDataSilos.js","sourceRoot":"","sources":["../../src/consent-manager/dataFlowsToDataSilos.ts"],"names":[],"mappings":";;;;;;AACA,yDAAiC;AAGjC;;;;;;GAMG;AACH,SAAgB,oBAAoB,CAClC,MAAuB,EACvB,EACE,cAAc,GAAG,CAAC,YAAY,CAAC,EAC/B,cAAc,EACd,6BAA6B,GAI9B;IAOD,iCAAiC;IACjC,IAAI,oBAAoB,GAAa,EAAE,CAAC;IAExC,+BAA+B;IAC/B,MAAM,kBAAkB,GAAa,EAAE,CAAC;IAExC,uCAAuC;IACvC,MAAM,sBAAsB,GAAgC,EAAE,CAAC;IAE/D,yBAAyB;IACzB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,mCAAmC;QACnC,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC;QAC1C,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,iBAAiB,EAAE;YAC7C,OAAO;SACR;QAED,oCAAoC;QACpC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CACnC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CACzC,CAAC;QAEF,6DAA6D;QAC7D,IAAI,aAAa,EAAE;YACjB,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE;gBACpC,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;aACtC;YACD,sBAAsB,CAAC,OAAO,CAAE,CAAC,IAAI,CACnC,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CACjD,CACF,CAAC;YACF,sBAAsB,CAAC,OAAO,CAAC,GAAG;gBAChC,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;aAC5C,CAAC;SACH;QAED,wBAAwB;QACxB,IAAI,IAAA,eAAK,EAAC,IAAI,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;YAC3D,8BAA8B;YAC9B,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjC,6BAA6B;YAC7B,IAAI,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;gBAC1C,oBAAoB,GAAG,oBAAoB,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO,CACrB,CAAC;aACH;SACF;aAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;YAChD,wBAAwB;YACxB,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SACpC;IACH,CAAC,CAAC,CAAC;IAEH,0CAA0C;IAC1C,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzE,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC;QAC9B,GAAG,CAAC,6BAA6B,CAAC,OAAO,CAAC;YACxC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;YAC9B,CAAC,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;QAChE,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,WAAW;gBAChB,MAAM,EAAE,CAAC,SAAS,CAAC;aACpB;YACD;gBACE,GAAG,EAAE,iBAAiB;gBACtB,MAAM,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,EAAE;aAC9C;SACF;KACF,CAAC,CAAC,CAAC;IAEJ,4CAA4C;IAC5C,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAC9D,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACZ,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC;QAC9B,GAAG,CAAC,6BAA6B,CAAC,OAAO,CAAC;YACxC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;YAC9B,CAAC,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QAC7D,UAAU,EAAE;YACV;gBACE,GAAG,EAAE,WAAW;gBAChB,MAAM,EAAE,CAAC,WAAW,CAAC;aACtB;YACD;gBACE,GAAG,EAAE,iBAAiB;gBACtB,MAAM,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,EAAE;aAC9C;SACF;KACF,CAAC,CACH,CAAC;IAEF,OAAO;QACL,iBAAiB;QACjB,eAAe;KAChB,CAAC;AACJ,CAAC;AAhHD,oDAgHC"}

package/build/graphql/fetchCatalogs.d.ts

@@ -14,4 +14,24 @@ export interface Catalog {
  * @returns Integration catalogs
  */
 export declare function fetchAllCatalogs(client: GraphQLClient): Promise<Catalog[]>;
+export interface IndexedCatalogs {
+    /** Mapping from service name to service title */
+    serviceToTitle: {
+        [k in string]: string;
+    };
+    /** Mapping from service name to boolean indicate if service has API integration support */
+    serviceToSupportedIntegration: {
+        [k in string]: boolean;
+    };
+}
+/**
+ * Fetch all integration catalogs and index them for usage in common utility manners
+ *
+ * @param client - Client
+ * @returns Integration catalogs
+ */
+export declare function fetchAndIndexCatalogs(client: GraphQLClient): Promise<{
+    /** List of all catalogs */
+    catalogs: Catalog[];
+} & IndexedCatalogs>;
 //# sourceMappingURL=fetchCatalogs.d.ts.map

package/build/graphql/fetchCatalogs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetchCatalogs.d.ts","sourceRoot":"","sources":["../../src/graphql/fetchCatalogs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD,MAAM,WAAW,OAAO;IACtB,uBAAuB;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAID;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,OAAO,EAAE,CAAC,CAyBpB"}
+{"version":3,"file":"fetchCatalogs.d.ts","sourceRoot":"","sources":["../../src/graphql/fetchCatalogs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD,MAAM,WAAW,OAAO;IACtB,uBAAuB;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAID;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,OAAO,EAAE,CAAC,CAyBpB;AAED,MAAM,WAAW,eAAe;IAC9B,iDAAiD;IACjD,cAAc,EAAE;SAAG,CAAC,IAAI,MAAM,GAAG,MAAM;KAAE,CAAC;IAC1C,2FAA2F;IAC3F,6BAA6B,EAAE;SAAG,CAAC,IAAI,MAAM,GAAG,OAAO;KAAE,CAAC;CAC3D;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CACzE;IACE,2BAA2B;IAC3B,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB,GAAG,eAAe,CACpB,CAyBA"}

package/build/graphql/fetchCatalogs.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchAllCatalogs = void 0;
+exports.fetchAndIndexCatalogs = exports.fetchAllCatalogs = void 0;
 const gqls_1 = require("./gqls");
 const makeGraphQLRequest_1 = require("./makeGraphQLRequest");
 const PAGE_SIZE = 100;
@@ -29,4 +29,26 @@ async function fetchAllCatalogs(client) {
     return catalogs;
 }
 exports.fetchAllCatalogs = fetchAllCatalogs;
+/**
+ * Fetch all integration catalogs and index them for usage in common utility manners
+ *
+ * @param client - Client
+ * @returns Integration catalogs
+ */
+async function fetchAndIndexCatalogs(client) {
+    // Fetch all integrations in the catalog
+    const catalogs = await fetchAllCatalogs(client);
+    // Create mapping from service name to service title
+    const serviceToTitle = catalogs.reduce((acc, catalog) => Object.assign(acc, { [catalog.integrationName]: catalog.title }), {});
+    // Create mapping from service name to boolean indicate if service has API integration support
+    const serviceToSupportedIntegration = catalogs.reduce((acc, catalog) => Object.assign(acc, {
+        [catalog.integrationName]: catalog.hasApiFunctionality,
+    }), {});
+    return {
+        catalogs,
+        serviceToTitle,
+        serviceToSupportedIntegration,
+    };
+}
+exports.fetchAndIndexCatalogs = fetchAndIndexCatalogs;
 //# sourceMappingURL=fetchCatalogs.js.map