@contrail/cli 1.0.53 → 1.0.55

package/lib/types/type-property-policy-uploader.js

@@ -0,0 +1,173 @@
+import { Entities } from '@contrail/sdk';
+import fs from 'fs';
+import { getSeparatorByPlatform } from '../common';
+import { authUser } from '../auth/login';
+import chalk from 'chalk';
+import { getTypes } from './types-extractor';
+const yaml = require('js-yaml');
+export async function uploadTypePropertyPolicies(args) {
+    try {
+        await authUser();
+        console.log('%s Uploading', chalk.yellow.bold('TYPE PROPERTY POLICY'));
+        const [reducedTypePropertyPolicies, reducedTypePropertyPolicyLinks] = await loadTPPolicies();
+        await uploadTPPolicyObjects(reducedTypePropertyPolicies);
+        await uploadTypePropertyPolicyLinks(reducedTypePropertyPolicyLinks);
+        console.log('%s Uploaded', chalk.yellow.bold('TYPE PROPERTY POLICY'));
+    }
+    catch (e) {
+        console.log(`%s Type Property Policy Upload Failed. ${e}`, chalk.red.bold('FAIL'));
+        return;
+    }
+    console.log('%s Type Property Policy Upload Complete.', chalk.green.bold('COMPLETE'));
+}
+function loadTPPolicies() {
+    const TARGET_DIR = process.cwd();
+    try {
+        const separator = getSeparatorByPlatform();
+        const doc = yaml.load(fs.readFileSync(TARGET_DIR + separator + 'types.yml', 'utf8'));
+        return [doc.reducedTypePropertyPolicies, doc.reducedTypePropertyPolicyLinks];
+    }
+    catch (e) {
+        throw new Error(`ERROR PARSING YAML: ${e}`);
+    }
+}
+async function uploadTPPolicyObjects(reducedTypePropertyPolicies) {
+    const currentTPPolicies = await new Entities().get({
+        entityName: 'type-property-policy',
+    });
+    const currentTPPByLabel = {};
+    for (const currentTPP of currentTPPolicies) {
+        currentTPPByLabel[currentTPP.label] = currentTPP;
+    }
+    const loadedTPPByName = {};
+    for (const loadedTPP of reducedTypePropertyPolicies) {
+        loadedTPPByName[loadedTPP.label] = loadedTPP;
+    }
+    console.debug('current tp policy:', currentTPPolicies);
+    console.debug('reduced tp policy:', reducedTypePropertyPolicies);
+    for (const [label, loadedTPP] of Object.entries(loadedTPPByName)) {
+        try {
+            const found = currentTPPByLabel[label];
+            if (!found) {
+                console.log('Creating: ', label);
+                await new Entities().create({
+                    entityName: 'type-property-policy',
+                    object: loadedTPP
+                });
+            }
+            else {
+                console.log('Updating: ', label);
+                await new Entities().update({
+                    entityName: 'type-property-policy',
+                    id: found.id,
+                    object: loadedTPP,
+                });
+            }
+        }
+        catch (err) {
+            throw new Error(`Error Uploading Type Property Policy: ${err}`);
+        }
+    }
+    for (const [label, currentTPP] of Object.entries(currentTPPByLabel)) {
+        try {
+            const found = loadedTPPByName[label];
+            if (!found) {
+                console.log('Deleting: ', label);
+                await new Entities().delete({
+                    entityName: 'type-property-policy',
+                    id: currentTPP['id'],
+                });
+            }
+        }
+        catch (err) {
+            throw new Error(`Error Type Property Policy: ${err}`);
+        }
+    }
+}
+async function uploadTypePropertyPolicyLinks(reducedTypePropertyPolicyLinks) {
+    const typesWithProperties = await getTypes();
+    const typeProperties = typesWithProperties
+        .filter(t => !!t.typeProperties)
+        .map(t => t.typeProperties)
+        .reduce((a, b) => a.concat(b));
+    const currentTPPs = await new Entities().get({
+        entityName: 'type-property-policy',
+    });
+    const currentTPPLinks = await new Entities().get({
+        entityName: 'type-property-policy-link',
+    });
+    const currentLinksHashes = {};
+    for (const currentLink of currentTPPLinks) {
+        const foundTypeProperty = typeProperties.find(tp => tp.id === currentLink.typePropertyId);
+        const foundTPP = currentTPPs.find(tpp => tpp.id === currentLink.typePropertyPolicyId);
+        if (!foundTypeProperty || !foundTPP) {
+            console.log('skipping unhashable, invalid policy link:', currentLink);
+        }
+        const hash = `${foundTypeProperty.slug}|${foundTPP.label}`;
+        currentLinksHashes[hash] = currentLink;
+    }
+    const reducedLinksByHashes = {};
+    for (const loadedLink of reducedTypePropertyPolicyLinks) {
+        const hash = `${loadedLink.typePropertySlug}|${loadedLink.typePropertyPolicyLabel}`;
+        reducedLinksByHashes[hash] = loadedLink;
+    }
+    console.debug('current type property policy links:', currentLinksHashes);
+    console.debug('reduced type property policy links:', reducedLinksByHashes);
+    for (const loadedTPPLink of reducedTypePropertyPolicyLinks) {
+        try {
+            const found = currentLinksHashes[`${loadedTPPLink.typePropertySlug}|${loadedTPPLink.typePropertyPolicyLabel}`];
+            if (!found) {
+                if (!transformReducedLink(loadedTPPLink, typeProperties, currentTPPs)) {
+                    continue;
+                }
+                console.log('Creating: ', loadedTPPLink);
+                await new Entities().create({
+                    entityName: 'type-property-policy-link',
+                    object: {
+                        typePropertyPolicyId: loadedTPPLink.typePropertyPolicyId,
+                        typePropertyId: loadedTPPLink.typePropertyId,
+                    },
+                });
+            }
+            else {
+                if (!transformReducedLink(loadedTPPLink, typeProperties, currentTPPs)) {
+                    continue;
+                }
+                console.log('skipping existing link: ', loadedTPPLink);
+            }
+        }
+        catch (err) {
+            throw new Error(`Error Uploading type property policy links: ${err}`);
+        }
+    }
+    for (const [currentLinkHash, currentLink] of Object.entries(currentLinksHashes)) {
+        try {
+            const found = reducedLinksByHashes[currentLinkHash];
+            if (!found) {
+                console.log('Deleting: ', currentLinkHash);
+                await new Entities().delete({
+                    entityName: 'type-property-policy-link',
+                    id: currentLink['id'],
+                });
+            }
+        }
+        catch (err) {
+            throw new Error(`Error Deleting rule set link: ${err}`);
+        }
+    }
+}
+function transformReducedLink(TPPLink, typeProperties, typePropertyPolicies) {
+    const foundTP = typeProperties.find(tp => tp.slug === TPPLink.typePropertySlug);
+    if (!foundTP) {
+        console.error('Could not find type property with label', TPPLink.typePropertySlug);
+        return false;
+    }
+    TPPLink.typePropertyId = foundTP.id;
+    const foundTPP = typePropertyPolicies.find(tpp => tpp.label === TPPLink.typePropertyPolicyLabel);
+    if (!foundTPP) {
+        console.error('Could not find type property policy with label', TPPLink.typePropertyPolicyLabel);
+        return false;
+    }
+    TPPLink.typePropertyPolicyId = foundTPP.id;
+    return true;
+}

package/lib/types/types-extractor.d.ts

@@ -1,2 +1,51 @@
-export declare function getTypes(args: any): Promise<void>;
-export declare function getReducedTypes(optionSets: any): Promise<any[]>;
+export interface DataHolder {
+    types: any[];
+    optionSets: any[];
+    optionSetHierarchies: any[];
+    ruleSets: any[];
+    ruleSetLinks: any[];
+    usageInfo: any[];
+    typePropertyPolicies: any[];
+    typePropertyPolicyLinks: any[];
+}
+export declare function getAll(args: any): Promise<void>;
+/**
+ *  stores reduced option sets and returns full option sets */
+export declare function getOptionSets(): Promise<any>;
+export declare function getTypes(): Promise<any[]>;
+export declare function reduceTypes(dataHolder: DataHolder): any[];
+export declare function reduceData(data: DataHolder): {
+    reducedTypes: any[];
+    reducedOptionSets: {
+        name: any;
+        isArchived: any;
+        optionSet: any;
+    }[];
+    reducedOptionSetHierarchies: {
+        hierarchy: any;
+        options: any;
+        name: any;
+    }[];
+    reducedRuleSets: {
+        ruleSet: any;
+        name: any;
+    }[];
+    reducedRuleSetLinks: any[];
+    reducedUsageInfo: {
+        used: any;
+        editable: any;
+        typeSlug: any;
+        typePropertySlug: any;
+    }[];
+    reducedTypePropertyPolicies: {
+        label: any;
+        policy: {
+            version: any;
+            statements: any;
+        };
+    }[];
+    reducedTypePropertyPolicyLinks: {
+        typePropertySlug: any;
+        typePropertyPolicyLabel: any;
+    }[];
+};

package/lib/types/types-extractor.js

@@ -3,23 +3,29 @@ import chalk from 'chalk';
 import fs from 'fs';
 import { authUser } from '../auth/login';
 import { getSeparatorByPlatform } from '../common';
+import pLimit from 'p-limit';
+const limit = pLimit(30);
 const yaml = require('js-yaml');
-export async function getTypes(args) {
+export async function getAll(args) {
     try {
         const userConfig = await authUser();
         console.log(`%s Getting all types for orgId: ${userConfig.orgId}`, chalk.blue.bold('START'));
-        console.log('%s Retrieving', chalk.yellow.bold('OPTION SETS'));
-        const optionSets = await new Entities().get({
-            entityName: 'type-property-option-set',
-        });
-        await getOptionSets(optionSets);
-        console.log('%s Retrieved', chalk.yellow.bold('OPTION SETS'));
-        console.log('%s Retrieving', chalk.yellow.bold('TYPES'));
-        const reducedTypes = await getReducedTypes(optionSets);
-        const typesHolder = {
-            types: reducedTypes,
+        // make directory to house exports
+        const dir = `./types-${Date.now()}`;
+        fs.mkdirSync(dir);
+        const data = {
+            types: await getTypes(),
+            optionSets: await getOptionSets(),
+            optionSetHierarchies: await getOptionSetHierarchies(),
+            ruleSets: await getRuleSets(),
+            ruleSetLinks: await getRuleSetLinks(),
+            usageInfo: await getUsageInfo(),
+            typePropertyPolicies: await getTypePropertyPolicies(),
+            typePropertyPolicyLinks: await getTypePropertyPolicyLinks(),
         };
-        saveData(typesHolder);
+        const reducedData = reduceData(data);
+        storeOptionSets(data, dir);
+        saveData(reducedData, dir);
         console.log('%s Retrieved', chalk.yellow.bold('TYPES'));
     }
     catch (e) {
@@ -28,43 +34,170 @@ export async function getTypes(args) {
     }
     console.log('%s Types Retrieval Complete.', chalk.green.bold('COMPLETE'));
 }
-async function getOptionSets(optionSets) {
-    let reducedOptionSets = [];
-    for (const optionSet of optionSets) {
-        const reducedOptionSet = {
+/**
+ *  stores reduced option sets and returns full option sets */
+export async function getOptionSets() {
+    console.log('%s Retrieving', chalk.yellow.bold('OPTION SETS'));
+    const optionSets = await new Entities().get({
+        entityName: 'type-property-option-set',
+    });
+    console.log('%s Retrieved', chalk.yellow.bold('OPTION SETS'));
+    return optionSets;
+}
+export async function getTypes() {
+    console.log('%s Retrieving', chalk.yellow.bold('TYPES'));
+    const entitiesClient = new Entities();
+    const types = await entitiesClient.get({
+        entityName: 'type',
+    });
+    console.log(`${chalk.yellow.bold('TYPES')} Found ${types.length} types`);
+    const typesWithProperties = [];
+    const promises = [];
+    for (const type of types) {
+        const promise = limit(async () => {
+            const t = await entitiesClient.get({
+                entityName: 'type',
+                id: type.id,
+                relations: ['typeProperties'],
+            });
+            typesWithProperties.push(t);
+        });
+        promises.push(promise);
+    }
+    await Promise.all(promises);
+    return typesWithProperties;
+}
+async function getOptionSetHierarchies() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'option-set-hierarchy',
+    });
+}
+async function getRuleSets() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'type-rule-set',
+    });
+}
+async function getRuleSetLinks() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'type-rule-set-link',
+    });
+}
+async function getUsageInfo() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'type-property-usage-info',
+    });
+}
+async function getTypePropertyPolicies() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'type-property-policy',
+    });
+}
+async function getTypePropertyPolicyLinks() {
+    const entitiesClient = new Entities();
+    return await entitiesClient.get({
+        entityName: 'type-property-policy-link',
+    });
+}
+function reduceTypePropertyPolicies(data) {
+    var _a;
+    return (_a = data === null || data === void 0 ? void 0 : data.typePropertyPolicies) === null || _a === void 0 ? void 0 : _a.map(x => {
+        return {
+            label: x.label,
+            policy: {
+                version: x.policy.version,
+                statements: x.policy.statements,
+            }
+        };
+    });
+}
+function reduceTypePropertyPolicyLinks(data) {
+    var _a, _b;
+    const typeProperties = (_a = data === null || data === void 0 ? void 0 : data.types) === null || _a === void 0 ? void 0 : _a.map(x => x.typeProperties).reduce((x, y) => x.concat(y));
+    return (_b = data === null || data === void 0 ? void 0 : data.typePropertyPolicyLinks) === null || _b === void 0 ? void 0 : _b.map(x => {
+        var _a, _b;
+        return {
+            typePropertySlug: (_a = typeProperties.find(y => y.id === x.typePropertyId)) === null || _a === void 0 ? void 0 : _a.slug,
+            typePropertyPolicyLabel: (_b = data.typePropertyPolicies.find(y => y.id === x.typePropertyPolicyId)) === null || _b === void 0 ? void 0 : _b.label,
+        };
+    });
+}
+function reduceUsageInfo(data) {
+    var _a, _b;
+    const typeProperties = (_a = data === null || data === void 0 ? void 0 : data.types) === null || _a === void 0 ? void 0 : _a.map(x => x.typeProperties).reduce((x, y) => x.concat(y));
+    return (_b = data === null || data === void 0 ? void 0 : data.usageInfo) === null || _b === void 0 ? void 0 : _b.map(x => {
+        var _a, _b;
+        return {
+            used: x.used,
+            editable: x.editable,
+            typeSlug: (_a = data.types.find(y => y.id === x.typeId)) === null || _a === void 0 ? void 0 : _a.slug,
+            typePropertySlug: (_b = typeProperties.find(y => y.id === x.typePropertyId)) === null || _b === void 0 ? void 0 : _b.slug,
+        };
+    });
+}
+function reduceRuleSetLinks(data) {
+    var _a, _b;
+    const reducedRuleSetLinks = [];
+    if (!(data === null || data === void 0 ? void 0 : data.ruleSetLinks.length)) {
+        return [];
+    }
+    for (const ruleSetLink of data.ruleSetLinks) {
+        const ruleSet = (_a = data === null || data === void 0 ? void 0 : data.ruleSets) === null || _a === void 0 ? void 0 : _a.find(x => x.id === ruleSetLink.typeRuleSetId);
+        const type = (_b = data === null || data === void 0 ? void 0 : data.types) === null || _b === void 0 ? void 0 : _b.find(x => x.id === ruleSetLink.typeId);
+        if (!ruleSet || !type) {
+            console.log('%s Skipping bad rule set link', ruleSetLink.id, chalk.red.bold('TYPES'));
+        }
+        reducedRuleSetLinks.push({
+            ruleSet: ruleSet.name,
+            type: type.slug
+        });
+    }
+    return reducedRuleSetLinks;
+}
+function reduceRuleSets(data) {
+    var _a;
+    return (_a = data === null || data === void 0 ? void 0 : data.ruleSets) === null || _a === void 0 ? void 0 : _a.map(x => {
+        return {
+            ruleSet: x.ruleSet,
+            name: x.name,
+        };
+    });
+}
+function reduceOptionSets(data) {
+    var _a;
+    return (_a = data.optionSets) === null || _a === void 0 ? void 0 : _a.map(optionSet => {
+        return {
             name: optionSet.name,
             isArchived: optionSet.isArchived,
             optionSet: optionSet.optionSet,
         };
-        reducedOptionSets.push(reducedOptionSet);
-    }
-    const optionSetsHolder = {
-        optionSets: reducedOptionSets,
-    };
-    reducedOptionSets = reducedOptionSets.sort((one, two) => (one.name < two.name ? -1 : 1));
-    const CURRENT_DIR = process.cwd();
-    const separator = getSeparatorByPlatform();
-    const yamlStr = yaml.dump(optionSetsHolder);
-    fs.writeFileSync(CURRENT_DIR + separator + 'option_sets.yml', yamlStr, 'utf8');
+    });
 }
-export async function getReducedTypes(optionSets) {
-    const types = await new Entities().get({
-        entityName: 'type',
+function reduceOptionSetHierarchies(data) {
+    var _a;
+    return (_a = data.optionSetHierarchies) === null || _a === void 0 ? void 0 : _a.map(x => {
+        const hierarchyByName = x.hierarchy.map(optionSetId => { var _a, _b; return (_b = (_a = data.optionSets) === null || _a === void 0 ? void 0 : _a.find(x => x.id === optionSetId)) === null || _b === void 0 ? void 0 : _b.name; });
+        return {
+            hierarchy: hierarchyByName,
+            options: x.options,
+            name: x.name
+        };
     });
-    console.log('Found types: ', types.length);
+}
+export function reduceTypes(dataHolder) {
+    console.log('Found types: ', dataHolder.types.length);
     let reducedTypes = [];
-    for (const type of types) {
-        const typeWithProperties = await new Entities().get({
-            entityName: 'type',
-            id: type.id,
-            relations: ['typeProperties'],
-        });
-        const optionSetNameById = {};
-        for (const optionSet of optionSets) {
-            optionSetNameById[optionSet.id] = optionSet.name;
-        }
+    const optionSetNameById = {};
+    for (const optionSet of dataHolder.optionSets) {
+        optionSetNameById[optionSet.id] = optionSet.name;
+    }
+    for (const type of dataHolder.types) {
         const reducedTypeProperties = [];
-        for (const typeProperty of typeWithProperties.typeProperties) {
+        for (const typeProperty of type.typeProperties) {
             const reducedTypeProperty = {
                 slug: typeProperty.slug,
                 id: typeProperty.id,
@@ -86,10 +219,10 @@ export async function getReducedTypes(optionSets) {
             reducedTypeProperties.push(reducedTypeProperty);
         }
         const reducedType = {
-            typePath: typeWithProperties.typePath,
-            slug: typeWithProperties.slug,
-            label: typeWithProperties.label,
-            isArchived: typeWithProperties.isArchived,
+            typePath: type.typePath,
+            slug: type.slug,
+            label: type.label,
+            isArchived: type.isArchived,
             typeProperties: reducedTypeProperties,
         };
         reducedTypes.push(reducedType);
@@ -97,9 +230,32 @@ export async function getReducedTypes(optionSets) {
     reducedTypes = reducedTypes.sort((one, two) => (one.typePath < two.typePath ? -1 : 1));
     return reducedTypes;
 }
-function saveData(data) {
-    const yamlStr = yaml.dump(data);
+export function reduceData(data) {
+    return {
+        reducedTypes: reduceTypes(data),
+        reducedOptionSets: reduceOptionSets(data),
+        reducedOptionSetHierarchies: reduceOptionSetHierarchies(data),
+        reducedRuleSets: reduceRuleSets(data),
+        reducedRuleSetLinks: reduceRuleSetLinks(data),
+        reducedUsageInfo: reduceUsageInfo(data),
+        reducedTypePropertyPolicies: reduceTypePropertyPolicies(data),
+        reducedTypePropertyPolicyLinks: reduceTypePropertyPolicyLinks(data),
+    };
+}
+function storeOptionSets(data, dir) {
+    let reducedOptionSets = reduceOptionSets(data);
+    reducedOptionSets = reducedOptionSets.sort((one, two) => (one.name < two.name ? -1 : 1));
+    const optionSetsHolder = {
+        reducedOptionSets,
+    };
+    const CURRENT_DIR = process.cwd();
+    const separator = getSeparatorByPlatform();
+    const yamlStr = yaml.dump(optionSetsHolder);
+    fs.writeFileSync(CURRENT_DIR + separator + dir + separator + 'option_sets.yml', yamlStr, 'utf8');
+}
+function saveData(reducedData, dir) {
+    const yamlStr = yaml.dump(reducedData);
     const CURRENT_DIR = process.cwd();
     const separator = getSeparatorByPlatform();
-    fs.writeFileSync(CURRENT_DIR + separator + 'types.yml', yamlStr, 'utf8');
+    fs.writeFileSync(CURRENT_DIR + separator + dir + separator + 'types.yml', yamlStr, 'utf8');
 }

package/lib/types/types-properties-merger.js

@@ -1,5 +1,4 @@
-import { Entities } from '@contrail/sdk';
-import { getReducedTypes } from './types-extractor';
+import { getOptionSets, getTypes, reduceData } from './types-extractor';
 import fs from 'fs';
 import { getSeparatorByPlatform } from '../common';
 import { authUser } from '../auth/login';
@@ -40,18 +39,25 @@ function loadTypeProperties(fileLocation) {
 async function mergeTypeProperties(fileLocation, typePaths) {
     const uploadedTypeProperties = loadTypeProperties(fileLocation);
     await authUser();
-    const optionSets = await new Entities().get({
-        entityName: 'type-property-option-set',
-    });
-    const reducedTypes = await getReducedTypes(optionSets);
+    const data = {
+        types: await getTypes(),
+        optionSets: await getOptionSets(),
+        optionSetHierarchies: [],
+        ruleSets: [],
+        ruleSetLinks: [],
+        usageInfo: [],
+        typePropertyPolicies: [],
+        typePropertyPolicyLinks: [],
+    };
+    const reducedData = await reduceData(data);
     for (const typePath of typePaths) {
-        const foundType = reducedTypes.find(reducedTypes => reducedTypes.typePath === typePath);
+        const foundType = reducedData.reducedTypes.find(reducedTypes => reducedTypes.typePath === typePath);
         if (!foundType) {
             console.log(`%s No Type found for: ${typePath}`, chalk.red.bold('WARNING'));
         }
         mergeType(foundType, uploadedTypeProperties);
     }
-    saveData({ types: reducedTypes });
+    saveData({ types: reducedData.reducedTypes });
 }
 function mergeType(reducedType, typePropertiesToAdd) {
     for (const typeProperty of typePropertiesToAdd) {

package/lib/types/types-uploader.js

@@ -24,7 +24,7 @@ function loadReducedTypes() {
     try {
         const separator = getSeparatorByPlatform();
         const doc = yaml.load(fs.readFileSync(TARGET_DIR + separator + 'types.yml', 'utf8'));
-        return doc.types;
+        return doc.reducedTypes;
     }
     catch (e) {
         throw new Error(`ERROR PARSING YAML: ${e}`);

package/lib/types/usage-info-uploader.d.ts

@@ -0,0 +1 @@
+export declare function uploadUsageInfo(args: any): Promise<void>;