@contrail/cli 1.0.54 → 1.0.56

package/src/types/types-extractor.ts

@@ -1,34 +1,48 @@
 import { Entities } from '@contrail/sdk';
-import { ReducedType, ReducedOptionSet, ReducedTypeProperty } from './interfaces';
+import { ReducedType, ReducedTypeProperty } from './interfaces';
 import chalk from 'chalk';
 import fs from 'fs';
 import { authUser } from '../auth/login';
-import { getSeparatorByPlatform } from '../common';
+import * as path from 'path';
+import pLimit from 'p-limit';
+const limit = pLimit(30);
 
 const yaml = require('js-yaml');
 
-export async function getTypes(args) {
+export interface DataHolder {
+  types: any[],
+  optionSets: any[],
+  optionSetHierarchies: any[],
+  ruleSets: any[],
+  ruleSetLinks: any[],
+  usageInfo: any[],
+  typePropertyPolicies: any[],
+  typePropertyPolicyLinks: any[],
+}
+
+export async function getAll(args) {
   try {
     const userConfig = await authUser();
     console.log(`%s Getting all types for orgId: ${userConfig.orgId}`, chalk.blue.bold('START'));
-    console.log('%s Retrieving', chalk.yellow.bold('OPTION SETS'));
 
-    const optionSets = await new Entities().get({
-      entityName: 'type-property-option-set',
-    });
-    await getOptionSets(optionSets);
+    // make directory to house exports
+    const dir = `./contrail-export-${Date.now()}`;
+    fs.mkdirSync(dir);
 
-    console.log('%s Retrieved', chalk.yellow.bold('OPTION SETS'));
-    console.log('%s Retrieving', chalk.yellow.bold('TYPES'));
-
-    const reducedTypes = await getReducedTypes(optionSets);
-    const typesHolder = {
-      types: reducedTypes,
+    const data: DataHolder = {
+      types: await getTypes(),
+      optionSets: await getOptionSets(),
+      optionSetHierarchies: await getOptionSetHierarchies(),
+      ruleSets: await getRuleSets(),
+      ruleSetLinks: await getRuleSetLinks(),
+      usageInfo: await getUsageInfo(),
+      typePropertyPolicies: await getTypePropertyPolicies(),
+      typePropertyPolicyLinks: await getTypePropertyPolicyLinks(),
     };
-    saveData(typesHolder);
+    const reducedData = reduceData(data);
+    saveData(reducedData, dir);
 
     console.log('%s Retrieved', chalk.yellow.bold('TYPES'));
-
   }
   catch (e) {
     console.log(`%s Types Retrieval Failed. ${e}`, chalk.red.bold('FAIL'));
@@ -38,53 +52,179 @@ export async function getTypes(args) {
   console.log('%s Types Retrieval Complete.', chalk.green.bold('COMPLETE'));
 }
 
-async function getOptionSets(optionSets) {
-  let reducedOptionSets = [];
-  for (const optionSet of optionSets) {
-    const reducedOptionSet: ReducedOptionSet = {
-      name: optionSet.name,
-      isArchived: optionSet.isArchived,
-      optionSet: optionSet.optionSet,
-    };
+/**
+ *  stores reduced option sets and returns full option sets */
+export async function getOptionSets() {
+  console.log('%s Retrieving', chalk.yellow.bold('OPTION SETS'));
+  const optionSets = await new Entities().get({
+    entityName: 'type-property-option-set',
+  });
+  console.log('%s Retrieved', chalk.yellow.bold('OPTION SETS'));
+  return optionSets;
+}
 
-    reducedOptionSets.push(reducedOptionSet);
+export async function getTypes() {
+  console.log('%s Retrieving', chalk.yellow.bold('TYPES'));
+  const entitiesClient = new Entities();
+  const types = await entitiesClient.get({
+    entityName: 'type',
+  });
+  console.log(`${chalk.yellow.bold('TYPES')} Found ${types.length} types`);
+  const typesWithProperties = [];
+  const promises = [];
+  for (const type of types) {
+    const promise = limit(async () => {
+      const t = await entitiesClient.get({
+        entityName: 'type',
+        id: type.id,
+        relations: ['typeProperties'],
+      });
+      typesWithProperties.push(t);
+    });
+    promises.push(promise);
   }
+  await Promise.all(promises);
+  return typesWithProperties;
+}
 
-  const optionSetsHolder = {
-    optionSets: reducedOptionSets,
-  };
+async function getOptionSetHierarchies() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'option-set-hierarchy',
+  });
+}
 
-  reducedOptionSets = reducedOptionSets.sort((one, two) => (one.name < two.name ? -1 : 1));
+async function getRuleSets() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'type-rule-set',
+  });
+}
 
-  const CURRENT_DIR = process.cwd();
-  const separator = getSeparatorByPlatform();
+async function getRuleSetLinks() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'type-rule-set-link',
+  });
+}
 
-  const yamlStr = yaml.dump(optionSetsHolder);
-  fs.writeFileSync(CURRENT_DIR + separator + 'option_sets.yml', yamlStr, 'utf8');
+async function getUsageInfo() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'type-property-usage-info',
+  });
 }
 
-export async function getReducedTypes(optionSets) {
-  const types = await new Entities().get({
-    entityName: 'type',
+async function getTypePropertyPolicies() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'type-property-policy',
+  });
+}
+async function getTypePropertyPolicyLinks() {
+  const entitiesClient = new Entities();
+  return await entitiesClient.get({
+    entityName: 'type-property-policy-link',
   });
+}
 
-  console.log('Found types: ', types.length);
+function reduceTypePropertyPolicies(data: DataHolder) {
+  return data?.typePropertyPolicies?.map(x => {
+    return {
+      label: x.label,
+      policy: {
+        version: x.policy.version,
+        statements: x.policy.statements,
+      }
+    };
+  });
+}
 
-  let reducedTypes = [];
-  for (const type of types) {
-    const typeWithProperties = await new Entities().get({
-      entityName: 'type',
-      id: type.id,
-      relations: ['typeProperties'],
-    });
+function reduceTypePropertyPolicyLinks(data: DataHolder) {
+  const typeProperties = data?.types?.map(x => x.typeProperties).reduce((x, y) => x.concat(y));
+  return data?.typePropertyPolicyLinks?.map(x => {
+    return {
+      typePropertySlug: typeProperties.find(y => y.id === x.typePropertyId)?.slug,
+      typePropertyPolicyLabel: data.typePropertyPolicies.find(y => y.id === x.typePropertyPolicyId)?.label,
+    };
+  });
+}
+
+function reduceUsageInfo(data: DataHolder) {
+  const typeProperties = data?.types?.map(x => x.typeProperties).reduce((x, y) => x.concat(y));
+  return data?.usageInfo?.map(x => {
+    return {
+      used: x.used,
+      editable: x.editable,
+      typeSlug: data.types.find(y => y.id === x.typeId)?.slug,
+      typePropertySlug: typeProperties.find(y => y.id === x.typePropertyId)?.slug,
+    };
+  });
+}
 
-    const optionSetNameById = {};
-    for (const optionSet of optionSets) {
-      optionSetNameById[optionSet.id] = optionSet.name;
+function reduceRuleSetLinks(data: DataHolder) {
+  const reducedRuleSetLinks = [];
+  if (!data?.ruleSetLinks.length) {
+    return [];
+  }
+  for (const ruleSetLink of data.ruleSetLinks) {
+    const ruleSet = data?.ruleSets?.find(x => x.id === ruleSetLink.typeRuleSetId);
+    const type = data?.types?.find(x => x.id === ruleSetLink.typeId);
+
+    if (!ruleSet || !type) {
+      console.log('%s Skipping bad rule set link', ruleSetLink.id, chalk.red.bold('TYPES'));
     }
+    reducedRuleSetLinks.push({
+      ruleSet: ruleSet.name,
+      type: type.slug
+    });
+  }
+  return reducedRuleSetLinks;
+}
+
+function reduceRuleSets(data: DataHolder) {
+  return data?.ruleSets?.map(x => {
+    return {
+      ruleSet: x.ruleSet,
+      name: x.name,
+    };
+  });
+}
+
+function reduceOptionSets(data: DataHolder) {
+  return data.optionSets?.map(optionSet => {
+    return {
+      name: optionSet.name,
+      isArchived: optionSet.isArchived,
+      optionSet: optionSet.optionSet,
+    };
+  });
+}
 
+function reduceOptionSetHierarchies(data: DataHolder) {
+  return data.optionSetHierarchies?.map(x => {
+    const hierarchyByName = x.hierarchy.map(
+      optionSetId => data.optionSets?.find(x => x.id === optionSetId)?.name
+    );
+    return {
+      hierarchy: hierarchyByName,
+      options: x.options,
+      name: x.name
+    };
+  });
+}
+
+export function reduceTypes(dataHolder: DataHolder) {
+  console.log('Found types: ', dataHolder.types.length);
+
+  let reducedTypes = [];
+  const optionSetNameById = {};
+  for (const optionSet of dataHolder.optionSets) {
+    optionSetNameById[optionSet.id] = optionSet.name;
+  }
+  for (const type of dataHolder.types) {
     const reducedTypeProperties = [];
-    for (const typeProperty of typeWithProperties.typeProperties) {
+    for (const typeProperty of type.typeProperties) {
       const reducedTypeProperty: ReducedTypeProperty = {
         slug: typeProperty.slug,
         id: typeProperty.id,
@@ -108,10 +248,10 @@ export async function getReducedTypes(optionSets) {
     }
 
     const reducedType: ReducedType = {
-      typePath: typeWithProperties.typePath,
-      slug: typeWithProperties.slug,
-      label: typeWithProperties.label,
-      isArchived: typeWithProperties.isArchived,
+      typePath: type.typePath,
+      slug: type.slug,
+      label: type.label,
+      isArchived: type.isArchived,
       typeProperties: reducedTypeProperties,
     };
 
@@ -122,9 +262,50 @@ export async function getReducedTypes(optionSets) {
   return reducedTypes;
 }
 
-function saveData(data) {
-  const yamlStr = yaml.dump(data);
-  const CURRENT_DIR = process.cwd();
-  const separator = getSeparatorByPlatform();
-  fs.writeFileSync(CURRENT_DIR + separator + 'types.yml', yamlStr, 'utf8');
+export function reduceData(data: DataHolder) {
+  return {
+    reducedTypes: reduceTypes(data),
+    reducedOptionSets: reduceOptionSets(data),
+    reducedOptionSetHierarchies: reduceOptionSetHierarchies(data),
+    reducedRuleSets: reduceRuleSets(data),
+    reducedRuleSetLinks: reduceRuleSetLinks(data),
+    reducedUsageInfo: reduceUsageInfo(data),
+    reducedTypePropertyPolicies: reduceTypePropertyPolicies(data),
+    reducedTypePropertyPolicyLinks: reduceTypePropertyPolicyLinks(data),
+  };
+}
+
+function saveData(reducedData, dir: string) {
+
+  const propertyToFolderMap = {
+    reducedTypes: { filename: 'types', split: true, fileLabel: 'slug' },
+    reducedOptionSets: { filename: 'option-sets', split: true, fileLabel: 'name' },
+    reducedOptionSetHierarchies: { filename: 'option-set-hierarchies', split: true, fileLabel: 'name' },
+    reducedRuleSets: { filename: 'rule-sets', split: true, fileLabel: 'name' },
+    reducedRuleSetLinks: { filename: 'rule-set-links', split: false, fileLabel: '' },
+    reducedUsageInfo: { filename: 'usage-info', split: false, fileLabel: '' },
+    reducedTypePropertyPolicies: { filename: 'type-property-policies', split: true, fileLabel: 'label' },
+    reducedTypePropertyPolicyLinks: { filename: 'type-property-policy-links', split: false, fileLabel: '' },
+  };
+  for(const [key, folder] of Object.entries(propertyToFolderMap)) {
+    const CURRENT_DIR = process.cwd();
+    if(folder.split) {
+      const subDir = path.join(CURRENT_DIR, dir, folder.filename);
+      if(!fs.existsSync(subDir)) {
+        fs.mkdirSync(subDir);
+      }
+      for(const item of reducedData[key]) {
+        const yamlStr = yaml.dump(item);
+        const fileLabel = item[folder.fileLabel];
+        if(!fileLabel) {
+          console.warn('skipping bad data:', item);
+        }
+        fs.writeFileSync(path.join(subDir, `${fileLabel}.yml`), yamlStr, 'utf8');
+      }
+    } else {
+      const yamlStr = yaml.dump(reducedData[key]);
+      fs.writeFileSync(path.join(CURRENT_DIR, dir, `${folder.filename}.yml`), yamlStr, 'utf8');
+    }
+
+  }
 }

package/src/types/types-properties-merger.ts

@@ -1,5 +1,4 @@
-import { Entities } from '@contrail/sdk';
-import { getReducedTypes } from './types-extractor';
+import { DataHolder, getOptionSets, getTypes, reduceData } from './types-extractor';
 import { ReducedTypeProperty, ReducedType } from './interfaces';
 import fs from 'fs';
 import { getSeparatorByPlatform } from '../common';
@@ -51,21 +50,27 @@ async function mergeTypeProperties(fileLocation, typePaths) {
 
   await authUser();
 
-  const optionSets = await new Entities().get({
-    entityName: 'type-property-option-set',
-  });
-
-  const reducedTypes = await getReducedTypes(optionSets);
+  const data: DataHolder = {
+    types: await getTypes(),
+    optionSets: await getOptionSets(),
+    optionSetHierarchies: [],
+    ruleSets: [],
+    ruleSetLinks: [],
+    usageInfo: [],
+    typePropertyPolicies: [],
+    typePropertyPolicyLinks: [],
+  };
+  const reducedData = await reduceData(data);
 
   for (const typePath of typePaths) {
-    const foundType = reducedTypes.find(reducedTypes => reducedTypes.typePath === typePath);
+    const foundType = reducedData.reducedTypes.find(reducedTypes => reducedTypes.typePath === typePath);
     if (!foundType){
       console.log(`%s No Type found for: ${typePath}`, chalk.red.bold('WARNING'));
     }
     mergeType(foundType, uploadedTypeProperties);
   }
 
-  saveData({types: reducedTypes});
+  saveData({types: reducedData.reducedTypes});
 }
 
 function mergeType(reducedType: ReducedType, typePropertiesToAdd: ReducedTypeProperty[]) {

package/src/types/types-uploader.ts

@@ -3,7 +3,7 @@ import { ReducedType } from './interfaces';
 import chalk from 'chalk';
 import fs from 'fs';
 import { authUser } from '../auth/login';
-import { getSeparatorByPlatform } from '../common';
+import * as path from 'path';
 import * as _ from 'lodash';
 
 import yaml from 'js-yaml';
@@ -29,9 +29,12 @@ export async function uploadTypes() {
 function loadReducedTypes(){
   const TARGET_DIR = process.cwd();
   try {
-    const separator = getSeparatorByPlatform();
-    const doc = yaml.load(fs.readFileSync(TARGET_DIR + separator + 'types.yml', 'utf8'));
-    return doc.types;
+    const files = fs.readdirSync(path.join(TARGET_DIR, 'types'));
+    const types = [];
+    for(const file of files) {
+      types.push(yaml.load(fs.readFileSync(path.join(TARGET_DIR, 'types', file), 'utf8')));
+    }
+    return types;
   } catch (e) {
     throw new Error(`ERROR PARSING YAML: ${e}`);
   }

package/src/types/usage-info-uploader.ts

@@ -0,0 +1,134 @@
+import { Entities } from '@contrail/sdk';
+import fs from 'fs';
+import { authUser } from '../auth/login';
+import chalk from 'chalk';
+import { getTypes } from './types-extractor';
+import * as path from 'path';
+
+const yaml = require('js-yaml');
+
+export async function uploadUsageInfo(args) {
+  try {
+    await authUser();
+
+    console.log('%s Uploading', chalk.yellow.bold('USAGE INFO'));
+    const reducedUsageInfo = await loadUsageInfo();
+    await uploadUsageInfoObjects(reducedUsageInfo);
+    console.log('%s Uploaded', chalk.yellow.bold('USAGE INFO'));
+
+  }
+  catch (e) {
+    console.log(`%s Usage Info Upload Failed. ${e}`, chalk.red.bold('FAIL'));
+    return;
+  }
+
+  console.log('%s Usage Info Upload Complete.', chalk.green.bold('COMPLETE'));
+}
+
+function loadUsageInfo() {
+  const TARGET_DIR = process.cwd();
+  try {
+    const doc = yaml.load(fs.readFileSync(path.join(TARGET_DIR, 'usage-info.yml'), 'utf8'));
+    return doc;
+  } catch (e) {
+    throw new Error(`ERROR PARSING YAML: ${e}`);
+  }
+}
+
+async function uploadUsageInfoObjects(loadedUsageInfo) {
+  const typesWithProperties = await getTypes();
+  const typeProperties = typesWithProperties
+    .filter(t=>!!t.typeProperties)
+    .map(t => t.typeProperties)
+    .reduce((a, b) => a.concat(b));
+
+  const currentUsageInfo = await new Entities().get({
+    entityName: 'type-property-usage-info',
+  });
+
+  const currentUsageInfosHashes = {};
+  for(const currentUI of currentUsageInfo) {
+    const foundType = typesWithProperties.find(t => t.id === currentUI.typeId);
+    const foundTypeProperty = typeProperties.find(tp => tp.id === currentUI.typePropertyId);
+    if(!foundType || !foundTypeProperty) {
+      console.log('skipping unhashable, invalid usage info:', currentUI);
+      continue;
+    }
+    const hash = `${foundType.slug}|${foundTypeProperty.slug}`;
+    currentUsageInfosHashes[hash] = currentUI;
+  }
+
+  const reducedUsageInfoHashes = {};
+  for(const loadedUI of loadedUsageInfo) {
+    const hash = `${loadedUI.typeSlug}|${loadedUI.typePropertySlug}`;
+    reducedUsageInfoHashes[hash] = loadedUI;
+  }
+  console.debug('current usage info:', currentUsageInfo);
+  console.debug('loaded usage info:', loadedUsageInfo);
+
+  for (const loadedUI of loadedUsageInfo) {
+    try {
+      const found = currentUsageInfosHashes[`${loadedUI.typeSlug}|${loadedUI.typePropertySlug}`];
+      if (!found) {
+        if (!transformReducedUI(loadedUI, typesWithProperties, typeProperties)) { continue; }
+        console.debug('Creating: ', loadedUI);
+        await new Entities().create({
+          entityName: 'type-property-usage-info',
+          object: {
+            typePropertyId: loadedUI.typePropertyId,
+            typeId: loadedUI.typeId,
+            editable: loadedUI.editable,
+            used: loadedUI.used,
+          }
+        });
+      } else {
+        if (!transformReducedUI(loadedUI, typesWithProperties, typeProperties)) { continue; }
+        console.log('Updating: ', loadedUI);
+        await new Entities().update({
+          entityName: 'type-property-usage-info',
+          id: found.id,
+          object: {
+            editable: loadedUI.editable,
+            used: loadedUI.used,
+          }
+        });
+      }
+    } catch (err) {
+      throw new Error(`Error Uploading usage info: ${err}`);
+    }
+  }
+
+  for (const [hash, currentUI] of Object.entries(currentUsageInfosHashes)) {
+    try {
+      const found = reducedUsageInfoHashes[hash];
+      if (!found) {
+        console.debug('Deleting: ', hash);
+        await new Entities().delete({
+          entityName: 'type-property-usage-info',
+          id: currentUI['id'],
+        });
+      }
+    } catch (err) {
+      throw new Error(`Error Deleting usage info: ${err}`);
+    }
+  }
+
+}
+
+// transforms the reduced Usage Info from name slugs to ids
+function transformReducedUI(reducedUsageInfo, types, typeProperties) {
+  const foundType = types.find(t => t.slug === reducedUsageInfo.typeSlug);
+  if (!foundType) {
+    console.error('Could not find type with slug', reducedUsageInfo.typeSlug, 'while transforming', reducedUsageInfo);
+    return false;
+  }
+  reducedUsageInfo.typeId = foundType.id;
+
+  const foundTypeProperty = typeProperties.find(tp => tp.slug === reducedUsageInfo.typePropertySlug);
+  if (!foundTypeProperty) {
+    console.error('Could not find type property with slug', reducedUsageInfo.typePropertySlug, 'while transforming', reducedUsageInfo);
+    return false;
+  }
+  reducedUsageInfo.typePropertyId = foundTypeProperty.id;
+  return true;
+}