@coalescesoftware/coa 1.0.113 → 1.0.114

package/bin/CLIProfile.js

@@ -55,6 +55,7 @@ exports.GetDefaultLocationForCoaConfigFile = GetDefaultLocationForCoaConfigFile;
 exports.ICLIProfileExample = {
     profile: "Profile To Use",
     environmentID: "Environment ID",
+    parameters: "Parameters",
     snowflakeAccount: "Snowflake Account To Use",
     snowflakeAuthType: "Snowflake Auth Type (Basic, KeyPair)",
     snowflakeKeyPairPath: "Snowflake Key Pair Path",
@@ -68,21 +69,29 @@ exports.ICLIProfileExample = {
     exclude: "Coalesce Node Selector"
 };
 const ReadCLIProfiles = (filePath) => {
-    return fs.promises.readFile(filePath, 'utf-8').then((file) => {
+    let filePathToUse;
+    if (!filePath) {
+        filePathToUse = (0, exports.GetDefaultLocationForCoaConfigFile)();
+    }
+    else {
+        filePathToUse = filePath;
+    }
+    return fs.promises.readFile(filePathToUse, 'utf-8')
+        .then((file) => {
         return ini.parse(file);
-    }).catch((err) => {
+    })
+        .catch((err) => {
         CLILogger.error("unable to read cli profile file filePath:", filePath, err);
-        throw new Error("unable to read cli profile");
+        if (!filePath) //if no filepath was specified, silently proceed
+            return {};
+        else //unable to proceed couldnt read file
+            throw new Error(`unable to read cli profile:${filePathToUse}`);
     });
 };
 const GetFinalCLIProfile = (commandLineOverrides, configFileLocation) => {
     let profileToUseOverride = commandLineOverrides.profile ? commandLineOverrides.profile : null;
-    return ReadCLIProfiles(configFileLocation).then((cliProfiles) => {
-        return cliProfiles;
-    }).catch((err) => {
-        //unable to read cli profiles 
-        return {};
-    }).then((cliProfiles) => {
+    return ReadCLIProfiles(configFileLocation)
+        .then((cliProfiles) => {
         const defaultCLIProfile = cliProfiles.default;
         let finalCLIProfile = {};
         //if theres a default cli profile, start with that
@@ -124,7 +133,8 @@ const GetCLIConfig = (commandLineOverrides, configFileLocation) => {
                 snowflakeRole: cliProfile.snowflakeRole,
                 snowflakeUsername: cliProfile.snowflakeUsername,
                 snowflakeWarehouse: cliProfile.snowflakeWarehouse
-            }
+            },
+            runtimeParameters: cliProfile.parameters,
         };
         Shared.Common.CleanupUndefinedValuesFromObject(cliConfig);
         return cliConfig;

package/bin/CommonCLI.js

@@ -76,21 +76,20 @@ const GetKeyPairKey = (keyPairPath) => {
 };
 const GetUserConnectionForCLI = (userID, runInfo) => {
     return new Promise((resolve, reject) => {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a, _b, _c, _d, _e, _f;
         const output = {
             connectionDetails: {
                 userID,
                 user: (_a = runInfo.userCredentials) === null || _a === void 0 ? void 0 : _a.snowflakeUsername,
                 role: (_b = runInfo.userCredentials) === null || _b === void 0 ? void 0 : _b.snowflakeRole,
                 warehouse: (_c = runInfo.userCredentials) === null || _c === void 0 ? void 0 : _c.snowflakeWarehouse,
-                authenticator: (_d = runInfo.userCredentials) === null || _d === void 0 ? void 0 : _d.snowflakeAuthType
             },
-            connectionType: (_e = runInfo.userCredentials) === null || _e === void 0 ? void 0 : _e.snowflakeAuthType
+            connectionType: (_d = runInfo.userCredentials) === null || _d === void 0 ? void 0 : _d.snowflakeAuthType
         };
-        if (!((_f = runInfo.userCredentials) === null || _f === void 0 ? void 0 : _f.snowflakeAuthType)) {
+        if (!((_e = runInfo.userCredentials) === null || _e === void 0 ? void 0 : _e.snowflakeAuthType)) {
             reject(new Error("ERROR (GetUserConnectionForCLI): no auth type provided"));
         }
-        else if (((_g = runInfo.userCredentials) === null || _g === void 0 ? void 0 : _g.snowflakeAuthType) === Shared.ConnectionOperations.EUserConnectionTypes.keyPair) {
+        else if (((_f = runInfo.userCredentials) === null || _f === void 0 ? void 0 : _f.snowflakeAuthType) === Shared.ConnectionOperations.EUserConnectionTypes.keyPair) {
             GetKeyPairKey(Shared.Common.getValueSafe(runInfo, ["userCredentials", "snowflakeKeyPairPath"], ""))
                 .then((keyPair) => {
                 output.connectionDetails.keyPair = keyPair;

package/bin/Deploy.js

@@ -50,7 +50,7 @@ const DeployWithCLI = (plan, config, token) => {
         logContext = Shared.Logging.CreateLogContext(teamID, environmentID, userID);
         const connectionCache = new Shared.Snowflake.ConnectionStorageClass();
         RunSQL = Shared.SQLExecutorCreators.CreateRunSQLWithoutScheduler(teamDetails, connectionCache);
-        runInfo = Shared.DeployOperations.CreateDeployRequestObject(plan.plan, plan.environmentState, teamInfo, plan.gitInfo, plan.targetEnvironment, "");
+        runInfo = Shared.DeployOperations.CreateDeployRequestObject(plan.plan, plan.environmentState, teamInfo, plan.gitInfo, plan.targetEnvironment, plan.runtimeParameters);
         runInfo.userCredentials = config.userCredentials;
         return CommonCLI.GetUserConnectionForCLI(userID, runInfo);
     })

package/bin/Package.js

@@ -0,0 +1,731 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GetPackageSupportedEntityTypes = exports.RemoveNamespacedWorkspaceData_Testing = exports.NamespaceWorkspaceData_Testing = exports.UninstallPackage = exports.InstallPackage = exports.PackageProvider = exports.GetPackageListMessageCLI = exports.InitializePackageContext = exports.localInstallKeyword = void 0;
+/* eslint-disable multiline-comment-style */
+const Shared = __importStar(require("@coalescesoftware/shared"));
+const fs_1 = __importDefault(require("fs"));
+const os = require("os");
+/**
+ * coa package uses the same config file that should be setup to leverage coa.
+ *
+ * project - the user's ws data
+ * package - the targetted repo being applied to the project
+ * dependency - any package installed to a project/another package
+ *
+ * coa add works at a high level in these steps:
+ * 1 - collects ws data from a targetted repo (local or remote)
+ * 2 - namespaces that ws data w/package ID
+ * 3 - adds to firestore packages collection the info for the package being installed (dependency information) w/status: 'adding'
+ * 4 - updates project ws data on firestore with namespaced ws data from package
+ * 5 - updates firestore package dependency information w/status: 'added' and a timestamp
+ * 6 - in case of error, will clean up any added package information including dependency info from project ws data on FS
+ *
+ * coa remove works similarly
+ */
+exports.localInstallKeyword = "file:";
+const PackagesLogger = Shared.Logging.GetLogger(Shared.Logging.LoggingArea.CLI);
+var EPackageStatus;
+(function (EPackageStatus) {
+    EPackageStatus["adding"] = "adding";
+    EPackageStatus["added"] = "added";
+    EPackageStatus["removing"] = "removing";
+    EPackageStatus["error"] = "error";
+})(EPackageStatus || (EPackageStatus = {}));
+var EVersionType;
+(function (EVersionType) {
+    EVersionType["file"] = "file";
+    EVersionType["url"] = "url";
+})(EVersionType || (EVersionType = {}));
+////////////////////////////////////////////////////////////////////
+//////////////////// reusable namespacing helper functions
+////////////////////////////////////////////////////////////////////
+/**
+ * Recieves a string or a number that will be namespaced (NAMESPACEVALUE::STRINGorNUMBER)
+ * @param value
+ * @param namespace
+ * @returns
+ */
+const UpdateValueWithNamespace = (value, namespace) => {
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, typeof namespace === "string", `Namespace was not a string, but was type ${typeof namespace}`);
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, !!namespace.trim(), `Namespace cannot be an empty string`);
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, typeof value === "string" || typeof value === "number", `Value was not a string, but was type ${typeof value}`);
+    const stringValue = String(value);
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, stringValue.indexOf("::") === -1, `value to namespace contained invalid character ":"`);
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, namespace.indexOf("::") === -1, `namespace contained invalid character ":"`);
+    if (stringValue.trim() === "")
+        return stringValue; //empty string should not be namespaced but returned as is
+    return `${namespace}::${value}`;
+};
+/**
+ * Updates originalObject field values (fieldsToUpdate) using fieldValueUpdateCB and returns original value for non selected fields
+ *
+ * @param entity object with field values to update
+ * @param entityFieldsToUpdate an array of fields to update, or null to update all fields
+ * @param namespacerCB CB to call on each field to update
+ * @returns entity with field values that have been namespaced or updated with CB
+ */
+const EntityNamespacer = (entity, entityFieldsToUpdate, namespacerCB) => {
+    const updatedEntity = Object.assign({}, entity);
+    if (!entityFieldsToUpdate) {
+        // update all
+        Object.keys(entity).forEach(field => {
+            updatedEntity[field] = namespacerCB(updatedEntity[field]);
+        });
+    }
+    else {
+        Object.keys(entity).forEach(key => {
+            // if key included in keypath array, update, otherwise return original value
+            if (entityFieldsToUpdate.includes(key)) {
+                updatedEntity[key] = namespacerCB(updatedEntity[key]);
+            }
+            else {
+                updatedEntity[key] = entity[key];
+            }
+        });
+    }
+    return updatedEntity;
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Parent namespacing functions
+////////////////////////////////////////////////////////////////////
+/**
+ * internal functions that will add or remove namespaced package data
+ * to/from a project's ws data and return the modified workspace data
+ */
+/**
+ * function adds namespaced package data to workspace data and return the modified workspace data
+ *
+ * @param allWorkspaceData
+ * @param packageInfo
+ * @param packageContext
+ * @returns AllWorkspaceData with added namespaced data
+ */
+const GetNamespacedWorkspaceData = (allWorkspaceData, packageInfo, packageContext) => {
+    const { id, version } = packageInfo;
+    const namespacedWorkspaceData = {};
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`Preparing workspace data for import: ${wsKey}`);
+                namespacedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].add(id, version, allWorkspaceData[wsKey]);
+            }
+        }
+        catch (error) {
+            const message = `error when preparing workspace data for import: ${wsKey}`;
+            if (error instanceof Error) {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error.message, error.name, error.stack, error);
+            }
+            else {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error);
+            }
+            throw error;
+        }
+    });
+    return namespacedWorkspaceData;
+};
+/**
+ * will remove namespaced package workspace data from allWorkspaceData and return the updated object
+ *
+ * @param allWorkspaceData
+ * @param packageInfo
+ * @param packageContext
+ * @returns AllWorkspaceData with added namespaced data
+ */
+const RemoveNamespacedWorkspaceData = (allWorkspaceData, packageInfo, packageContext) => {
+    const prunedWorkspaceData = {};
+    const { id } = packageInfo;
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            // if included in entities that should be imported, then add with namespace
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`Pruning workspace data: ${wsKey}`);
+                prunedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].remove(id, allWorkspaceData[wsKey]);
+            }
+            else {
+                // else just add back the original data
+                prunedWorkspaceData[wsKey] = allWorkspaceData[wsKey];
+            }
+        }
+        catch (error) {
+            const message = `error when pruning workspace data for: ${wsKey}`;
+            if (error instanceof Error) {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error.message, error.name, error.stack, error);
+            }
+            else {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error);
+            }
+            throw error;
+        }
+    });
+    return prunedWorkspaceData;
+};
+/**
+ * Takes a parentEntityObject (ie 'stepTypes') that contains nested entity objects (ie 'stepType-1') that will be namespaced
+ *     a) all nested entity object keys will be namespaced
+ *     b) all nested entity object field values (fieldsToUpdate) will be namespaced
+ *     c) (optional) additional work can be done on the newly namespaced nested entity object using the nestedObjectUpdateCB
+ *
+ * @param parentEntityObject object containing nested entity objects which will be iterated through and namespaced
+ * @param packageID used as new namespace
+ * @param fieldsToUpdate array of fields whose values will be namespaced for each nested entity object, or null to update all fields
+ * @param nestedObjectUpdateCB (optional) any additional work that should be done to namespaced nested entity object
+ * @param entity the entity (singular) being updated, ie. "step type" or "folder", etc. used for info logger
+ * @returns
+ */
+const NamespaceEntity = (parentEntityObject, packageID, packageVersion, fieldsToUpdate, nestedObjectUpdateCB, entity) => {
+    const namespacedDataObject = {};
+    Object.keys(parentEntityObject).forEach(key => {
+        const isPackageImportedObject = !!Shared.Common.getValueSafe(parentEntityObject[key], ["packageInfo", "id"], null);
+        if (isPackageImportedObject)
+            return; // do not namespace already namespaced data
+        const namespacedKey = UpdateValueWithNamespace(key, packageID);
+        PackagesLogger.info(`Adding ${entity}: ${key} as ${namespacedKey}`);
+        namespacedDataObject[namespacedKey] = Object.assign({}, EntityNamespacer(parentEntityObject[key], fieldsToUpdate, (fieldValueToUpdate) => UpdateValueWithNamespace(fieldValueToUpdate, packageID)));
+        // do additional work if needed on newly namespaced nested object
+        if (nestedObjectUpdateCB) {
+            nestedObjectUpdateCB(namespacedDataObject[namespacedKey], key);
+        }
+        // add package information to top level of entity
+        namespacedDataObject[namespacedKey]["packageInfo"] = BuildPackageInfoForEntity(packageID, packageVersion);
+    });
+    return namespacedDataObject;
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Namespacing function dictionary
+////////////////////////////////////////////////////////////////////
+// to start packages will support import/removal of macros and steptypes
+const WorkspaceEntityNamespaceFunctionLookup = {
+    macros: {
+        add: (packageID, packageVersion, macros) => MacrosNamespacer(packageID, macros, packageVersion),
+        remove: (packageID, macros) => MacrosNamespaceRemover(packageID, macros)
+    },
+    stepTypes: {
+        add: (packageID, packageVersion, stepTypes) => StepTypesNamespacer(packageID, stepTypes, packageVersion),
+        remove: (packageID, stepTypes) => StepTypesNamespaceRemover(packageID, stepTypes)
+    },
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Namespacing functions by entity (adding)
+////////////////////////////////////////////////////////////////////
+const MacrosNamespacer = (packageID, macros, packageVersion) => {
+    return NamespaceEntity(macros, packageID, packageVersion, ["id"], null, "macro");
+};
+const StepTypesNamespacer = (packageID, stepTypes, packageVersion) => {
+    const filteredStepTypes = Object.keys(stepTypes)
+        .filter(stepTypeID => !Shared.StepTypes.isDefaultStepType(stepTypeID)) // remove default steptypes
+        .reduce((acc, id) => {
+        acc[id] = stepTypes[id];
+        return acc;
+    }, {});
+    const updateStepTypeCB = (namespacedStepTypeData) => {
+        namespacedStepTypeData.metadata["defaultStorageLocation"] = null;
+    };
+    return NamespaceEntity(filteredStepTypes, packageID, packageVersion, ["id"], updateStepTypeCB, "stepType");
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Namespace Remover functions by entity (remove)
+////////////////////////////////////////////////////////////////////
+const EntityRemover = (packageID, parentEntityObject, entityName) => {
+    const prunedEntityObject = {};
+    Object.keys(parentEntityObject)
+        .filter(entityKey => {
+        const entityPackageInfo = Shared.Common.getValueSafe(parentEntityObject[entityKey], ["packageInfo"], null);
+        // if no info, then not imported, if info doesn't match, then not imported from this package
+        if (!entityPackageInfo || (!!entityPackageInfo && entityPackageInfo.id !== packageID)) {
+            return entityKey;
+        }
+        else {
+            PackagesLogger.info(`Removing ${entityName}: ${entityKey}`);
+        }
+    })
+        .forEach(prunedEntityKey => prunedEntityObject[prunedEntityKey] = parentEntityObject[prunedEntityKey]);
+    return prunedEntityObject;
+};
+const MacrosNamespaceRemover = (packageID, macros) => {
+    return EntityRemover(packageID, macros, "macro");
+};
+const StepTypesNamespaceRemover = (packageID, stepTypes) => {
+    return EntityRemover(packageID, stepTypes, "step type");
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Package/Dependency getters/builders
+////////////////////////////////////////////////////////////////////
+const BuildPackageInfoForEntity = (packageID, version) => {
+    return {
+        id: packageID,
+        version
+    };
+};
+const InitializePackageDependencyInfoForPackage = (id, versionInfo, version, userID) => {
+    return {
+        id,
+        version,
+        versionInfo,
+        status: EPackageStatus.adding,
+        createdAt: null,
+        addedBy: userID
+    };
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Collection functions 
+////////////////////////////////////////////////////////////////////
+const GetRepoNameFromURL = (repoURL) => {
+    try {
+        const nameIndex = repoURL.lastIndexOf("/") + 1;
+        const gitIndex = repoURL.lastIndexOf(".git");
+        return repoURL.substring(nameIndex, gitIndex);
+    }
+    catch (error) {
+        PackagesLogger.error(`Error when reading package dependency URL: ${repoURL}`, error);
+        throw error;
+    }
+};
+const GetAllWorkspaceDataAndCommitHashFromRemoteRepository = (location) => {
+    return new Promise((resolve, reject) => {
+        // ensure a unique temp folder name
+        const tempRepoFolder = `COA_TEMP_${GetRepoNameFromURL(location)}_${Math.random()}`;
+        const tempFolderPath = os.tmpdir();
+        let commitHashToReturn;
+        let commitOrBranch = null;
+        let locationToUse = location;
+        const indexOfHash = location.indexOf("#");
+        // if hash indicator, then grab commit and remove #info from URL
+        if (indexOfHash > -1) {
+            commitOrBranch = location.slice(indexOfHash + 1);
+            locationToUse = location.substring(0, indexOfHash);
+        }
+        const toExecute = commitOrBranch ? `&& git checkout ${commitOrBranch} && git log -n 1 ${commitOrBranch}` : `&& git log -n 1`;
+        PackagesLogger.info(`Cloning Coalesce package from repo URL: ${locationToUse}`);
+        return Shared.Common.executeCommand(`cd ${tempFolderPath} && git clone ${locationToUse} ${tempRepoFolder} && ls && cd ${tempRepoFolder} && ls ${toExecute}`)
+            .then(res => {
+            const fileSystemSettings = { fs: fs_1.default, dir: `${tempFolderPath}/${tempRepoFolder}` };
+            const getCommitFromGitLogRegex = new RegExp(/commit ([\s\S]*?)\n/);
+            const regexResult = getCommitFromGitLogRegex.exec(res[0]);
+            if (!regexResult) {
+                const message = `No commit was found for ${commitOrBranch}`;
+                throw new Error(message);
+            }
+            commitHashToReturn = regexResult[1];
+            return GetAllWorkspaceDataFromFileSystem(fileSystemSettings);
+        })
+            .then((workspaceData) => {
+            const additionalLogInfo = commitOrBranch ? ` at commit or branch: "${commitOrBranch}"` : "";
+            PackagesLogger.info(`Reading data from Coalesce package ${workspaceData.projectName || GetRepoNameFromURL(locationToUse)}${additionalLogInfo}...`);
+            resolve({
+                workspaceData,
+                hashFromRemote: commitHashToReturn
+            });
+        })
+            .catch(error => {
+            PackagesLogger.error(`Error while collecting workspace data for repo ${GetRepoNameFromURL(locationToUse)}`, error);
+            reject(error);
+        });
+    });
+};
+const GetAllWorkspaceDataFromFileSystem = (fileSystemSettings) => {
+    return Shared.Workspaces.GetAllWorkspaceDataFromGitCommit(fileSystemSettings, null, null);
+};
+////////////////////////////////////////////////////////////////////
+///////////////// Other helpers
+////////////////////////////////////////////////////////////////////
+/**
+ * this function will remove any ws data associated with packageInfo from projectWorkspaceData
+ * before reinstalling to ensure up-to-date package information
+ *
+ * @param packageInfo
+ * @param packageWorkspaceData
+ * @param projectWorkspaceData
+ * @param packageContext
+ * @returns {
+ *      desiredState: what FS should look like
+ *      stateToReplace: what FS looks like now
+ * }
+ */
+const GetNamespacedDataAndDesiredStateForFSFlush = (packageInfo, packageWorkspaceData, projectWorkspaceData, packageContext) => {
+    if (!packageInfo.id) {
+        const message = `PackageID was ${packageInfo.id} when it should be a string`;
+        Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message);
+        throw new Error(message);
+    }
+    PackagesLogger.info(`Cleaning up any stale package data in your project...`);
+    // remove entities that were installed by current package
+    const prunedProjectWorkspaceData = RemoveNamespacedWorkspaceData(projectWorkspaceData, packageInfo, packageContext);
+    // get namespaced data for current package
+    const namespacedDataFromPackage = GetNamespacedWorkspaceData(packageWorkspaceData, packageInfo, packageContext);
+    // add namespaced data to project data
+    const projectWorkspaceDataWithUpdates = CombineWorkspaceDataByEntities(namespacedDataFromPackage, prunedProjectWorkspaceData);
+    return {
+        desiredState: projectWorkspaceDataWithUpdates,
+        namespacedData: namespacedDataFromPackage
+    };
+};
+// get a package context object utilized by much of the functionality in this file
+const InitializePackageContext = (authInfo, environmentID) => {
+    return {
+        firestore: authInfo.firebase.firestore(),
+        timestamp: Shared.Firebase.serverTimestamp,
+        teamID: authInfo.teamInfo.fbTeamID,
+        environmentID,
+        loggerContext: { orgID: authInfo.teamInfo.fbTeamID, userID: authInfo.teamInfo.fbUserID }
+    };
+};
+exports.InitializePackageContext = InitializePackageContext;
+const CombineWorkspaceDataByEntities = (namespacedWorkspaceData, projectWorkspaceData) => {
+    let updatedProjectWorkspaceData = {};
+    Object.keys(projectWorkspaceData).forEach(entityKey => {
+        updatedProjectWorkspaceData[entityKey] = projectWorkspaceData[entityKey];
+    });
+    Object.keys(namespacedWorkspaceData).forEach(entityKey => {
+        updatedProjectWorkspaceData[entityKey] = Object.assign(Object.assign({}, updatedProjectWorkspaceData[entityKey]), namespacedWorkspaceData[entityKey]);
+    });
+    return updatedProjectWorkspaceData;
+};
+const InitializeVersionInfo = (commitOrBranch, repoLocation) => {
+    if (!!commitOrBranch) {
+        return {
+            commit: commitOrBranch,
+        };
+    }
+    else {
+        return {
+            filePath: repoLocation,
+        };
+    }
+};
+const BuildManifest = (workspaceData) => {
+    const manifest = {};
+    const getEntityNameFromPackageManifest = (entityType, entityID) => {
+        return Shared.Common.getValueSafe(workspaceData, [entityType, entityID, "name"], entityID);
+    };
+    Object.keys(workspaceData).forEach(entityType => {
+        const entityImportedData = Object.keys(workspaceData[entityType]);
+        manifest[entityType] = {};
+        entityImportedData.forEach(namedEntityKey => {
+            debugger;
+            manifest[entityType][namedEntityKey] = {
+                id: namedEntityKey,
+                name: getEntityNameFromPackageManifest(entityType, namedEntityKey)
+            };
+        });
+    });
+    return manifest;
+};
+////////////////////////////////////////////////////
+/////////////  Firestore read/write funcs
+////////////////////////////////////////////////////
+const GetPackageDocRefAdmin = (packageInfo, packageContext) => {
+    const { environmentID, teamID, firestore } = packageContext;
+    return Shared.CommonOperations.getOrgConfigDocRefAdmin(firestore, teamID).collection("workspaces").doc(environmentID.toString()).collection("packages").doc(packageInfo.id);
+};
+const UpdatePackageInformationFS = (packageInfo, status, packageContext, setTimestamp) => {
+    // example: setting timestamp once package has completed install
+    const createdAt = setTimestamp ? packageContext.timestamp : packageInfo.createdAt;
+    const update = Object.assign({}, Object.assign(Object.assign({}, packageInfo), { status, createdAt }));
+    return GetPackageDocRefAdmin(packageInfo, packageContext).set(update)
+        .then(() => {
+        return { status, id: packageInfo.id };
+    });
+};
+const RemovePackageInformationFS = (packageInfo, packageContext) => {
+    const { environmentID, teamID, firestore } = packageContext;
+    return Shared.CommonOperations.getOrgConfigDocRefAdmin(firestore, teamID).collection("workspaces").doc(environmentID.toString()).collection("packages").doc(packageInfo.id).delete();
+};
+const GetPackageListMessageCLI = (packageContext) => {
+    const { environmentID, teamID, firestore } = packageContext;
+    return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+        .then(wsData => {
+        const packages = Shared.Common.getValueSafe(wsData, ["packages"], null);
+        if (packages && Object.keys(packages).length) {
+            const listForCLI = Object.keys(packages).map(packageID => `${packageID} \n`);
+            listForCLI.unshift(`\nPackages added to environment ${environmentID}: \n`);
+            return listForCLI.join("");
+        }
+        else {
+            return `No packages added to environment ${environmentID}`;
+        }
+    });
+};
+exports.GetPackageListMessageCLI = GetPackageListMessageCLI;
+//////////////////////////////////////////////////////////
+///////////// install/remove kickoff functions and classes
+//////////////////////////////////////////////////////////
+class PackageProvider {
+    constructor(dirOrURL, loggerContext) {
+        this.version = dirOrURL;
+        this.loggerContext = loggerContext;
+        // these vars will be null until getWorkspaceDataFS is called
+        this.commitHash = null;
+        this.id = null;
+        this.workspaceData = null;
+        this.versionInfo = null;
+        // init logic
+        if (!(typeof this.version === "string")) {
+            const message = `Location should be a string, but is of type ${typeof this.version}`;
+            PackagesLogger.errorContext(this.loggerContext, message);
+            throw new Error(`Invalid Package package location: ${message}`);
+        }
+    }
+    _getVersionType() {
+        const fileKeywordIndex = this.version.indexOf(exports.localInstallKeyword);
+        const httpIndex = this.version.indexOf("http");
+        if (fileKeywordIndex === 0) {
+            return EVersionType.file;
+        }
+        else if (httpIndex === 0) {
+            return EVersionType.url;
+        }
+        else {
+            const message = `Error with URL or filepath: "${this.version}" - filepath missing prefix "filepath:" or URL missing "http://" or "https://"`;
+            PackagesLogger.errorContext(this.loggerContext, message);
+            throw new Error(message);
+        }
+    }
+    _isProjectNameValid(name) {
+        let isValid = true;
+        const indexOfNamespace = name.indexOf("::");
+        // projectName should not be only whitespace
+        if (indexOfNamespace > -1) {
+            isValid = false;
+        }
+        return isValid;
+    }
+    getWorkspaceDataFS() {
+        const versionType = this._getVersionType();
+        let wsDataFunc;
+        switch (versionType) {
+            case EVersionType.file: {
+                const dirWithoutKeyword = this.version.substring(exports.localInstallKeyword.length);
+                const fsSettings = { fs: fs_1.default, dir: dirWithoutKeyword };
+                this.commitHash = null;
+                wsDataFunc = GetAllWorkspaceDataFromFileSystem(fsSettings);
+                break;
+            }
+            case EVersionType.url: {
+                wsDataFunc = GetAllWorkspaceDataAndCommitHashFromRemoteRepository(this.version)
+                    .then((response) => {
+                    const { workspaceData, hashFromRemote } = response;
+                    this.commitHash = hashFromRemote;
+                    return workspaceData;
+                });
+                break;
+            }
+            default: {
+                const message = `versionType was invariant: ${versionType}`;
+                Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message);
+                throw new Error(message);
+            }
+        }
+        return wsDataFunc
+            .then((workspaceData) => {
+            // ensure package ID
+            if (!workspaceData.projectName) {
+                PackagesLogger.errorContext(this.loggerContext, `Invalid Package: Missing project name`);
+                PackagesLogger.errorContext(this.loggerContext, `Package owner must add a project name in Coalesce git settings before deploying commit for Package`);
+                throw new Error(`Invalid Package: Missing package ID; Package owner must add a package name in Coalesce git settings before deploying commit for Package`);
+            }
+            // validate project name
+            if (!this._isProjectNameValid(workspaceData.projectName)) {
+                PackagesLogger.errorContext(this.loggerContext, `Invalid Package: Project name contains reserved namespacing characters "::"`);
+                throw new Error(`${workspaceData.projectName} contains "::" and is not a valid project name`);
+            }
+            this.id = workspaceData.projectName.toUpperCase(); // should be uppercase
+            this.workspaceData = workspaceData;
+            this.versionInfo = InitializeVersionInfo(this.commitHash, this.version);
+            return this.workspaceData;
+        });
+    }
+}
+exports.PackageProvider = PackageProvider;
+const InstallPackage = (packageProvider, packageContext) => {
+    const { firestore, teamID, environmentID, loggerContext } = packageContext;
+    return new Promise((resolve, reject) => {
+        let projectWorkspaceData;
+        let packageDependencyInfo;
+        let manifest;
+        PackagesLogger.infoContext(loggerContext, `Gathering your project workspace data for environment: ${environmentID}`);
+        return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+            .then((projectWorkspaceDataFromFirestore) => {
+            projectWorkspaceData = projectWorkspaceDataFromFirestore;
+            return packageProvider.getWorkspaceDataFS();
+        })
+            .then(() => {
+            const { id, versionInfo } = packageProvider; // at this point these vars have been defined
+            if (!versionInfo) {
+                const message = `versionInfo was ${versionInfo}`;
+                Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message);
+                throw new Error(message);
+            }
+            PackagesLogger.infoContext(loggerContext, `Installing package using namespace "${id}::"`);
+            packageDependencyInfo = InitializePackageDependencyInfoForPackage(id, versionInfo, packageProvider.version, loggerContext.userID || "unknown");
+            return UpdatePackageInformationFS(packageDependencyInfo, EPackageStatus.adding, packageContext, false);
+        })
+            .then((packageInfoFS) => {
+            const { status, id } = packageInfoFS;
+            PackagesLogger.infoContext(loggerContext, `${status} ${id}...building firestore updates payload...`);
+            const { desiredState, namespacedData } = GetNamespacedDataAndDesiredStateForFSFlush(packageDependencyInfo, packageProvider.workspaceData, projectWorkspaceData, packageContext);
+            manifest = BuildManifest(namespacedData);
+            PackagesLogger.infoContext(loggerContext, "Applying updates to firestore...");
+            return Shared.Workspaces.FlushFirestoreWorkspaceWithAllWorkspaceData(firestore, teamID, environmentID, desiredState, projectWorkspaceData);
+        })
+            .then(() => {
+            return UpdatePackageInformationFS(Object.assign(Object.assign({}, packageDependencyInfo), { manifest }), EPackageStatus.added, packageContext, true);
+        })
+            .then((packageInfoFS) => {
+            const { status, id } = packageInfoFS;
+            PackagesLogger.infoContext(loggerContext, `${status} ${id}...update successful!`);
+            resolve();
+        })
+            .catch(error => {
+            if (!!packageDependencyInfo) {
+                return UpdatePackageInformationFS(packageDependencyInfo, EPackageStatus.error, packageContext, false)
+                    .then((packageInfo) => {
+                    const { status, id } = packageInfo;
+                    PackagesLogger.errorContext(loggerContext, `Update failed for ${id}. Package ${id} status: ${status}, removing package information from project: `, error);
+                    PackagesLogger.errorContext(loggerContext, error);
+                    return (0, exports.UninstallPackage)(id, packageContext);
+                })
+                    .then(() => {
+                    const message = `All data from package ${packageDependencyInfo.id} removed from project after error during installation.`;
+                    PackagesLogger.infoContext(loggerContext, message);
+                    resolve();
+                })
+                    .catch(error => reject(error));
+            }
+            else {
+                PackagesLogger.errorContext(loggerContext, `Update failed when installing package from ${packageProvider.version}`, error);
+                reject(error);
+            }
+        });
+    });
+};
+exports.InstallPackage = InstallPackage;
+const UninstallPackage = (packageID, packageContext) => {
+    const { firestore, teamID, environmentID, loggerContext } = packageContext;
+    return new Promise((resolve, reject) => {
+        let projectWorkspaceData;
+        let prunedWorkspaceData;
+        PackagesLogger.infoContext(loggerContext, `Gathering your project workspace data for environment: ${environmentID}`);
+        return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+            .then((projectWorkspaceDataFromFirestore) => {
+            projectWorkspaceData = projectWorkspaceDataFromFirestore;
+            const allProjectDependencies = Shared.Common.getValueSafe(projectWorkspaceDataFromFirestore, ["packages"], {});
+            const thisDependencyInfo = allProjectDependencies[packageID];
+            if (!thisDependencyInfo) {
+                const depKeys = Object.keys(allProjectDependencies);
+                reject(new Error(`Environment ${environmentID} has ${depKeys.length} packages installed: ${depKeys}. Package "${packageID}" is not installed to this environment.`));
+                return;
+            }
+            PackagesLogger.infoContext(loggerContext, `Removing package workspace data from your project`);
+            prunedWorkspaceData = RemoveNamespacedWorkspaceData(projectWorkspaceData, thisDependencyInfo, packageContext);
+            return UpdatePackageInformationFS(thisDependencyInfo, EPackageStatus.removing, packageContext, false)
+                .then(() => {
+                PackagesLogger.infoContext(loggerContext, `Applying updates to firestore...`);
+                return Shared.Workspaces.FlushFirestoreWorkspaceWithAllWorkspaceData(firestore, teamID, environmentID, prunedWorkspaceData, projectWorkspaceData);
+            })
+                .then(() => {
+                return RemovePackageInformationFS(thisDependencyInfo, packageContext);
+            })
+                .then(() => {
+                PackagesLogger.infoContext(loggerContext, `Removed ${packageID} workspace data from your project`);
+                resolve();
+            })
+                .catch((error) => {
+                if (!!thisDependencyInfo) {
+                    return UpdatePackageInformationFS(thisDependencyInfo, EPackageStatus.error, packageContext, false)
+                        .then(() => {
+                        PackagesLogger.errorContext(loggerContext, `Encountered a problem when removing ${thisDependencyInfo.id}, removal aborted. Please try again.`);
+                        reject(error);
+                    })
+                        .catch(error => reject(error));
+                }
+                else {
+                    PackagesLogger.errorContext(loggerContext, `Update failed when removing package`, error);
+                    reject(error);
+                }
+            });
+        });
+    });
+};
+exports.UninstallPackage = UninstallPackage;
+////////////////////////////////////////////////////
+/////////////  helper Jest test funcs
+////////////////////////////////////////////////////
+const NamespaceWorkspaceData_Testing = (allWorkspaceData, packageInfo) => {
+    const { id, version } = packageInfo;
+    const namespacedWorkspaceData = {};
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`JEST TEST: Preparing workspace data for import: ${wsKey}`);
+                namespacedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].add(id, version, allWorkspaceData[wsKey]);
+            }
+        }
+        catch (error) {
+            const message = `JEST TEST: error when preparing workspace data for import: ${wsKey}`;
+            if (error instanceof Error) {
+                PackagesLogger.error(message, error.message, error.name, error.stack, error);
+            }
+            else {
+                PackagesLogger.error(message, error);
+            }
+            throw error;
+        }
+    });
+    return namespacedWorkspaceData;
+};
+exports.NamespaceWorkspaceData_Testing = NamespaceWorkspaceData_Testing;
+// will remove namespaced package workspace data from allWorkspaceData and return the updated object
+const RemoveNamespacedWorkspaceData_Testing = (allWorkspaceData, packageInfo) => {
+    const prunedWorkspaceData = {};
+    const { id } = packageInfo;
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`JEST TEST: Pruning workspace data: ${wsKey}`);
+                prunedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].remove(id, allWorkspaceData[wsKey]);
+            }
+            else {
+                prunedWorkspaceData[wsKey] = allWorkspaceData[wsKey];
+            }
+        }
+        catch (error) {
+            const message = `JEST TEST: error when pruning workspace data for: ${wsKey}`;
+            if (error instanceof Error) {
+                PackagesLogger.error(message, error.message, error.name, error.stack, error);
+            }
+            else {
+                PackagesLogger.error(message, error);
+            }
+            throw error;
+        }
+    });
+    return prunedWorkspaceData;
+};
+exports.RemoveNamespacedWorkspaceData_Testing = RemoveNamespacedWorkspaceData_Testing;
+const GetPackageSupportedEntityTypes = () => {
+    return Object.keys(WorkspaceEntityNamespaceFunctionLookup);
+};
+exports.GetPackageSupportedEntityTypes = GetPackageSupportedEntityTypes;
+//# sourceMappingURL=Package.js.map

package/bin/Plan.js

@@ -25,12 +25,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CreatePlan = void 0;
 const Shared = __importStar(require("@coalescesoftware/shared"));
 const fs_1 = __importDefault(require("fs"));
-const CreatePlan = (path, environmentID, token, message) => {
+const CreatePlan = (path, environmentID, token, message, runtimeParameters) => {
     const fsSettings = {
         fs: fs_1.default,
         dir: path,
     };
-    return Shared.DeployOperations.CreatePlanCLI(environmentID, fsSettings, token, Shared.Templates.PlatformType.snowflake)
+    return Shared.DeployOperations.CreatePlanCLI(environmentID, fsSettings, token, Shared.Templates.PlatformType.snowflake, runtimeParameters)
         .then(result => {
         const { plan } = result;
         plan.gitInfo = { commit: { message: message }, oid: "" };

package/bin/index.js

@@ -40,7 +40,7 @@ const LogCLIInternal = Shared.Logging.GetLogger(Shared.Logging.LoggingArea.CLI_I
 const program = new commander_1.Command();
 program.version(require("../package.json").version, "-v, --version");
 program.option("-b, --debug", "Output extra debugging", false);
-program.option("--config <coa-config-location>", "coa config file location", CLIProfile.GetDefaultLocationForCoaConfigFile());
+program.option("--config <coa-config-location>", "coa config file location");
 const AddOverridesToCommand = (command) => {
     Object.keys(CLIProfile.ICLIProfileExample).forEach((key) => {
         command.option(`--${key} <value>`, CLIProfile.ICLIProfileExample[key], "");
@@ -94,10 +94,17 @@ AddOverridesToCommand(program.command(Shared.CLIOperations.ECLICommands.Plan))
         .then((configResult) => {
         config = configResult;
         return Shared.SchedulerOperations.AuthenticateFirebaseTokenAndRetrieveTeamInfoForCLI(config.token, undefined);
-    }).then(({ teamInfo }) => {
+    })
+        .then(({ teamInfo, firebase }) => {
+        const firestore = firebase.firestore();
         Shared.Logging.SetBackendGlobalContext({ userID: teamInfo.fbUserID, orgID: teamInfo.fbTeamID });
+        const teamID = teamInfo.fbTeamID;
+        const environmentID = config.runDetails.environmentID;
+        return Shared.SchedulerOperations.GetRuntimeParametersObjectForRun(firestore, teamID, environmentID, config.runtimeParameters);
+    })
+        .then((runtimeParametersObject) => {
         const envID = config.runDetails.environmentID;
-        return Plan.CreatePlan(directoryPath, envID, config.token, `Deploy to environment: ${envID}`);
+        return Plan.CreatePlan(directoryPath, envID, config.token, `Deploy to environment: ${envID}`, JSON.stringify(runtimeParametersObject));
     }).then(result => {
         const { plan, logContext } = result;
         logContextToUse = logContext;
@@ -155,7 +162,7 @@ AddOverridesToCommand(program.command(Shared.CLIOperations.ECLICommands.Deploy))
                 environmentID: config.runDetails.environmentID
             },
             userCredentials: config.userCredentials,
-            runTimeParameters: {}
+            runTimeParameters: plan.runtimeParameters || {},
         };
         Shared.Common.assert(Shared.Logging.LoggingArea.CLI, !!runInfo, "runinfo must exist to continue");
         return Deploy.DeployWithCLI(plan, runInfo, config.token)
@@ -212,8 +219,14 @@ AddOverridesToCommand(program.command(Shared.CLIOperations.ECLICommands.Refresh)
         config = configResult;
         return Shared.SchedulerOperations.AuthenticateFirebaseTokenAndRetrieveTeamInfoForCLI(config.token, undefined);
     })
-        .then(({ teamInfo }) => {
+        .then(({ teamInfo, firebase }) => {
+        const firestore = firebase.firestore();
         Shared.Logging.SetBackendGlobalContext({ userID: teamInfo.fbUserID, orgID: teamInfo.fbTeamID });
+        const teamID = teamInfo.fbTeamID;
+        const environmentID = config.runDetails.environmentID;
+        return Shared.SchedulerOperations.GetRuntimeParametersObjectForRun(firestore, teamID, environmentID, config.runtimeParameters);
+    })
+        .then((runtimeParametersObject) => {
         runInfo = {
             runType: Shared.Runner.ERunType.refresh,
             runStatus: Shared.Runner.ERunStatus.running,
@@ -224,12 +237,12 @@ AddOverridesToCommand(program.command(Shared.CLIOperations.ECLICommands.Refresh)
                 excludeNodesSelector: config.runDetails.excludeNodesSelector
             },
             userCredentials: config.userCredentials,
-            runTimeParameters: {}
+            runTimeParameters: runtimeParametersObject,
         };
         Shared.Common.CleanupUndefinedValuesFromObject(runInfo);
         LogCLIInternal.infoContext(logContextToUse, "cliOverrides", cliOverrides);
-        LogCLIInternal.infoContext(logContextToUse, "got the cli config", config);
-        LogCLIInternal.infoContext(logContextToUse, "got the run info", config);
+        //LogCLIInternal.infoContext(logContextToUse, "got the cli config", config)
+        //LogCLIInternal.infoContext(logContextToUse, "got the run info", config)
         Shared.Common.assert(Shared.Logging.LoggingArea.CLI, !!runInfo, "runinfo must exist to continue");
         return Refresh.RefreshWithCLI(config.token, runInfo)
             .then((result) => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@coalescesoftware/coa",
-    "version": "1.0.113",
+    "version": "1.0.114",
     "license": "ISC",
     "author": "Coalesce Automation, Inc.",
     "main": "index.js",
@@ -12,7 +12,7 @@
         "start-cli-debug": "yarn run start --debug"
     },
     "dependencies": {
-        "@coalescesoftware/shared": "^1.0.113",
+        "@coalescesoftware/shared": "^1.0.114",
         "chalk": "^4.1.2",
         "commander": "^9.2.0",
         "firebase": "8.2.0",