@coalescesoftware/coa 1.0.119 → 1.0.120

package/src/Package.ts

@@ -0,0 +1,1018 @@
+/* eslint-disable multiline-comment-style */
+import * as Shared from "@coalescesoftware/shared"
+import fs from "fs"
+
+const os = require("os");
+
+/**
+ * coa package uses the same config file that should be setup to leverage coa.
+ * 
+ * project - the user's ws data
+ * package - the targetted repo being applied to the project
+ * dependency - any package installed to a project/another package 
+ * 
+ * coa add works at a high level in these steps: 
+ * 1 - collects ws data from a targetted repo (local or remote)
+ * 2 - namespaces that ws data w/package ID
+ * 3 - adds to firestore packages collection the info for the package being installed (dependency information) w/status: 'adding'
+ * 4 - updates project ws data on firestore with namespaced ws data from package 
+ * 5 - updates firestore package dependency information w/status: 'added' and a timestamp
+ * 6 - in case of error, will clean up any added package information including dependency info from project ws data on FS
+ * 
+ * coa remove works similarly
+ */
+
+export const localInstallKeyword = "file:"
+const PackagesLogger = Shared.Logging.GetLogger(Shared.Logging.LoggingArea.CLI)
+
+type TPackageIdentifier = string // the ID of the package (currently comes from project name assigned in the git settings)
+type TPackageVersion = string // for now version is a URL for the git repo
+
+type TPackageGitCommitInfo = {
+    commit: string
+}
+
+type TPackageFileInfo = {
+    filePath: TPackageVersion
+}
+
+export type TVersionInfo = TPackageGitCommitInfo | TPackageFileInfo
+
+interface IEntityPackageInformation { // added to entity to track versioning under entity.packageInfo
+    id: TPackageIdentifier,
+    version: TPackageVersion,
+}
+
+interface IPackageInformation { // added to workspace as own entity in workspaceData.packages
+    id: TPackageIdentifier,
+    status: EPackageStatus,
+    addedBy: string, // fbTeam userID
+    createdAt: firebase.default.firestore.Timestamp | null, // should be timestamped by firstore
+    version: TPackageVersion,
+    versionInfo: TVersionInfo,
+    manifest?: Shared.Workspaces.AllWorkspaceData // added on success of firestore flush - all entities that have been added by the package
+}
+
+// now a collection in AllWorkspaceData
+export interface IPackages {
+    [packageID: string]: IPackageInformation
+}
+
+enum EPackageStatus {
+    adding = "adding",
+    added = "added",
+    removing = "removing",
+    error = "error",
+}
+
+export interface IPackageContext {
+    firestore
+    teamID: string
+    environmentID: number
+    loggerContext: Shared.Logging.LogContext
+    timestamp // a firestore timestamp function called on firestore at time of successful install
+}
+
+type TNamespacedEntityData = 
+    Shared.Runner.IBSSteps | 
+    Shared.State.Workspaces | 
+    Shared.FolderOperations.FoldersData | 
+    Shared.JobOperations.JobsData | 
+    Shared.MacroOperations.MacrosData | 
+    Shared.StepTypes.AllStepTypesParsed | 
+    Shared.MetadataOperations.ILocations
+
+interface INamespacerDictionary {
+    [entity: string]: {
+        add: (packageID: TPackageIdentifier, packageURL: string, ...args: any) => TNamespacedEntityData
+        remove: (packageID: TPackageIdentifier, ...args: any) => TNamespacedEntityData
+    }
+}
+
+enum EVersionType {
+    file = "file",
+    url = "url"
+}
+
+////////////////////////////////////////////////////////////////////
+//////////////////// reusable namespacing helper functions
+////////////////////////////////////////////////////////////////////
+
+/**
+ * Recieves a string or a number that will be namespaced (NAMESPACEVALUE::STRINGorNUMBER)
+ * @param value 
+ * @param namespace 
+ * @returns 
+ */
+const UpdateValueWithNamespace = (value: string | number, namespace: TPackageIdentifier): string => {
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, typeof namespace === "string", `Namespace was not a string, but was type ${typeof namespace}`)
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, !!namespace.trim(), `Namespace cannot be an empty string`)
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, typeof value === "string" || typeof value === "number", `Value was not a string, but was type ${typeof value}`)
+    const stringValue = String(value)
+
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, stringValue.indexOf("::") === -1, `value to namespace contained invalid character ":"`)
+    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, namespace.indexOf("::") === -1, `namespace contained invalid character ":"`)
+
+    if (stringValue.trim() === "") return stringValue //empty string should not be namespaced but returned as is
+
+    return `${namespace}::${value}`
+}
+
+/**
+ * Updates originalObject field values (fieldsToUpdate) using fieldValueUpdateCB and returns original value for non selected fields
+ * 
+ * @param entity object with field values to update
+ * @param entityFieldsToUpdate an array of fields to update, or null to update all fields
+ * @param namespacerCB CB to call on each field to update
+ * @returns entity with field values that have been namespaced or updated with CB
+ */
+const EntityNamespacer = (
+    entity: any, 
+    entityFieldsToUpdate: (string | number)[] | null, 
+    namespacerCB: (fieldValueToUpdate) => any
+): any => {
+    const updatedEntity = { ...entity }
+
+    if (!entityFieldsToUpdate) {
+        // update all
+        Object.keys(entity).forEach(field => {
+            updatedEntity[field] = namespacerCB(updatedEntity[field])
+        })
+    } else {
+        Object.keys(entity).forEach(key => {
+            // if key included in keypath array, update, otherwise return original value
+            if (entityFieldsToUpdate.includes(key)) {
+                updatedEntity[key] = namespacerCB(updatedEntity[key])
+            } else {
+                updatedEntity[key] = entity[key]
+            }
+        })
+    }
+
+    return updatedEntity
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Parent namespacing functions
+////////////////////////////////////////////////////////////////////
+/**
+ * internal functions that will add or remove namespaced package data
+ * to/from a project's ws data and return the modified workspace data
+ */
+
+/**
+ * function adds namespaced package data to workspace data and return the modified workspace data
+ * 
+ * @param allWorkspaceData 
+ * @param packageInfo 
+ * @param packageContext 
+ * @returns AllWorkspaceData with added namespaced data
+ */
+const GetNamespacedWorkspaceData = (
+	allWorkspaceData: Shared.Workspaces.AllWorkspaceData, 
+	packageInfo: IPackageInformation,
+    packageContext: IPackageContext
+): Shared.Workspaces.AllWorkspaceData => { 
+    const { id, version } = packageInfo
+    const namespacedWorkspaceData = {}
+
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`Preparing workspace data for import: ${wsKey}`)
+                namespacedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].add(id, version, allWorkspaceData[wsKey])
+            }
+        } catch (error) {
+            const message = `error when preparing workspace data for import: ${wsKey}`
+
+            if (error instanceof Error) {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error.message, error.name, error.stack, error)   
+            } else {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error)
+            }
+
+            throw error
+        }
+    })
+
+    return namespacedWorkspaceData as Shared.Workspaces.AllWorkspaceData
+}
+
+/**
+ * will remove namespaced package workspace data from allWorkspaceData and return the updated object
+ * 
+ * @param allWorkspaceData 
+ * @param packageInfo 
+ * @param packageContext 
+ * @returns AllWorkspaceData with added namespaced data
+ */
+const RemoveNamespacedWorkspaceData = (
+	allWorkspaceData: Shared.Workspaces.AllWorkspaceData, 
+	packageInfo: IPackageInformation,
+    packageContext: IPackageContext
+): Shared.Workspaces.AllWorkspaceData => {
+    const prunedWorkspaceData = {}
+    const { id } = packageInfo
+
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            // if included in entities that should be imported, then add with namespace
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`Pruning workspace data: ${wsKey}`)
+                prunedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].remove(id, allWorkspaceData[wsKey])
+            } else {
+            // else just add back the original data
+                prunedWorkspaceData[wsKey] = allWorkspaceData[wsKey]
+            }
+        } catch (error) {
+            const message = `error when pruning workspace data for: ${wsKey}`
+
+            if (error instanceof Error) {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error.message, error.name, error.stack, error)   
+            } else {
+                PackagesLogger.errorContext(packageContext.loggerContext, message, error)
+            }
+            
+            throw error
+        }
+    })
+
+    return prunedWorkspaceData as Shared.Workspaces.AllWorkspaceData
+}
+
+/**
+ * Takes a parentEntityObject (ie 'stepTypes') that contains nested entity objects (ie 'stepType-1') that will be namespaced
+ *     a) all nested entity object keys will be namespaced
+ *     b) all nested entity object field values (fieldsToUpdate) will be namespaced 
+ *     c) (optional) additional work can be done on the newly namespaced nested entity object using the nestedObjectUpdateCB
+ * 
+ * @param parentEntityObject object containing nested entity objects which will be iterated through and namespaced
+ * @param packageID used as new namespace
+ * @param fieldsToUpdate array of fields whose values will be namespaced for each nested entity object, or null to update all fields
+ * @param nestedObjectUpdateCB (optional) any additional work that should be done to namespaced nested entity object
+ * @param entity the entity (singular) being updated, ie. "step type" or "folder", etc. used for info logger
+ * @returns 
+ */
+const NamespaceEntity = (
+    parentEntityObject: any, 
+    packageID: TPackageIdentifier,
+    packageVersion: TPackageVersion, 
+    fieldsToUpdate: string[] | null, 
+    nestedObjectUpdateCB: ((nestedObject: any, key: string) => void) | null, 
+    entity: string
+) => {
+    const namespacedDataObject = {}
+
+    Object.keys(parentEntityObject).forEach(key => {
+        const isPackageImportedObject = !!Shared.Common.getValueSafe(parentEntityObject[key], [ "packageInfo", "id" ], null)
+        
+        if (isPackageImportedObject) return // do not namespace already namespaced data
+
+        const namespacedKey = UpdateValueWithNamespace(key, packageID)
+        PackagesLogger.info(`Adding ${entity}: ${key} as ${namespacedKey}`)
+        
+        namespacedDataObject[namespacedKey] = { 
+            ...EntityNamespacer(
+                parentEntityObject[key], 
+                fieldsToUpdate,
+                (fieldValueToUpdate: string | number) => UpdateValueWithNamespace(fieldValueToUpdate, packageID)
+            ) 
+        }
+        
+        // do additional work if needed on newly namespaced nested object
+        if (nestedObjectUpdateCB) {
+            nestedObjectUpdateCB(namespacedDataObject[namespacedKey], key)
+        }
+
+        // add package information to top level of entity
+        namespacedDataObject[namespacedKey]["packageInfo"] = BuildPackageInfoForEntity(packageID, packageVersion)
+    })
+
+    return namespacedDataObject
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Namespacing function dictionary
+////////////////////////////////////////////////////////////////////
+
+// to start packages will support import/removal of macros and steptypes
+const WorkspaceEntityNamespaceFunctionLookup: INamespacerDictionary = {
+    macros: {
+        add: (
+            packageID: TPackageIdentifier, 
+            packageVersion: TPackageVersion,
+            macros: Shared.MacroOperations.MacrosData
+        ): Shared.MacroOperations.MacrosData => MacrosNamespacer(packageID, macros, packageVersion),
+        remove: (
+            packageID: TPackageIdentifier, 
+            macros: Shared.MacroOperations.MacrosData
+        ): Shared.MacroOperations.MacrosData => MacrosNamespaceRemover(packageID, macros)
+    },
+    stepTypes: {
+        add: (
+            packageID: TPackageIdentifier, 
+            packageVersion: TPackageVersion,
+            stepTypes: Shared.StepTypes.AllStepTypesParsed
+        ): Shared.StepTypes.AllStepTypesParsed => StepTypesNamespacer(packageID, stepTypes, packageVersion),
+        remove: (
+            packageID: TPackageIdentifier, 
+            stepTypes: Shared.StepTypes.AllStepTypesParsed
+        ): Shared.StepTypes.AllStepTypesParsed => StepTypesNamespaceRemover(packageID, stepTypes)
+    },
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Namespacing functions by entity (adding)
+////////////////////////////////////////////////////////////////////
+
+const MacrosNamespacer = (packageID: TPackageIdentifier, macros: Shared.MacroOperations.MacrosData, packageVersion: TPackageVersion): Shared.MacroOperations.MacrosData => {
+    return NamespaceEntity(macros, packageID, packageVersion, [ "id" ], null, "macro")
+}
+
+const StepTypesNamespacer = (packageID: TPackageIdentifier, stepTypes: Shared.StepTypes.AllStepTypesParsed, packageVersion: TPackageVersion): Shared.StepTypes.AllStepTypesParsed => {
+    const filteredStepTypes = Object.keys(stepTypes)
+        .filter(stepTypeID => !Shared.StepTypes.isDefaultStepType(stepTypeID)) // remove default steptypes
+        .reduce((acc, id) => { // turn back into object
+            acc[id] = stepTypes[id]
+            
+            return acc
+        }, {})
+
+    const updateStepTypeCB = (namespacedStepTypeData: Shared.StepTypes.StepTypeParsed) => {
+        namespacedStepTypeData.metadata["defaultStorageLocation"] = null
+    }
+
+    return NamespaceEntity(filteredStepTypes, packageID, packageVersion, [ "id" ], updateStepTypeCB, "stepType")
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Namespace Remover functions by entity (remove)
+////////////////////////////////////////////////////////////////////
+
+const EntityRemover = (packageID: TPackageIdentifier, parentEntityObject: any, entityName: string) => {
+    const prunedEntityObject: any = {}
+
+    Object.keys(parentEntityObject)
+        .filter(entityKey => {
+            const entityPackageInfo: IEntityPackageInformation = Shared.Common.getValueSafe(parentEntityObject[entityKey], [ "packageInfo" ], null)
+            
+            // if no info, then not imported, if info doesn't match, then not imported from this package
+            if (!entityPackageInfo || (!!entityPackageInfo && entityPackageInfo.id !== packageID)) {
+                return entityKey
+            } else {
+                PackagesLogger.info(`Removing ${entityName}: ${entityKey}`)
+            } 
+        })
+        .forEach(prunedEntityKey => prunedEntityObject[prunedEntityKey] = parentEntityObject[prunedEntityKey])
+
+    return prunedEntityObject
+}
+
+const MacrosNamespaceRemover = (packageID: TPackageIdentifier, macros: Shared.MacroOperations.MacrosData): Shared.MacroOperations.MacrosData => {
+    return EntityRemover(packageID, macros, "macro")
+}
+
+const StepTypesNamespaceRemover = (packageID: TPackageIdentifier, stepTypes: Shared.StepTypes.AllStepTypesParsed): Shared.StepTypes.AllStepTypesParsed => {
+    return EntityRemover(packageID, stepTypes, "step type")
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Package/Dependency getters/builders
+////////////////////////////////////////////////////////////////////
+
+const BuildPackageInfoForEntity = (packageID: TPackageIdentifier, version: TPackageVersion): IEntityPackageInformation => {
+    return {
+        id: packageID,
+        version
+    }
+}
+
+const InitializePackageDependencyInfoForPackage = (
+    id: TPackageIdentifier, 
+    versionInfo: TVersionInfo, 
+    version: TPackageVersion,
+    userID: string
+): IPackageInformation => {
+    return {
+        id,
+        version,
+        versionInfo,
+        status: EPackageStatus.adding,
+        createdAt: null, //timestamped on FS
+        addedBy: userID
+    }
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Workspace Data Collection functions 
+////////////////////////////////////////////////////////////////////
+
+const GetRepoNameFromURL = (repoURL: string) => {
+    try {
+        const nameIndex = repoURL.lastIndexOf("/") + 1
+        const gitIndex = repoURL.lastIndexOf(".git")
+        return repoURL.substring(nameIndex, gitIndex)
+    } catch (error) {
+        PackagesLogger.error(`Error when reading package dependency URL: ${repoURL}`, error)
+        throw error
+    }
+}
+
+const GetAllWorkspaceDataAndCommitHashFromRemoteRepository = (location: TPackageVersion): Promise<{ workspaceData: Shared.Workspaces.AllWorkspaceData, hashFromRemote: string | null }> => {    
+    return new Promise((resolve, reject) => {
+        // ensure a unique temp folder name
+        const tempRepoFolder = `COA_TEMP_${GetRepoNameFromURL(location)}_${Math.random()}`
+        const tempFolderPath = os.tmpdir()
+        let commitHashToReturn: string
+
+        let commitOrBranch: string | null = null
+        let locationToUse: string = location
+        
+        const indexOfHash = location.indexOf("#")   
+        // if hash indicator, then grab commit and remove #info from URL
+        if (indexOfHash > -1) {
+            commitOrBranch = location.slice(indexOfHash + 1)
+            locationToUse = location.substring(0, indexOfHash)
+        }
+        
+        const toExecute = commitOrBranch ? `&& git checkout ${commitOrBranch} && git log -n 1 ${commitOrBranch}` : `&& git log -n 1`
+
+        PackagesLogger.info(`Cloning Coalesce package from repo URL: ${locationToUse}`)
+        
+        return Shared.Common.executeCommand(`cd ${tempFolderPath} && git clone ${locationToUse} ${tempRepoFolder} && ls && cd ${tempRepoFolder} && ls ${toExecute}`)
+        .then(res => {
+            const fileSystemSettings: Shared.GitOperations.Filesystem = { fs, dir: `${tempFolderPath}/${tempRepoFolder}` }
+            const getCommitFromGitLogRegex = new RegExp(/commit ([\s\S]*?)\n/)
+            const regexResult = getCommitFromGitLogRegex.exec(res[0])
+            
+            if (!regexResult) {
+                const message = `No commit was found for ${commitOrBranch}`
+                throw new Error(message)
+            } 
+
+            commitHashToReturn = regexResult[1]            
+            
+            return GetAllWorkspaceDataFromFileSystem(fileSystemSettings)
+        })
+        .then((workspaceData) => {
+            const additionalLogInfo = commitOrBranch ? ` at commit or branch: "${commitOrBranch}"` : ""
+            PackagesLogger.info(`Reading Coalesce package data from repo "${GetRepoNameFromURL(locationToUse)}"${additionalLogInfo}...`)
+            
+            resolve({
+                workspaceData,
+                hashFromRemote: commitHashToReturn
+            })
+        })
+        .catch(error => {
+            PackagesLogger.error(`Error while collecting workspace data for repo ${GetRepoNameFromURL(locationToUse)}`, error)
+            reject(error)
+        })
+    })
+}
+
+const GetAllWorkspaceDataFromFileSystem = (fileSystemSettings: Shared.GitOperations.Filesystem) => {
+    return Shared.Workspaces.GetAllWorkspaceDataFromGitCommit(fileSystemSettings, null, null)
+}
+
+////////////////////////////////////////////////////////////////////
+///////////////// Other helpers
+////////////////////////////////////////////////////////////////////
+
+/**
+ * this function will remove any ws data associated with packageInfo from projectWorkspaceData 
+ * before reinstalling to ensure up-to-date package information
+ * 
+ * @param packageInfo 
+ * @param packageWorkspaceData 
+ * @param projectWorkspaceData 
+ * @param packageContext 
+ * @returns {
+ *      desiredState: what FS should look like
+ *      stateToReplace: what FS looks like now
+ * }
+ */
+const GetNamespacedDataAndDesiredStateForFSFlush = (
+    packageInfo: IPackageInformation,
+    packageWorkspaceData: Shared.Workspaces.AllWorkspaceData,
+    projectWorkspaceData: Shared.Workspaces.AllWorkspaceData,
+    packageContext: IPackageContext
+): { desiredState: Shared.Workspaces.AllWorkspaceData, namespacedData: Shared.Workspaces.AllWorkspaceData } => {
+    if (!packageInfo.id) {
+        const message = `PackageID was ${packageInfo.id} when it should be a string`
+        Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message)
+        throw new Error(message)
+    }
+
+    PackagesLogger.info(`Cleaning up any stale package data in your project...`)
+    
+    // remove entities that were installed by current package
+    const prunedProjectWorkspaceData = RemoveNamespacedWorkspaceData(projectWorkspaceData, packageInfo, packageContext)
+    // get namespaced data for current package
+    const namespacedDataFromPackage = GetNamespacedWorkspaceData(packageWorkspaceData, packageInfo, packageContext)
+    // add namespaced data to project data
+    const projectWorkspaceDataWithUpdates = CombineWorkspaceDataByEntities(namespacedDataFromPackage, prunedProjectWorkspaceData)
+
+    return {
+        desiredState: projectWorkspaceDataWithUpdates,
+        namespacedData: namespacedDataFromPackage
+    }
+}
+
+// get a package context object utilized by much of the functionality in this file
+export const InitializePackageContext = (
+    authInfo: Shared.ConnectionOperations.ITeamInfoAndFirebase,
+    environmentID: number
+): IPackageContext => {
+    return {
+        firestore: authInfo.firebase.firestore(),
+        timestamp: Shared.Firebase.serverTimestamp,
+        teamID: authInfo.teamInfo.fbTeamID,
+        environmentID,
+        loggerContext: { orgID: authInfo.teamInfo.fbTeamID, userID: authInfo.teamInfo.fbUserID }
+    }
+}
+
+const CombineWorkspaceDataByEntities = (
+    namespacedWorkspaceData: Shared.Workspaces.AllWorkspaceData, 
+    projectWorkspaceData: Shared.Workspaces.AllWorkspaceData
+): Shared.Workspaces.AllWorkspaceData => {
+    let updatedProjectWorkspaceData = {}
+
+    Object.keys(projectWorkspaceData).forEach(entityKey => {
+        updatedProjectWorkspaceData[entityKey] = projectWorkspaceData[entityKey]
+    })
+
+    Object.keys(namespacedWorkspaceData).forEach(entityKey => {
+        updatedProjectWorkspaceData[entityKey] = { ...updatedProjectWorkspaceData[entityKey], ...namespacedWorkspaceData[entityKey] }
+    })
+
+    return updatedProjectWorkspaceData as Shared.Workspaces.AllWorkspaceData
+}
+
+const InitializeVersionInfo = (commitOrBranch: string | null, repoLocation: string): TVersionInfo => {
+    if (!!commitOrBranch) {
+        return {
+            commit: commitOrBranch,
+        }
+    } else {
+        return {
+            filePath: repoLocation,
+        }
+    }
+}
+
+const BuildManifest = (workspaceData: Shared.Workspaces.AllWorkspaceData): Shared.Workspaces.AllWorkspaceData => {
+    const manifest = {}
+
+    const getEntityNameFromPackageManifest = (entityType: string, entityID: string) => {
+        return Shared.Common.getValueSafe(workspaceData, [ entityType, entityID, "name" ], entityID);
+    }
+
+    Object.keys(workspaceData).forEach(entityType => {
+        const entityImportedData = Object.keys(workspaceData[entityType])
+        
+        manifest[entityType] = {}
+        
+        entityImportedData.forEach(namedEntityKey => {
+            debugger
+            manifest[entityType][namedEntityKey] = {
+                id: namedEntityKey,
+                name: getEntityNameFromPackageManifest(entityType, namedEntityKey)
+            }
+            
+        })
+    })
+
+    return manifest as Shared.Workspaces.AllWorkspaceData
+}
+
+/**
+ * Returns a boolean telling whether the package being installed is already installed - note, does not differentiate by commit or branch, but detects by package name
+ * @param packages 
+ * @param version 
+ */
+export const IsPackageAlreadyInstalled = (installedPackages: IPackages, packageID: string) => {
+    return packageID! in installedPackages
+}
+
+////////////////////////////////////////////////////
+/////////////  Firestore read/write funcs
+////////////////////////////////////////////////////
+
+const GetPackageDocRefAdmin = (
+    packageInfo: IPackageInformation, 
+    packageContext: IPackageContext,
+) => {
+    const { environmentID, teamID, firestore } = packageContext
+
+    return Shared.CommonOperations.getOrgConfigDocRefAdmin(firestore, teamID).collection("workspaces").doc(environmentID.toString()).collection("packages").doc(packageInfo.id)
+}
+
+const UpdatePackageInformationFS = (
+    packageInfo: IPackageInformation, 
+    status: EPackageStatus,
+    packageContext: IPackageContext,
+    setTimestamp: boolean
+) => {
+    // example: setting timestamp once package has completed install
+    const createdAt = setTimestamp ? packageContext.timestamp : packageInfo.createdAt
+    const update = Object.assign({}, { ...packageInfo, status, createdAt }) as IPackageInformation 
+
+    return GetPackageDocRefAdmin(packageInfo, packageContext).set(update)
+        .then(() => {
+            return { status, id: packageInfo.id }
+        })
+}
+
+const RemovePackageInformationFS = (
+    packageInfo: IPackageInformation, 
+    packageContext: IPackageContext
+) => {
+    const { environmentID, teamID, firestore } = packageContext
+
+    return Shared.CommonOperations.getOrgConfigDocRefAdmin(firestore, teamID).collection("workspaces").doc(environmentID.toString()).collection("packages").doc(packageInfo.id).delete()
+}
+
+const FlushFirestoreWorkspaceWithAllWorkspaceDataForCLIPackages = (
+    firestore: firebase.default.firestore.Firestore, 
+    teamID: string, 
+    workspaceIDForTargetFlush: number, 
+    desiredWorkspaceDataState: Shared.Workspaces.AllWorkspaceData, 
+    workspaceDataToReplace: Shared.Workspaces.AllWorkspaceData
+) => {
+    // we do not want to compare packages when installing a package
+    delete desiredWorkspaceDataState.packages
+    delete workspaceDataToReplace.packages
+    
+    return Shared.Workspaces.FlushFirestoreWorkspaceWithAllWorkspaceData(
+        firestore, 
+        teamID, 
+        workspaceIDForTargetFlush, 
+        desiredWorkspaceDataState, 
+        workspaceDataToReplace
+    )
+}
+
+export const GetPackageListMessageCLI = (packageContext: IPackageContext) => {
+    const { environmentID, teamID, firestore } = packageContext
+
+    return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+    .then(wsData => {
+        const packages = Shared.Common.getValueSafe(wsData, [ "packages" ], null)
+
+        if (packages && Object.keys(packages).length) {
+            const listForCLI = Object.keys(packages).map(packageID => `${packageID} \n`)
+            listForCLI.unshift(`\nPackages on workspace ${environmentID}: \n`)
+            return listForCLI.join("")
+        } else {
+            return `No packages added to workspace ${environmentID}`
+        }
+    })
+}
+
+//////////////////////////////////////////////////////////
+///////////// install/remove kickoff functions and classes
+//////////////////////////////////////////////////////////
+
+export class PackageProvider {
+    id: string | null
+    version: TPackageVersion
+    loggerContext: Shared.Logging.LogContext
+    versionInfo: TVersionInfo | null
+    workspaceData: Shared.Workspaces.AllWorkspaceData | null
+    commitHash: string | null
+
+    constructor(dirOrURL: string, loggerContext: Shared.Logging.LogContext) {
+        this.version = dirOrURL
+        this.loggerContext = loggerContext
+        // these vars will be null until getWorkspaceDataFS is called
+        this.commitHash = null
+        this.id = null
+        this.workspaceData = null
+        this.versionInfo = null
+        
+        // init logic
+        if (!(typeof this.version === "string")) {
+            const message = `Location should be a string, but is of type ${typeof this.version}`
+            PackagesLogger.errorContext(this.loggerContext, message)
+                    
+            throw new Error(`Invalid Package package location: ${message}`)
+        }
+    }
+    
+    _getVersionType(): EVersionType {
+        const fileKeywordIndex = this.version.indexOf(localInstallKeyword)
+        const httpIndex = this.version.indexOf("http")
+
+        if (fileKeywordIndex === 0) {
+            return EVersionType.file
+        } else if (httpIndex === 0) {
+            return EVersionType.url
+        } else {
+            const message = `Error with URL or filepath: "${this.version}" - filepath missing prefix "filepath:" or URL missing "http://" or "https://"`
+            PackagesLogger.errorContext(this.loggerContext, message)
+            throw new Error(message)
+        }
+    }
+
+    _isProjectNameValid(name: string): boolean {
+        let isValid = true
+        const indexOfNamespace = name.indexOf("::")
+
+        // projectName should not be only whitespace
+        if (indexOfNamespace > -1) {
+            isValid = false
+        }
+
+        return isValid
+    }
+
+    getWorkspaceDataFS(): Promise<Shared.Workspaces.AllWorkspaceData> {
+        const versionType = this._getVersionType()
+        let wsDataFunc
+
+        switch (versionType) {
+            case EVersionType.file: {
+                const dirWithoutKeyword = this.version.substring(localInstallKeyword.length)
+                const fsSettings: Shared.GitOperations.Filesystem = { fs, dir: dirWithoutKeyword }
+                
+                this.commitHash = null
+                wsDataFunc = GetAllWorkspaceDataFromFileSystem(fsSettings)
+                
+                break
+            }
+            case EVersionType.url: {
+                wsDataFunc = GetAllWorkspaceDataAndCommitHashFromRemoteRepository(this.version)
+                .then((response) => {
+                    const { workspaceData, hashFromRemote } = response
+                    
+                    this.commitHash = hashFromRemote
+                    
+                    return workspaceData
+                })
+                break
+            }
+            default: {
+                const message = `versionType was invariant: ${versionType}`
+                Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message)
+                throw new Error(message)
+            }
+        }
+
+        return wsDataFunc
+            .then((workspaceData: Shared.Workspaces.AllWorkspaceData) => {
+                const projects = workspaceData.projects
+                
+                if (!projects || !Object.keys(projects)) {
+                    PackagesLogger.errorContext(this.loggerContext, `Invalid Package: Missing projects, cannot generate package ID`)
+                    throw new Error(`Invalid Package: Missing projects, cannot generate package ID; Package owner must add a project name in Coalesce project settings and commit`)
+                }
+
+                // ensure package ID to be used as namespace, which is gathered from the default project name as of Lhotse 4.0
+                const projectName = Shared.ProjectOperations.GetDefaultProjectName(projects!)
+
+                if (!projectName) {
+                    PackagesLogger.errorContext(this.loggerContext, `Invalid Package: Missing project name`)
+                    throw new Error(`Invalid Package: Missing package ID; Package owner must add a project name in Coalesce project settings and commit`)
+                } 
+
+                const defaultProject = projects![Shared.ProjectOperations.defaultProjectID] // currently only one default project
+                const isDefaultProjectValid = Shared.ProjectOperations.ValidateProjectAsRunType(defaultProject)
+
+                if (!isDefaultProjectValid.success) {
+                    const errorMessage = `Invalid Package: projects failed Run Type validation - projects["1"]: ${defaultProject}`
+                    PackagesLogger.errorContext(this.loggerContext, errorMessage, isDefaultProjectValid.errorString)
+                    
+                    throw new Error(errorMessage)
+                }
+
+                // validate project name
+                if (!this._isProjectNameValid(projectName)) {
+                    PackagesLogger.errorContext(this.loggerContext, `Invalid Package: Project name contains reserved namespacing characters "::"`)
+
+                    throw new Error(`${projectName} contains "::" and is not a valid project name`)
+                }
+
+                this.id = projectName.toUpperCase() // should be uppercase
+                this.workspaceData = workspaceData
+                this.versionInfo = InitializeVersionInfo(this.commitHash, this.version)
+                
+                return this.workspaceData
+            })
+    }
+}
+
+export const InstallPackage = (
+    packageProvider: PackageProvider,
+    packageContext: IPackageContext
+): Promise<void> => {
+    const { firestore, teamID, environmentID, loggerContext } = packageContext
+
+    return new Promise((resolve, reject) => {
+        let projectWorkspaceData: Shared.Workspaces.AllWorkspaceData
+        let packageDependencyInfo: IPackageInformation
+        let manifest: Shared.Workspaces.AllWorkspaceData
+
+        PackagesLogger.infoContext(loggerContext, `Gathering your project workspace data for environment: ${environmentID}`)
+
+        return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+            .then((projectWorkspaceDataFromFirestore) => {
+                projectWorkspaceData = projectWorkspaceDataFromFirestore
+
+                return packageProvider.getWorkspaceDataFS()
+            })
+            .then(() => {
+                const { id, versionInfo } = packageProvider // at this point these vars have been defined
+                
+                if (!versionInfo) {
+                    const message = `versionInfo was ${versionInfo}`
+                    Shared.Common.assert(Shared.Logging.LoggingArea.Packages, false, message)
+                    throw new Error(message)
+                }
+                
+                PackagesLogger.infoContext(loggerContext, `Installing package using namespace "${id!}::"`)
+
+                packageDependencyInfo = InitializePackageDependencyInfoForPackage(id!, versionInfo, packageProvider.version, loggerContext.userID! || "unknown")
+                
+                return UpdatePackageInformationFS(packageDependencyInfo, EPackageStatus.adding, packageContext, false)
+            })
+            .then((packageInfoFS) => {
+                const { status, id } = packageInfoFS
+
+                PackagesLogger.infoContext(loggerContext, `${status} ${id}...building firestore updates payload...`)
+                const { desiredState, namespacedData } = GetNamespacedDataAndDesiredStateForFSFlush(packageDependencyInfo, packageProvider.workspaceData!, projectWorkspaceData, packageContext)
+                
+                manifest = BuildManifest(namespacedData)
+
+                PackagesLogger.infoContext(loggerContext, "Applying updates to firestore...")
+                return FlushFirestoreWorkspaceWithAllWorkspaceDataForCLIPackages(
+                    firestore,
+                    teamID, 
+                    environmentID,
+                    desiredState,
+                    projectWorkspaceData
+                )
+            })
+            .then(() => {
+                return UpdatePackageInformationFS({ ...packageDependencyInfo, manifest }, EPackageStatus.added, packageContext, true)
+            })
+            .then((packageInfoFS) => {
+                const { status, id } = packageInfoFS
+                PackagesLogger.infoContext(loggerContext, `${status} ${id}...update successful!`)
+                resolve()
+            })
+            .catch(error => {
+                if (!!packageDependencyInfo) {
+                    return UpdatePackageInformationFS(packageDependencyInfo, EPackageStatus.error, packageContext, false)
+                        .then((packageInfo) => {
+                            const { status, id } = packageInfo
+                            PackagesLogger.errorContext(loggerContext, `Update failed for ${id}. Package ${id} status: ${status}, removing package information from project: `, error)
+                            PackagesLogger.errorContext(loggerContext, error)
+                            
+                            return UninstallPackage(id, packageContext)
+                        })
+                        .then(() => {
+                            const message = `All data from package ${packageDependencyInfo.id} removed from project after error during installation.`
+                            PackagesLogger.infoContext(loggerContext, message)
+                            resolve()
+                        })
+                        .catch(error => reject(error))
+                } else {
+                    PackagesLogger.errorContext(loggerContext, `Update failed when installing package from ${packageProvider.version}`, error)
+                    reject(error)
+                }
+            })
+    })
+}
+
+export const UninstallPackage = (
+    packageID: TPackageIdentifier,
+    packageContext: IPackageContext
+): Promise<void> => {
+    const { firestore, teamID, environmentID, loggerContext } = packageContext
+
+    return new Promise((resolve, reject) => {
+        let projectWorkspaceData: Shared.Workspaces.AllWorkspaceData
+        let prunedWorkspaceData: Shared.Workspaces.AllWorkspaceData
+
+        PackagesLogger.infoContext(loggerContext, `Gathering your project workspace data for environment: ${environmentID}`)
+
+        return Shared.Workspaces.getAllWorkspaceDataFromFirebase(firestore, teamID, environmentID)
+            .then((projectWorkspaceDataFromFirestore) => {
+                projectWorkspaceData = projectWorkspaceDataFromFirestore
+                const allProjectDependencies: IPackages = Shared.Common.getValueSafe(projectWorkspaceDataFromFirestore, [ "packages" ], {})
+                const thisDependencyInfo = allProjectDependencies[packageID]
+
+                if (!thisDependencyInfo) {
+                    const depKeys = Object.keys(allProjectDependencies)
+                    reject(new Error(`Environment ${environmentID} has ${depKeys.length} packages installed: ${depKeys}. Package "${packageID}" is not installed to this environment.`))
+                    return
+                }
+
+                PackagesLogger.infoContext(loggerContext, `Removing package workspace data from your project`)
+                prunedWorkspaceData = RemoveNamespacedWorkspaceData(projectWorkspaceData, thisDependencyInfo, packageContext)
+
+                return UpdatePackageInformationFS(thisDependencyInfo, EPackageStatus.removing, packageContext, false)
+            .then(() => {
+                PackagesLogger.infoContext(loggerContext, `Applying updates to firestore...`)
+
+                return FlushFirestoreWorkspaceWithAllWorkspaceDataForCLIPackages(
+                    firestore,
+                    teamID, 
+                    environmentID,
+                    prunedWorkspaceData,
+                    projectWorkspaceData
+                )
+            })
+            .then(() => {
+                return RemovePackageInformationFS(thisDependencyInfo, packageContext)
+            })
+            .then(() => {
+                PackagesLogger.infoContext(loggerContext, `Removed ${packageID} workspace data from your project`)
+                resolve()
+            })
+            .catch((error) => {
+                if (!!thisDependencyInfo) {
+                    return UpdatePackageInformationFS(thisDependencyInfo, EPackageStatus.error, packageContext, false)
+                        .then(() => {
+                            PackagesLogger.errorContext(loggerContext, `Encountered a problem when removing ${thisDependencyInfo.id}, removal aborted. Please try again.`)
+                            reject(error)
+                        })
+                        .catch(error => reject(error))
+                } else {
+                    PackagesLogger.errorContext(loggerContext, `Update failed when removing package`, error)
+                    reject(error)
+                }
+            })
+        })
+    })
+}
+
+////////////////////////////////////////////////////
+/////////////  helper Jest test funcs
+////////////////////////////////////////////////////
+
+export const NamespaceWorkspaceData_Testing = (
+	allWorkspaceData: Shared.Workspaces.AllWorkspaceData, 
+	packageInfo: IPackageInformation,
+): Shared.Workspaces.AllWorkspaceData => { 
+    const { id, version } = packageInfo
+    const namespacedWorkspaceData = {}
+    
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`JEST TEST: Preparing workspace data for import: ${wsKey}`)
+                namespacedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].add(id, version, allWorkspaceData[wsKey])
+            }
+        } catch (error) {
+            const message = `JEST TEST: error when preparing workspace data for import: ${wsKey}`
+
+            if (error instanceof Error) {
+                PackagesLogger.error(message, error.message, error.name, error.stack, error)   
+            } else {
+                PackagesLogger.error(message, error)
+            }
+
+            throw error
+        }
+    })
+
+    return namespacedWorkspaceData as Shared.Workspaces.AllWorkspaceData
+}
+
+// will remove namespaced package workspace data from allWorkspaceData and return the updated object
+export const RemoveNamespacedWorkspaceData_Testing = (
+	allWorkspaceData: Shared.Workspaces.AllWorkspaceData, 
+	packageInfo: IPackageInformation
+): Shared.Workspaces.AllWorkspaceData => {
+    const prunedWorkspaceData = {}
+    const { id } = packageInfo
+
+    Object.keys(allWorkspaceData).forEach(wsKey => {
+        try {
+            if (WorkspaceEntityNamespaceFunctionLookup[wsKey]) {
+                PackagesLogger.info(`JEST TEST: Pruning workspace data: ${wsKey}`)
+                prunedWorkspaceData[wsKey] = WorkspaceEntityNamespaceFunctionLookup[wsKey].remove(id, allWorkspaceData[wsKey])
+            } else {
+                prunedWorkspaceData[wsKey] = allWorkspaceData[wsKey]
+            }
+        } catch (error) {
+            const message = `JEST TEST: error when pruning workspace data for: ${wsKey}`
+
+            if (error instanceof Error) {
+                PackagesLogger.error(message, error.message, error.name, error.stack, error)   
+            } else {
+                PackagesLogger.error(message, error)
+            }
+            
+            throw error
+        }
+    })
+
+    return prunedWorkspaceData as Shared.Workspaces.AllWorkspaceData
+}
+
+export const GetPackageSupportedEntityTypes = (): string[] => {
+    return Object.keys(WorkspaceEntityNamespaceFunctionLookup)
+}