@tremho/mist-lift 1.0.0

package/src/commands/publish.ts

@@ -0,0 +1,350 @@
+
+import {
+    LambdaClient,
+    ListFunctionsCommand, ResourceNotFoundException,
+} from "@aws-sdk/client-lambda";
+import {
+    APIGatewayClient,
+    ImportRestApiCommand,
+    GetRestApisCommand,
+    GetResourcesCommand,
+    DeleteRestApiCommand,
+    Resource,
+    PutIntegrationRequest,
+    PutIntegrationCommand,
+    GetStageCommand,
+    NotFoundException,
+    CreateStageCommand, CreateDeploymentCommand, GetDeploymentsCommand, GetDeploymentCommand
+} from "@aws-sdk/client-api-gateway";
+
+import {getAWSCredentials, getSettings} from "../lib/LiftConfig";
+
+import {doDeployAsync} from "./deploy";
+import * as ac from "ansi-colors"
+
+import {executeCommand} from "../lib/executeCommand";
+
+import path from 'path';
+import fs, {mkdirSync} from 'fs';
+import {all} from "axios";
+import {resolvePaths} from "../lib/pathResolve";
+import {gatherFunctionDefinitions} from "../lib/openAPI/ApiBuildCollector";
+import {delay} from "../lib/utils";
+import {addBuiltInDefinitions, MakeBuiltinApiDoc, MakePublicApiDoc} from "./builtin/ApiDocMaker";
+import {DeployApiBuiltin, DeployRootFileserves, DeployWebrootBuiltIn} from "./builtin/BuiltInHandler";
+import {getProjectName, getProjectVersion} from "../lib/LiftVersion";
+import md5 from 'md5';
+
+let projectPaths:any;
+
+export async function doPublishAsync(stageName?:string): Promise<number>
+{
+    // let retCode = await doDeployAsync([])
+    // if(retCode) return retCode
+    let retCode = 0;
+
+    projectPaths = resolvePaths()
+
+    await publishApi(stageName ?? "Dev")
+
+    return retCode;
+}
+
+
+async function publishApi(stageName:string)
+{
+    if(!stageName) stageName = "Dev"
+    console.log(ac.green.bold(`Publishing Api to ${stageName}`));
+
+    // do the built-in deploys
+    await DeployWebrootBuiltIn();
+    await DeployRootFileserves();
+    await DeployApiBuiltin();
+
+    const projectPaths = resolvePaths();
+
+    // make a private api
+    const fs = require('fs');
+    const os = require('os');
+    const path = require('path');
+
+    let tmpDir;
+    let apiBytes;
+    const appPrefix = 'MistLift';
+    try {
+        tmpDir = path.join(os.tmpdir(), appPrefix);
+        // console.log("making "+tmpDir)
+        mkdirSync(tmpDir, {recursive:true});
+
+        const yamlFile = path.join(tmpDir, 'papi.yaml')
+        await MakeBuiltinApiDoc(yamlFile);
+        // console.log("reading "+yamlFile)
+        apiBytes = fs.readFileSync(yamlFile)
+    }
+    catch (e:any) {
+        console.error(ac.red.bold("Error creating temp staging folder ")+e.message);
+        throw Error();
+    }
+    finally {
+        try {
+            if (tmpDir) {
+                // console.log("Removing "+tmpDir)
+                fs.rmSync(tmpDir, { recursive: true });
+            }
+        }
+        catch (e) {
+            console.error(`An error has occurred while removing the temp folder at ${tmpDir}. Please remove it manually. Error: ${e}`);
+        }
+    }
+
+    const client = new APIGatewayClient(getAWSCredentials());
+
+    await RemoveExistingVersions(client)
+
+    const command = new ImportRestApiCommand({
+        failOnWarnings: false,
+        body: apiBytes
+    })
+    let apiId;
+    try {
+        const response = await client.send(command)
+        const {name, id, rootResourceId, version, warnings} = response
+        console.log(ac.grey(`\n\nAPI ${name} version ${version}  [${id}] schema created`))
+        apiId = id;
+        if (warnings?.length) {
+            console.log(ac.magenta(` with ${warnings.length} warnings`))
+            for (let warning of warnings) {
+                console.log(ac.grey.italic(`    ${warning}`))
+            }
+        }
+    }
+    catch(e:any) {
+        console.error(ac.bold.red(e.message));
+    }
+    var apidoc = new TextDecoder().decode(apiBytes);
+    // console.log(apidoc)
+    if(!apiId) return;
+
+    console.log(ac.grey("Continuing with binding..."))
+    const prereq = await PrequisiteValues(apiId);
+    const intRequests = prereq.MakeRequests();
+    await PutIntegrations(intRequests);
+    await DeployApi(apiId, stageName);
+    const region = getSettings().awsPreferredRegion;
+    console.log(ac.green.bold(`\n Successfully deployed to https://${apiId}.execute-api.${region}.amazonaws.com/${stageName}`));
+}
+function findApiName()
+{
+    const infoFile = path.join(projectPaths.functionPath, "apiService.info.json")
+    const pkgFile =  projectPaths.packagePath;
+    let pkg:any = {};
+    if(fs.existsSync(pkgFile)) { pkg = JSON.parse(fs.readFileSync(pkgFile).toString()) }
+    let info:any = {};
+    if(fs.existsSync(infoFile)) { info = JSON.parse(fs.readFileSync(infoFile).toString()) }
+
+    return info.title ?? pkg.name ?? "API";
+}
+
+function RemoveExistingVersions(client:APIGatewayClient)
+{
+    return new Promise(resolve => {
+        const ourName = findApiName();
+        // ClogDebug("looking for name "+ ourName)
+        const listCommand = new GetRestApisCommand({})
+        const all:any = [];
+        client.send(listCommand).then((response:any) => {
+            // ClogInfo(`There are ${response.items.length} other APIs`)
+
+
+            const DeleteMatchingApis = (i: number = 0):Promise<any> => {
+                if (i >= response.items.length) {
+                    resolve(undefined);
+                    return Promise.resolve(undefined);
+                }
+                const item = response.items[i];
+                if (item.name === ourName) {
+                    // ClogInfo(`Found previous ${ourName}, [${item.id}] -- deleting...`)
+                    const deleteCommand = new DeleteRestApiCommand({
+                        restApiId: item.id
+                    })
+                    return client.send(deleteCommand).then(() => {
+                        delay(5000).then(() => {
+                            console.log(ac.magenta.bold(`Removed previous id ${item.id}`));
+                            return DeleteMatchingApis(++i);
+                        })
+                    })
+                } else {
+                    return DeleteMatchingApis(++i);
+                }
+            }
+            DeleteMatchingApis();
+        });
+    })
+}
+
+
+class FunctionInfo {
+    name: string = ""
+    arn: string = ""
+}
+class APIInfo {
+    id: string = ""
+    parentId: string = ""
+    path:string = ""
+    pathPart: string = "";
+    resourceMethods: object = {}
+}
+
+
+class PrereqInfo {
+    requestApiId:string = ""
+    functions: FunctionInfo[] = []
+    apis: any[] = [] // Resource + method
+    defs: any[] = []
+
+    public constructor() {
+        this.functions = []
+        this.apis = []
+        this.defs = []
+    }
+
+
+    public findApi(pathMap:string, allowedMethods:string):any
+    {
+        // find the pathmap in the apis list
+        // console.log("findApi "+pathMap, allowedMethods)
+        // see that this method exists in method map
+        for(let api of this.apis) {
+            if (api?.path === pathMap) {
+                // console.log("resourceMethods", api.resourceMethods)
+                const methodList = Object.getOwnPropertyNames(api.resourceMethods);
+                // ClogTrace("methodList", methodList)
+                for(let meth of allowedMethods.toUpperCase().split(',')) {
+                    // console.log("meth", meth)
+                    if(methodList.indexOf(meth) !== -1) {
+                        (api as any).method = meth
+                        return api;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    public findARN(name:string):string
+    {
+        for(let f of this.functions ?? []) {
+            let lastus = f.name.lastIndexOf('_');
+            const fname = f.name.substring(0, lastus);
+            if(fname.toLowerCase() === name.toLowerCase()) return f.arn;
+        }
+        console.log("$$$ Couldn't find "+name+" in ", this.functions)
+        return "";
+    }
+
+    public MakeRequests():PutIntegrationRequest[]
+    {
+        const region = getSettings().awsPreferredRegion;
+        const out:PutIntegrationRequest[] = [];
+        for(let d of this.defs) {
+            const def = (d as any);
+            const api = this.findApi(def.pathMap, def.allowedMethods)
+            const arn = this.findARN(def.name)
+            if(!arn) {
+                console.log(`>>> No ARN for ${def.name} ${def.pathMap}, ${api.id}`)
+            }
+            if(api) {
+                out.push({
+                    restApiId: this.requestApiId,
+                    resourceId: api.id, // api.parentId
+                    httpMethod: api.method,
+                    integrationHttpMethod: 'POST', // api.method,
+                    type: "AWS_PROXY",
+                    uri: `arn:aws:apigateway:${region}:lambda:path/2015-03-31/functions/${arn}/invocations`,
+                    credentials: "arn:aws:iam::545650260286:role/TBDLambdaExecution2"
+                })
+            }
+
+        }
+        return out;
+    }
+}
+
+
+async function PrequisiteValues(id:string):Promise<PrereqInfo>
+{
+    const pri = new PrereqInfo()
+    const out = await GetFunctionInfo(pri)
+    out.defs = gatherFunctionDefinitions()
+    addBuiltInDefinitions(out.defs)
+    out.requestApiId = id;
+    await GetMethodResources(id, out);
+
+    return out;
+}
+
+async function GetFunctionInfo(info:PrereqInfo):Promise<PrereqInfo>
+{
+    const client = new LambdaClient(getAWSCredentials());
+    const listCommand:any = new ListFunctionsCommand({})
+    const response:any = await client.send(listCommand);
+
+    const sfx = "_"+md5((getProjectName()??"")+(getProjectVersion()??""))
+
+    for(let func of response.Functions ?? []) {
+        if(func.FunctionName.endsWith(sfx)) {
+            info.functions.push({
+                name: func.FunctionName ?? "",
+                arn: func.FunctionArn ?? ""
+            })
+        }
+    }
+    return info;
+
+}
+
+async function GetMethodResources(id:string, info:PrereqInfo)
+{
+    const client = new APIGatewayClient(getAWSCredentials())
+    const listCommand = new GetResourcesCommand({
+        restApiId: id
+    });
+    const response:any|null = await client.send(listCommand);
+    // ClogInfo("Resources response", response)
+    if(response?.items) info.apis = response.items;
+
+    return info;
+}
+
+async function PutIntegrations(integrations:PutIntegrationRequest[])
+{
+    const client = new APIGatewayClient(getAWSCredentials())
+    for(let input of integrations) {
+
+        try {
+            const command = new PutIntegrationCommand(input);
+            const intResp = await client.send(command);
+            // ClogDebug("integration response", intResp);
+            await delay(5000);
+        } catch(e:any) {
+            console.error("Problem with integration for "+input.uri, {input});
+            throw e
+        }
+    }
+}
+
+async function DeployApi(restApiId:string, stageName:string)
+{
+    try {
+        const client = new APIGatewayClient(getAWSCredentials())
+        const command = new CreateDeploymentCommand({
+            restApiId,
+            stageName
+        })
+        const resp = await client.send(command);
+    }
+    catch(e:any) {
+        console.error(ac.red.bold("Error with deployApi: "+e.message));
+        throw e;
+    }
+}

package/src/commands/settings.ts

@@ -0,0 +1,76 @@
+import path from "path";
+import fs from "fs";
+import {LiftConfig, RuntimeType} from '../lib/LiftConfig'
+import {homedir} from 'os'
+import {ask} from "../lib/askQuestion";
+import * as ac from 'ansi-colors'
+
+
+export async function doSettings()
+{
+
+    try {
+        const mistLiftPath = path.join(homedir(), ".mistlift")
+        let settings:LiftConfig = {cloudHost: "AWS"}
+        if(fs.existsSync(mistLiftPath)) {
+            settings = JSON.parse(fs.readFileSync(mistLiftPath).toString())
+        }
+
+        let ok = false;
+        while(!ok) {
+            settings.cloudHost = ask("Type of cloud host (AWS)",
+                "cloud host",
+                settings.cloudHost
+            )
+            ok = settings.cloudHost == "AWS" // all we support now
+        }
+        ok = false
+
+        if(settings.cloudHost === "AWS") {
+            console.log("Choose from a configured AWS profile, or leave empty to use a custom credentials config file")
+            while (!ok) {
+                settings.awsIniProfile = ask("Name of AWS profile to use",
+                    "AWS Profile",
+                    settings.awsIniProfile??"default"
+                )
+                ok = settings.awsIniProfile !== "";
+            }
+            ok = false;
+            let runtime:string = settings.awsNodeRuntime as string
+            while(!ok) {
+                runtime = ask("AWS Node Runtime Version",
+                    "NodeJS Runtime",
+                    "Nodejs20.x"
+                )
+                ok = !!runtime
+            }
+            settings.awsNodeRuntime = runtime.toLowerCase() as RuntimeType
+
+            ok = false;
+            while(!ok) {
+                settings.awsPreferredRegion = ask("Preferred AWS region",
+                    "preferred AWS region",
+                    settings.awsPreferredRegion??"us-east-1"
+                )
+                ok = !!settings.awsPreferredRegion
+            }
+            ok = false;
+            console.log("Supply the ARN for the service role you have created in your AWS IAM account for Lambda, S3, and Cloudwatch access")
+            while(!ok) {
+                settings.awsServiceRoleARN = ask("AWS service role ARN",
+                    "serviceRole ARN",
+                    settings.awsServiceRoleARN??"arn:aws.iam::xxxxxxxxxxxx:role/service-role-name"
+                )
+                ok = settings.awsServiceRoleARN !== "arn:aws.iam::xxxxxxxxxxxx:role/service-role-name"
+                  && settings.awsServiceRoleARN !== ""
+            }
+        }
+        fs.writeFileSync(mistLiftPath, JSON.stringify(settings))
+
+        return 0;
+    }
+    catch(e:any) {
+        console.error(ac.bold.red("Error with settings:"), e)
+        return -1;
+    }
+}

package/src/commands/start.ts

@@ -0,0 +1,46 @@
+/** Run Express Server */
+import express from 'express'
+import fs from 'fs'
+import path from 'path'
+import bodyParser from 'body-parser'
+
+import functionRouter, {functionBinder} from "../expressRoutes/functionBinder"
+
+import apiRouter from "../expressRoutes/api"
+import allRouter, {allBinder} from "../expressRoutes/all"
+import {gatherFunctionDefinitions} from "../lib/openAPI/ApiBuildCollector";
+import {buildOpenApi} from "../lib/openAPI/openApiConstruction";
+import {resolvePaths} from "../lib/pathResolve";
+
+import * as ac from "ansi-colors"
+
+export function startLocalServer() {
+    const projectPaths = resolvePaths();
+    if(!projectPaths.verified) {
+        console.log(ac.bold.red("Cannot start local server"))
+        console.log(ac.blue.italic("Not in a valid MistLift project directory"))
+        console.log("")
+        return
+    }
+    const port = 8081
+    allBinder()
+    functionBinder()
+    const app = express()
+    // for JSON posts
+    // app.use(bodyParser.json({limit: '50mb'}))
+    app.use(express.json());
+    // for form posts
+    app.use(express.urlencoded({extended: true}))
+
+    app.use('/', functionRouter)
+    app.use('/api', apiRouter)
+    app.use('*', allRouter)
+
+// =========================================
+// Start server
+// http only
+    app.listen(port, function () {
+        console.log('http listening on port ' + port)
+    })
+// ================================================
+}

package/src/commands/test.ts

@@ -0,0 +1,38 @@
+import {executeCommand} from "../lib/executeCommand";
+import {doBuildAsync} from "./build";
+
+export async function doTestAsync(args:string[] ): Promise<number>
+{
+    if(await doBuildAsync(args)) return 1 // don't test if build fails
+    if(args.length === 0) args = ['*']
+    let ret = 0;
+    for (let funcName of args) {
+        const result = await executeCommand('tap', [
+            `build/functions/${funcName}/*-tests/*.js`,
+            /*
+                             - base -- looks a lot like terse
+                             - terse -- pass/fail counts
+                             - min -- errors only
+                             - dot -- like min, but a dot per assert
+                             - silent
+                             - json
+                             - jsonstream
+                             - markdown
+                             - junit
+                             - tap
+             */
+            '--reporter=base',
+            '--color',
+            '--passes',
+            '--allow-empty-coverage',
+            '--allow-incomplete-coverage'
+
+        ], '', true)
+
+        if (result.retcode) {
+            ret = result.retcode
+            break;
+        }
+    }
+    return ret;
+}

package/src/commands/user.ts

@@ -0,0 +1,20 @@
+/** Handles the set of user management commands
+ *
+ * Default and prime user is the default of the AWS profile set
+ * In configuration, this can be changed to a different profile
+ * The user command module is meant to
+ */
+
+// TODO:
+
+/*
+- lift user create <name>
+- lift user destroy <name>
+- lift user grant <name> <privilege>
+- lift user revoke <name> <privilege>
+- lift user show <name>
+- lift user list
+
+ */
+//    }
+//}

package/src/expressRoutes/all.ts

@@ -0,0 +1,104 @@
+import path from "path";
+import fs from "fs";
+
+import * as ac from "ansi-colors"
+import {resolvePaths} from "../lib/pathResolve";
+import {gatherFunctionDefinitions} from "../lib/openAPI/ApiBuildCollector";
+
+import express from 'express'
+const router = express.Router();
+
+export function allBinder() {
+    const projectPaths = resolvePaths();
+    const defs = gatherFunctionDefinitions();
+
+    /**
+     * @name /all
+     * @description
+     * Catch-all handler for URLs
+     * Provides some simple spam blocking and checks for file pass through at file root.
+     *
+     * ##### Filtering:
+     * - all .php extension references
+     *
+     * ##### File root:
+     * - file root is at service root + 'Public'
+     * - no path or path ending in / is appended with '/index.html' for folder root default behavior
+     * - File contents passed through using `sendFile` (will handle its own 404 if not found)
+     *
+     */
+    router.all('*', (req, res, next) => {
+
+        // any PHP requests should be ignored
+        // ClogTrace('incoming: '+req.originalUrl)
+        if (req.originalUrl.indexOf('.php') !== -1) {
+            return res.send('')
+        }
+        if (req.originalUrl.indexOf('.env') !== -1) {
+            return res.send('')
+        }
+
+
+
+        let filepath = req.originalUrl || '/'
+        // match cloud behavior for docs because /api doesn't change cwd there
+        if(filepath.substring(0, 10) == '/api/docs/') {
+            filepath = filepath.replace('/api', '');
+        }
+        let funcFound = filepath === '/api'; // reserved "function"
+        if(!funcFound) {
+            if (filepath.charAt(0) === '/') {
+                let n = filepath.indexOf('/', 1);
+                if (n === -1) n = filepath.length;
+                let rootEntity = filepath.substring(0, n)
+                n = rootEntity.indexOf('?');
+                if (n === -1) n = rootEntity.length;
+                rootEntity = rootEntity.substring(0, n);
+                for (const entry of defs) {
+                    let entryRoot = entry.pathMap;
+                    let n = entryRoot.indexOf("/{");
+                    if (n !== -1) entryRoot = entryRoot.substring(0, n);
+                    if (rootEntity === entryRoot) {
+                        funcFound = true;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (!funcFound) {
+            // check for '/' in path beyond the first one
+            if(filepath.indexOf('/',1) !== -1)
+            {
+                // console.log(ac.magenta.italic("Warning: / path delimiters will not be honored in a cloud deployment, use + instead ")+filepath)
+            }
+            // convert any incoming + to / to match cloud behavior
+            while(filepath.indexOf("+")!== -1) filepath = filepath.replace("+", "/")
+
+            // Put any other filtering here
+            if (filepath.indexOf('?') !== -1) {
+                filepath = filepath.substring(0, filepath.indexOf('?'))
+            }
+            if (filepath.substring(filepath.length - 1) === '/') filepath += 'index.html'
+            if(filepath.substring(0, 10) == '/api/docs/') {
+                filepath = filepath.replace('/api', '');
+            }
+            if(filepath.indexOf("/docs") !== -1) {
+                if(filepath.indexOf('.map') !== -1) {
+                    return res.sendStatus(200)
+                }
+            }
+            filepath = path.resolve(path.join(projectPaths.basePath, 'webroot', filepath)) // .. out of build
+            if (fs.existsSync(filepath)) {
+                return res.sendFile(filepath)
+            } else {
+                console.log(ac.red.bold("can't find " + filepath));
+                return res.sendStatus(404);
+            }
+        }
+        // if none of the above, tickle enumerator so we can process other paths
+        next()
+    })
+}
+export default router
+

package/src/expressRoutes/api.ts

@@ -0,0 +1,24 @@
+import express from 'express'
+const router = express.Router();
+
+import {getProjectName, getProjectVersion} from "../lib/LiftVersion";
+
+const openApiUI = require("openapi-ui");
+
+
+router.get('/', function(req, res, next) {
+  res.send(generateApiDoc())
+});
+
+export default router
+
+function generateApiDoc()
+{
+  console.log("---- Serving up api")
+  return openApiUI.generateIndex({
+    baseUrl: "docs",
+    title: `${getProjectName()} ${getProjectVersion()}`,
+    url: "docs/apidoc.yaml"
+  });
+
+}