@tremho/mist-lift 2.2.8 → 2.4.0

package/src/commands/actions/updateDeployedPermissions.ts

@@ -0,0 +1,80 @@
+import * as ac from 'ansi-colors'
+
+import path from 'path'
+import fs from 'fs'
+import { recurseDirectory } from '../../lib/DirectoryUtils'
+import { decoratedName, getAccountId } from '../../lib/IdSrc'
+import { getAWSCredentials, getSettings } from '../../lib/LiftConfig'
+import { resolvePaths } from '../../lib/pathResolve'
+import { LambdaClient, AddPermissionCommand } from '@aws-sdk/client-lambda'
+
+// let projectPaths: { basePath: string, buildPath: string, functionPath: string, packagePath: string, verified: boolean }
+
+export default async function updateDeployedPermissions (apiId: string): Promise<boolean | number | undefined> {
+  const projectPaths = resolvePaths()
+  if (!projectPaths.verified) {
+    console.log(ac.bold.magenta('current directory is not at project root'))
+    return -1
+  }
+
+  const funcsToDeploy: string[] = []
+  let firstDepth = 0
+  recurseDirectory(projectPaths.functionPath, (filepath, stats) => {
+    const depth = filepath.split(path.sep).length
+    if (firstDepth === 0) firstDepth = depth
+    if (stats.isDirectory() && depth === firstDepth) {
+      funcsToDeploy.push(path.basename(filepath))
+    }
+    return false
+  })
+
+  const accountId: string = await getAccountId()
+  const region: string = getSettings().awsPreferredRegion ?? ''
+
+  const client = new LambdaClient(getAWSCredentials())
+
+  for (const funcName of funcsToDeploy) {
+    // read the def file and get the method from there
+    const defPath = path.join(projectPaths.functionPath, funcName, 'src', 'definition.json')
+    const def = JSON.parse(fs.readFileSync(defPath).toString())
+    const method: string = def.method
+    const pathMap: string = def.pathMap
+
+    const arn = generateSourceArnForApiGateway(region, accountId, apiId, method, pathMap)
+    const dname = decoratedName(funcName)
+    console.log(ac.grey.italic('update permissions for resource ' + arn))
+    const command: any = new AddPermissionCommand({
+      FunctionName: dname,
+      StatementId: 'InvokePermission',
+      Action: 'lambda:invokeFunction',
+      Principal: 'apigateway.amazonaws.com',
+      SourceArn: arn
+    })
+    await client.send(command)
+  }
+}
+
+/**
+ * Generates the correct SourceArn to authorize API Gateway to invoke a Lambda function.
+ *
+ * @param region - The AWS region, e.g., 'us-west-1'
+ * @param accountId - Your AWS account ID
+ * @param apiId - The API Gateway ID, e.g., 'sfr7yltmc3'
+ * @param method - The HTTP method, e.g., 'POST'
+ * @param routePath - The API route path, e.g., '/upstart/{artistId}/{id}'
+ * @returns The complete SourceArn string
+ */
+export function generateSourceArnForApiGateway (
+  region: string,
+  accountId: string,
+  apiId: string,
+  method: string,
+  routePath: string
+): string {
+  // Clean leading slashes and replace path params with wildcards
+  const cleanedPath = routePath
+    .replace(/^\/+/, '') // remove leading slashes
+    .replace(/{[^}]+}/g, '*') // replace path params with *
+
+  return `arn:aws:execute-api:${region}:${accountId}:${apiId}/*/${method.toUpperCase()}/${cleanedPath}`
+}

package/src/commands/builtin/ApiDocMaker.ts

@@ -5,6 +5,7 @@ import { GetWebrootServePaths } from '../../lib/openAPI/WebrootFileSupport'
 import path from 'path'
 // import * as ac from 'ansi-colors'
 import { decoratedName } from '../../lib/IdSrc'
+import { getWebrootSettings } from './ExportWebroot'
 
 export async function MakePublicApiDoc
 (
@@ -22,29 +23,32 @@ export async function MakeBuiltinApiDoc
   const defs = gatherFunctionDefinitions()
   // console.log(ac.gray.dim('>> after gatherfunctions'), defs)
   // console.log(ac.gray.dim('>>> addBuiltInDefinitions '))
-  addBuiltInDefinitions(defs)
+  const wrs = await getWebrootSettings()
+  const withWebroot = (wrs.webrootMethod ?? 'SELF') === 'SELF'
+  // console.log("wrs.webrootMethod=",wrs.webrootMethod)
+  addBuiltInDefinitions(defs, withWebroot)
   // console.log(ac.gray.dim('>> after addBuiltIns'), defs)
   // console.log(ac.gray.dim('>>> buildOpenApi '))
-  return await buildOpenApi(defs, false, yamlFile) //, true)
+  return await buildOpenApi(defs, true, true, yamlFile)
 }
 
-export function addBuiltInDefinitions (defs: any[]): void {
+export function addBuiltInDefinitions (defs: any[], withWebroot: boolean): void {
   // console.warn("NOT ADDING ANY BUILTIN API INTEGRATIONS")
   // console.log("ADDING apiDef");
   // console.log(ac.gray.dim('>>>> pushing apiDef '), apiDef)
   defs.push(apiDef)
+  // console.log("withWebroot = "+withWebroot)
   // console.log(ac.gray.dim('>>>> pushing webrootDef '), webrootDef)
-  defs.push(webrootDef)
+  defs.push(webrootDef) // we need to do this to support index or anything else.
 
   // console.log(ac.gray.dim('>>>> Adding webroot literals '))
   // console.warn("Adding webroot literals");
   // do just /docs and see how that goes
-  /*
-  const fsdef = Object.assign({}, fileServeDef) // copy
-  fsdef.name = decoratedName('fileserve_docs')
-  fsdef.pathMap = '/docs/{path}'
-  defs.push(fsdef)
-   */
+  // const fsdef = Object.assign({}, fileServeDef) // copy
+  // fsdef.name = decoratedName('fileserve_docs')
+  // fsdef.pathMap = '/docs/{path}'
+  // defs.push(fsdef)
+  // }
   const roots = GetWebrootServePaths()
   // console.log("roots", roots)
   // console.log(ac.gray.dim('>>>> roots here: '), roots)
@@ -55,6 +59,9 @@ export function addBuiltInDefinitions (defs: any[]): void {
       let rootName = rootPath
       // console.log(ac.gray.dim('>> rootName = '+rootName))
       while (rootName.includes('/')) rootName = rootName.replace('/', '').toLowerCase().trim()
+      // TODO: Trying to exclude here seems to remove our ability to get anything from root (webroot, should be there already)
+      // console.log('fileserve '+ rootName)
+      // if(!withWebroot && rootName !== 'docs') continue; // only copy docs for non-self export types
       // console.log(ac.gray.dim('>> past stupid error = '+rootName))
       const fileserve = Object.assign({}, fileServeDef) // copy
       fileserve.name = decoratedName('fileserve_' + rootName)

package/src/commands/builtin/BuiltInHandler.ts

@@ -9,11 +9,16 @@ import path from 'path'
 
 export async function DeployWebrootBuiltIn
 (
+  externalizedFolders: boolean = true
 ): Promise<void> {
   // console.warn(">> DeployWebrootBuiltIn")
+  // const withRoot = externalizedFolders
+  // const withFolders = true
   const wrZip = await StageWebrootZip()
-  // console.log(">> Deploy Webroot Builtin from "+wrZip)
-  await DeployBuiltInZip('Webroot', wrZip)
+  const wrName = decoratedName('Webroot')
+  // console.log(">> Deploy Webroot Builtin from "+wrZip+" as "+wrName)
+
+  await DeployBuiltInZip(wrName, wrZip)
   // remove temp zip when done
   // console.warn("a.zip is left behind")
   fs.rmSync(wrZip, { recursive: true })

package/src/commands/builtin/ExportWebroot.ts

@@ -0,0 +1,195 @@
+import * as path from 'path'
+import * as fs from 'fs'
+import * as ac from 'ansi-colors'
+import { resolvePaths } from '../../lib/pathResolve'
+import { recurseDirectory } from '../../lib/DirectoryUtils'
+// import { isNewer } from '../../lib/fileCompare'
+import { executeCommand } from '../../lib/executeCommand'
+// import { mkdirSync } from 'fs'
+import { DeployRootFileserves, DeployWebrootBuiltIn } from './BuiltInHandler'
+// import { ServiceSettingsData } from '@tremho/inverse-y'
+import { exportWebrootToS3, getS3Prefix } from './webroot-export/s3webroot'
+
+/*
+Here we want to handle exporting the webroot to other serving sources according to configuration.
+
+- Read the config  options
+- enumerate the webroot and find recent changes
+- write these file paths to a temp file
+- per config, call the handler that will do this
+    - to S3
+    - to script
+    later:
+    - to Blob ?
+    -
+
+ */
+
+/*
+Script will be called with comma delimited list of files as arg 1, and options (string) as arg2
+ */
+export class WebrootScriptOptions {
+  public scriptName = '' // name of script to call (relative to project root)
+  public options = '' // options passed as second parameter to script (see comment above)
+}
+export class WebrootS3Options {
+  public unused = '' // empty (lint)
+}
+export class WebrootDefaultOptions {
+  public unused = '' // empty (lint)
+}
+
+export type WebrootOptions = WebrootDefaultOptions | WebrootScriptOptions | WebrootS3Options
+
+/*
+For non-SELF types, only the files *not* in subdirectories will be packaged to the deployed webroot
+ */
+export enum WebrootExportType {
+  SELF = 'SELF', // Webroot is packaged and deployed per existing implementation, including all subdirectories
+  SCRIPT = 'SCRIPT', // a build-time script is called to copy all updated files to their target
+  S3 = 'S3' // built-in function (instead of script) is called for an S3 target
+}
+
+/*
+This is what webrootConfig.json looks like
+ */
+export class WebrootExportConfig {
+  public type: WebrootExportType = WebrootExportType.SELF
+  public options: WebrootOptions = new WebrootDefaultOptions()
+}
+
+let basePath = ''
+
+/**
+ * Exports the webroot according to configuration
+ */
+export async function ExportWebroot (): Promise<any> {
+  const projectPaths = resolvePaths()
+  basePath = projectPaths.basePath
+  const webrootConfig = readWebrootConfig(path.join(basePath, 'webrootConfig.json'))
+  // console.log(">> ExportWebroot", {webrootConfig})
+
+  const fla = await getUpdatedExportFileList()
+  switch (webrootConfig.type) {
+    case WebrootExportType.SELF:
+      await exportSelf()
+      break
+    case WebrootExportType.S3:
+      await exportWebrootToS3(fla, webrootConfig.options as WebrootS3Options)
+      break
+    case WebrootExportType.SCRIPT:
+      await exportScript(fla, webrootConfig.options as WebrootScriptOptions)
+      break
+    default: {
+      throw new Error(`UNSUPPORTED WEBROOT EXPORT ${WebrootExportType[webrootConfig.type]}`)
+    }
+  }
+}
+export async function getWebrootSettings (): Promise<{ webrootMethod: string, webrootBaseUrl: string }> {
+  const projectPaths = resolvePaths()
+  basePath = projectPaths.basePath
+  const webrootConfig = readWebrootConfig(path.join(basePath, 'webrootConfig.json'))
+  let baseUrl = ''
+  switch (webrootConfig.type) {
+    case WebrootExportType.SELF:
+      baseUrl = ''
+      break
+    case WebrootExportType.S3:
+      baseUrl = await getS3Prefix()
+      break
+    case WebrootExportType.SCRIPT: {
+      const options = webrootConfig.options as WebrootScriptOptions
+      const scriptPath = path.join(basePath, options.scriptName)
+      const out = await executeCommand(scriptPath, ['--prefix'], basePath, false)
+      baseUrl = out.stdStr.trim()
+      break
+    }
+    default: {
+      const unreachable: never = webrootConfig.type
+      throw new Error(`UNSUPPORTED WEBROOT EXPORT ${unreachable}`)
+    }
+  }
+  return {
+    webrootMethod: webrootConfig.type.toString(),
+    webrootBaseUrl: baseUrl
+  }
+}
+
+let warned = false
+function readWebrootConfig (configPath: string): WebrootExportConfig {
+  let conf = new WebrootExportConfig()
+  let content = ''
+  // console.log("... reading from "+configPath)
+  try {
+    // console.log("  >> readWebrootConfig")
+    content = fs.readFileSync(configPath).toString()
+    conf = JSON.parse(content)
+  } catch (e: any) {
+    if (!warned) {
+      warned = true
+      console.error(ac.yellow.bold('Warning: Failed to read or parse ' + configPath))
+      console.warn(ac.yellow.bold.dim("Falling back to 'self' publish mode"))
+    }
+  }
+
+  return conf
+}
+
+async function exportSelf (): Promise<void> {
+  // zip up everything
+  // console.log(">> exportSelf")
+  // console.log("  >> deployWebrootBuiltIn")
+  await DeployWebrootBuiltIn(false)
+  // console.log("  >> deployRootFileserves")
+  await DeployRootFileserves()
+}
+
+async function exportScript (fla: string[], options: WebrootScriptOptions) {
+  // console.log(">> exportScript")
+  // console.log("  >> DeployWebrootBuiltIn")
+  // await DeployWebrootBuiltIn(true)
+  console.log(ac.bold.green(`calling webroot exportScript ${options.scriptName}`))
+  // console.log("files", fla)
+  const fl = fla.join(',')
+  const args = [fl]
+  // args.unshift(options.options)
+  // console.log("args", args)
+
+  const scriptPath = path.join(basePath, options.scriptName)
+  // console.log("Executing script", scriptPath)
+  await executeCommand(scriptPath, args, basePath, true)
+}
+
+async function getUpdatedExportFileList (): Promise<string[]> {
+  // console.log(">> getUpdatedExportFileList")
+  // find the last published time from .published file
+  let lastPublished = 0
+  try {
+    const pubjson = JSON.parse(fs.readFileSync(path.join(basePath, '.published')).toString())
+    lastPublished = pubjson.time
+    // console.log("last published ", lastPublished)
+  } catch (e: any) {
+    console.warn(ac.yellow.italic.dim('no .published information available - full export'))
+  }
+  const outbin: string[] = []
+  // enumerate at webroot and files from all non-root directories
+  const folder = path.join(basePath, 'webroot')
+  const rootpathsteps = folder.split(path.sep).length
+  const enumerating = true
+
+  function gatherFiles (filepath: string, stats: fs.Stats): boolean {
+    // console.log(filepath, {steps:filepath.split(path.sep).length, rootpathsteps, isDir:stats.isDirectory()})
+    if (filepath.split(path.sep).length > rootpathsteps + 1) {
+      // console.log({filepath, mtime:stats.mtime.getTime(), lastPublished})
+      if (stats.mtime.getTime() >= lastPublished) {
+        filepath = '+' + filepath
+        // console.log("will update "+filepath)
+      }
+      outbin.push(filepath)
+    }
+    return false
+  }
+  // console.log("recursing folder ", folder)
+  await recurseDirectory(folder, gatherFiles)
+  return outbin
+}

package/src/commands/builtin/StageWebrootZip.ts

@@ -11,6 +11,8 @@ import { rmSync } from 'fs'
 
 export async function StageWebrootZip
 (
+  withRoot: boolean = true,
+  withFolders: boolean = true
 ): Promise<string> {
   // console.log(">>>> staging webroot to zip")
   const projectPaths = resolvePaths()
@@ -43,6 +45,7 @@ export async function StageWebrootZip
   await fs.mkdirSync(zipFilesPath)
 
   // todo: read the redirects.json file
+  // todo: Document as deprecated
   let redirected: string[] = []
   try {
     const redir = JSON.parse(fs.readFileSync(path.join(webroot, 'redirects.json')).toString())
@@ -55,13 +58,18 @@ export async function StageWebrootZip
   // console.log(">>>> enumerating "+webroot)
   await recurseDirectory(webroot, (filepath, stats) => {
     const relPath = filepath.substring(webroot.length)
-    if (stats.isDirectory()) {
+    if (stats.isDirectory() && withFolders) {
       fs.mkdirSync(zipFilesPath + relPath, { recursive: true })
     } else {
-      // console.log(ac.grey.dim('>> considering ' + relPath))
-      if (!redirected.includes(relPath.substring(1))) {
-        // console.log(ac.grey.dim('>> copying...'))
-        fs.copyFileSync(filepath, zipFilesPath + relPath) // copy only if not in redirected list
+      if (
+        (relPath.includes(path.sep) && withFolders) ||
+         ((!relPath.includes(path.sep)) && withRoot)
+      ) {
+        // TODO: Deprecate redirection. we'll continue to supoort it  here, though for now
+        if (!redirected.includes(relPath.substring(1))) {
+          // console.log(ac.grey.dim('>> copying...'))
+          fs.copyFileSync(filepath, zipFilesPath + relPath) // copy only if not in redirected list
+        }
       }
     }
     return false

package/src/commands/builtin/webroot-export/s3webroot.ts

@@ -0,0 +1,78 @@
+import { S3Client } from '@aws-sdk/client-s3'
+import { s3ListObjects, s3CreateBucket, s3UploadFile, s3Delete, S3ActionsLog } from '@tremho/basic-s3-actions'
+import * as fs from 'fs'
+import * as ac from 'ansi-colors'
+import { WebrootS3Options, WebrootScriptOptions } from '../ExportWebroot'
+import { DeployRootFileserves, DeployWebrootBuiltIn } from '../BuiltInHandler'
+
+export async function exportWebrootToS3 (exportFileList: string[], options?: WebrootS3Options) {
+  const bucketName = determineBucketName()
+  console.log(ac.green.bold('webroot export to s3 bucket name ' + bucketName))
+  let existing: string[] = []
+  try {
+    // console.log("fetching existing")
+    existing = await getExistingDeployedFileList(bucketName)
+  } catch (e: any) {
+    console.log(ac.blue.italic('creating webroot bucket ' + bucketName))
+    await s3CreateBucket(bucketName, true)
+    // console.log("bucket created")
+  }
+
+  console.log(ac.blue.italic('Deploying base webroot handling'))
+  await DeployWebrootBuiltIn(true)
+  await DeployRootFileserves()
+
+  // console.log("existing ", existing)
+
+  let update = false
+  for (let file of exportFileList) {
+    update = (file.charAt(0) == '+')
+    if (update) file = file.substring(1)
+    const i = file.indexOf('/webroot/') + 9
+    const sfile = file.substring(i)
+    if (!sfile) continue
+    if (sfile.substring(0, 4) === 'docs') continue // keep these in self-serve only
+    const xi = existing.indexOf(sfile)
+    let disp = 'created'
+    if (xi !== -1) {
+      existing.splice(xi, 1)
+      disp = 'updated'
+    }
+    if (update) {
+      // console.log(sfile+"...")
+      try {
+        await s3UploadFile(bucketName, file, sfile)
+        console.log(ac.green.italic(`${disp} ${sfile}`))
+      } catch (e: any) {
+        break
+      }
+    }
+  }
+  for (const leftover of existing) {
+    await s3Delete(bucketName, leftover)
+    console.log(ac.green.italic(`deleted ${leftover}`))
+  }
+}
+
+function determineBucketName () {
+  let pkg: any = {}
+  try {
+    pkg = JSON.parse(fs.readFileSync('./package.json').toString())
+  } catch (e: any) {
+    console.error(ac.bold.red('Failed to read package.json ' + e.message))
+  }
+  return (pkg.author + '-' + pkg.name).toLowerCase()
+}
+
+async function getExistingDeployedFileList (bucketName: string) {
+  // S3ActionsLog.setMinimumLevel('Console', 'trace')
+  // S3ActionsLog.Debug("Getting bucket list for "+bucketName)
+  const list = await s3ListObjects(bucketName)
+  // S3ActionsLog.setMinimumLevel('Console', 'error')
+  return list
+}
+
+export function getS3Prefix () {
+  const name = determineBucketName()
+  return `https://${name}.s3.us-west-1.amazonaws.com/`
+}

package/src/commands/deploy.ts

@@ -106,11 +106,11 @@ export async function deployPackage (
   try {
     def = JSON.parse(fs.readFileSync(defFile).toString())
   } catch (e: any) {
-    // hack for assigning a definition -- the defs from ApiDocMaker don't appear here
-    if (funcName === 'Webroot') {
+    // hack for assigning a definition -- the defs from the MistBuiltIn def file don't appear here
+    if (funcName.includes('Webroot')) {
       def.name = 'Webroot'
       def.timeoutSeconds = 8
-      def.memorySize = 256
+      def.memorySize = 8192
     }
   }
   const timeout: number = def.timeoutSeconds ?? 0 // zero will mean default (3 seconds on AWS)
@@ -135,7 +135,9 @@ export async function deployPackage (
     const response: any = await CreateCloudFunction(dFuncName, zipFile, timeout, memorySize)
     const parts = response.FunctionArn.split(':')
     const principal = parts[4]
-    await AddPermissions(client, dFuncName, principal)
+    if (dFuncName.includes('Webroot') || dFuncName.includes('FileServe')) {
+      await AddPermissions(client, dFuncName, principal)
+    }
     let deployMsg: string = `Successfully deployed ${funcName}`
     // console.log(ac.gray.dim('>> deploy trace 1'))
     if (memorySize > 0 || timeout > 0) {

package/src/commands/package.ts

@@ -11,6 +11,8 @@ import { executeCommand } from '../lib/executeCommand'
 import { isNewerFile } from '../lib/fileCompare'
 import { doBuildAsync } from './build'
 import { FolderToZip } from '../lib/utils'
+import { getWebrootSettings } from './builtin/ExportWebroot'
+import { getServiceSettings, setAws, setWebroot } from '@tremho/inverse-y'
 
 // test then package
 export async function doPackageAsync (
@@ -123,6 +125,8 @@ async function packageFunction (funcName: string): Promise<number> {
   //  - write out new package.json to workPath
   // console.log("writing package.json", pkgjson)
   fs.writeFileSync(path.join(workPath, 'package.json'), JSON.stringify(pkgjson, null, 2))
+
+  const webrootExport = await getWebrootSettings()
   //  - execute npm i at workpath, create node_modules
   await executeCommand('npm i', [], workPath).then(() => {
     // copy the lambda function
@@ -148,10 +152,37 @@ async function packageFunction (funcName: string): Promise<number> {
     const runmainPath = path.join(workPath, 'runmain.mjs')
     fs.writeFileSync(runmainPath, fs.readFileSync(templatePath))
 
+    // put svcsettings.json into place
+
+    // Get publish Url from existing .published file (if any).
+    // If we haven't published before, we won't have this, but I think that's okay for now at least
+
+    try {
+      // console.log("creating service settings file")
+      const pubsrc = path.join(workPath, '..', '.published')
+      let pubInfo: any = {}
+      // console.log("checking pubsrc at ", pubsrc)
+      if (fs.existsSync(pubsrc)) {
+        // console.log("found it")
+        pubInfo = JSON.parse(fs.readFileSync(pubsrc).toString())
+        // console.log({pubInfo})
+      }
+      const publishUrl = pubInfo.url ?? '/'
+
+      const stage = publishUrl.substring(publishUrl.lastIndexOf('/') + 1)
+      setWebroot(webrootExport.webrootMethod, webrootExport.webrootBaseUrl)
+      setAws(stage, publishUrl)
+      // console.log("getServiceSettings", getServiceSettings())
+      const settingsFile = path.join(projectPaths.basePath, '.package_temp', 'svcsettings.json')
+      fs.writeFileSync(settingsFile, JSON.stringify(getServiceSettings()))
+      // if (fs.existsSync(settingsFile)) console.log("file confirmed at ", settingsFile)
+    } catch (e: any) {
+      console.error(e)
+    }
+    // console.log("push to zip stack ", {zipFile})
     all.push(FolderToZip(workPath, zipFile))
-    // console.log("now zip it")
   })
-
+  // console.log("all", {all})
   return await Promise.all(all).then(() => {
     return error
   })